Mirrors/fish-shell
2
0
Fork 0
mirror of https://github.com/fish-shell/fish-shell synced 2024-12-27 05:13:10 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'master' into parser_cleanup

Browse source 
Conflicts:
	parse_constants.h
	parse_tree.h
This commit is contained in:
ridiculousfish 2014-03-26 13:59:14 -07:00
commit f2a437bd3b
16 changed files with 954 additions and 399 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Makefile.in
View file

@ -91,7 +91,7 @@ FISH_OBJS := function.o builtin.o complete.o env.o exec.o expand.o \
signal.o io.o parse_util.o common.o screen.o path.o autoload.o \ signal.o io.o parse_util.o common.o screen.o path.o autoload.o \
parser_keywords.o iothread.o color.o postfork.o \ parser_keywords.o iothread.o color.o postfork.o \
builtin_test.o parse_tree.o parse_productions.o parse_execution.cpp \ builtin_test.o parse_tree.o parse_productions.o parse_execution.cpp \
pager.cpp pager.cpp utf8.o
FISH_INDENT_OBJS := fish_indent.o print_help.o common.o \ FISH_INDENT_OBJS := fish_indent.o print_help.o common.o \
parser_keywords.o wutil.o tokenizer.o parser_keywords.o wutil.o tokenizer.o
@ -117,7 +117,7 @@ FISH_TESTS_OBJS := $(FISH_OBJS) fish_tests.o
# #
FISHD_OBJS := fishd.o env_universal_common.o wutil.o print_help.o \ FISHD_OBJS := fishd.o env_universal_common.o wutil.o print_help.o \
common.o common.o utf8.o
# #

5
builtin_set_color.cpp
View file

@ -235,9 +235,8 @@ static int builtin_set_color(parser_t &parser, wchar_t **argv)
output_set_writer(saved_writer_func); output_set_writer(saved_writer_func);
/* Output the collected string */ /* Output the collected string */
std::string local_output; stdout_buffer.append(str2wcstring(builtin_set_color_output));
std::swap(builtin_set_color_output, local_output); builtin_set_color_output.clear();
stdout_buffer.append(str2wcstring(local_output));
return STATUS_BUILTIN_OK; return STATUS_BUILTIN_OK;
} }

11
configure.ac
View file

@ -105,8 +105,6 @@ echo "CXXFLAGS: $CXXFLAGS"
# #
# This mostly helps OS X users, since fink usually installs out of # This mostly helps OS X users, since fink usually installs out of
# tree and doesn't update CXXFLAGS. # tree and doesn't update CXXFLAGS.
#
# It also helps FreeBSD which puts libiconv in /usr/local/lib
for i in /usr/pkg /sw /opt /opt/local /usr/local; do for i in /usr/pkg /sw /opt /opt/local /usr/local; do
@ -151,9 +149,11 @@ AC_CONFIG_HEADERS(config.h)
AH_BOTTOM([#if __GNUC__ >= 3 AH_BOTTOM([#if __GNUC__ >= 3
#define __warn_unused __attribute__ ((warn_unused_result)) #define __warn_unused __attribute__ ((warn_unused_result))
#define __sentinel __attribute__ ((sentinel)) #define __sentinel __attribute__ ((sentinel))
#define __packed __attribute__ ((packed))
#else #else
#define __warn_unused #define __warn_unused
#define __sentinel #define __sentinel
#define __packed
#endif]) #endif])
@ -407,7 +407,7 @@ AC_DEFINE(
AC_SEARCH_LIBS( connect, socket, , [AC_MSG_ERROR([Cannot find the socket library, needed to build this package.] )] ) AC_SEARCH_LIBS( connect, socket, , [AC_MSG_ERROR([Cannot find the socket library, needed to build this package.] )] )
AC_SEARCH_LIBS( nanosleep, rt, , [AC_MSG_ERROR([Cannot find the rt library, needed to build this package.] )] ) AC_SEARCH_LIBS( nanosleep, rt, , [AC_MSG_ERROR([Cannot find the rt library, needed to build this package.] )] )
AC_SEARCH_LIBS( pthread_create, pthread, , [AC_MSG_ERROR([Cannot find the pthread library, needed to build this package.] )] ) AC_SEARCH_LIBS( pthread_create, pthread, , [AC_MSG_ERROR([Cannot find the pthread library, needed to build this package.] )] )
AC_SEARCH_LIBS( setupterm, [ncurses curses], , [AC_MSG_ERROR([Could not find a curses implementation, needed to build fish. If this is Linux, try running 'sudo apt-get install libncurses5-dev' or 'sudo yum install ncurses-devel'])] ) AC_SEARCH_LIBS( setupterm, [ncurses tinfo curses], , [AC_MSG_ERROR([Could not find a curses implementation, needed to build fish. If this is Linux, try running 'sudo apt-get install libncurses5-dev' or 'sudo yum install ncurses-devel'])] )
AC_SEARCH_LIBS( [nan], [m], [AC_DEFINE( [HAVE_NAN], [1], [Define to 1 if you have the nan function])] ) AC_SEARCH_LIBS( [nan], [m], [AC_DEFINE( [HAVE_NAN], [1], [Define to 1 if you have the nan function])] )
if test x$local_gettext != xno; then if test x$local_gettext != xno; then
@ -421,10 +421,6 @@ LIBS_SHARED=$LIBS
# #
LIBS="$LIBS_SHARED" LIBS="$LIBS_SHARED"
# Check for libiconv_open if we can't find iconv_open. Silly OS X does
# weird macro magic for the sole purpose of amusing me.
AC_SEARCH_LIBS( iconv_open, iconv, , [AC_SEARCH_LIBS( libiconv_open, iconv, , [AC_MSG_ERROR([Could not find an iconv implementation, needed to build fish])] )] )
LIBS_FISH=$LIBS LIBS_FISH=$LIBS
# #
@ -439,7 +435,6 @@ LIBS_FISH_INDENT=$LIBS
# #
LIBS="$LIBS_SHARED" LIBS="$LIBS_SHARED"
AC_SEARCH_LIBS( iconv_open, iconv, , [AC_SEARCH_LIBS( libiconv_open, iconv, , [AC_MSG_ERROR([Could not find an iconv implementation, needed to build fish])] )] )
LIBS_FISHD=$LIBS LIBS_FISHD=$LIBS
# #

16
doc_src/license.hdr
View file

@ -1402,4 +1402,20 @@ POSSIBILITY OF SUCH DAMAGES.
*/ */
<h2>License for UTF8</h2>
<p>Copyright (c) 2007 Alexey Vatchenko <av@bsdua.org>
<p>Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
<p>THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
\htmlonly </div> \endhtmlonly \htmlonly </div> \endhtmlonly

308
env_universal_common.cpp
View file

@ -21,7 +21,6 @@
#include <sys/stat.h> #include <sys/stat.h>
#include <dirent.h> #include <dirent.h>
#include <wctype.h> #include <wctype.h>
#include <iconv.h>
#include <errno.h> #include <errno.h>
#include <locale.h> #include <locale.h>
@ -39,6 +38,7 @@
#include "common.h" #include "common.h"
#include "wutil.h" #include "wutil.h"
#include "utf8.h"
#include "env_universal_common.h" #include "env_universal_common.h"
/** /**
@ -116,303 +116,28 @@ static void (*callback)(fish_message_type_t type,
const wchar_t *key, const wchar_t *key,
const wchar_t *val); const wchar_t *val);
/** /* UTF <-> wchar conversions. These return a string allocated with malloc. These call sites could be cleaned up substantially to eliminate the dependence on malloc. */
List of names for the UTF-8 character set. static wchar_t *utf2wcs(const char *input)
*/
static const char *iconv_utf8_names[]=
{ {
"utf-8", "UTF-8", wchar_t *result = NULL;
"utf8", "UTF8", wcstring converted;
0 if (utf8_to_wchar_string(input, &converted))
{
result = wcsdup(converted.c_str());
} }
; return result;
/**
List of wide character names, undefined byte length.
*/
static const char *iconv_wide_names_unknown[]=
{
"wchar_t", "WCHAR_T",
"wchar", "WCHAR",
0
}
;
/**
List of wide character names, 4 bytes long.
*/
static const char *iconv_wide_names_4[]=
{
"wchar_t", "WCHAR_T",
"wchar", "WCHAR",
"ucs-4", "UCS-4",
"ucs4", "UCS4",
"utf-32", "UTF-32",
"utf32", "UTF32",
0
}
;
/**
List of wide character names, 2 bytes long.
*/
static const char *iconv_wide_names_2[]=
{
"wchar_t", "WCHAR_T",
"wchar", "WCHAR",
"ucs-2", "UCS-2",
"ucs2", "UCS2",
"utf-16", "UTF-16",
"utf16", "UTF16",
0
}
;
template<class T>
class sloppy {};
static size_t hack_iconv(iconv_t cd, const char * const* inbuf, size_t *inbytesleft, char **outbuf, size_t *outbytesleft)
{
/* FreeBSD has this prototype: size_t iconv (iconv_t, const char **...)
OS X and Linux this one: size_t iconv (iconv_t, char **...)
AFAIK there's no single type that can be passed as both char ** and const char **.
Therefore, we let C++ figure it out, by providing a struct with an implicit conversion to both char** and const char **.
*/
struct sloppy_char
{
const char * const * t;
operator char** () const
{
return (char **)t;
}
operator const char** () const
{
return (const char**)t;
}
} slop_inbuf = {inbuf};
return iconv(cd, slop_inbuf, inbytesleft, outbuf, outbytesleft);
} }
/** static char *wcs2utf(const wchar_t *input)
Convert utf-8 string to wide string
*/
static wchar_t *utf2wcs(const char *in)
{ {
iconv_t cd=(iconv_t) -1; char *result = NULL;
int i,j; std::string converted;
if (wchar_to_utf8_string(input, &converted))
wchar_t *out;
/*
Try to convert to wchar_t. If that is not a valid character set,
try various names for ucs-4. We can't be sure that ucs-4 is
really the character set used by wchar_t, but it is the best
assumption we can make.
*/
const char **to_name=0;
switch (sizeof(wchar_t))
{ {
result = strdup(converted.c_str());
case 2:
to_name = iconv_wide_names_2;
break;
case 4:
to_name = iconv_wide_names_4;
break;
default:
to_name = iconv_wide_names_unknown;
break;
} }
return result;
/*
The line protocol fish uses is always utf-8.
*/
const char **from_name = iconv_utf8_names;
size_t in_len = strlen(in);
size_t out_len = sizeof(wchar_t)*(in_len+2);
size_t nconv;
char *nout;
out = (wchar_t *)malloc(out_len);
nout = (char *)out;
if (!out)
return 0;
for (i=0; to_name[i]; i++)
{
for (j=0; from_name[j]; j++)
{
cd = iconv_open(to_name[i], from_name[j]);
if (cd != (iconv_t) -1)
{
goto start_conversion;
} }
}
}
start_conversion:
if (cd == (iconv_t) -1)
{
/* Something went wrong. */
debug(0, L"Could not perform utf-8 conversion");
if (errno != EINVAL)
wperror(L"iconv_open");
/* Terminate the output string. */
free(out);
return 0;
}
/* FreeBSD has this prototype: size_t iconv (iconv_t, const char **...)
OS X and Linux this one: size_t iconv (iconv_t, char **...)
AFAIK there's no single type that can be passed as both char ** and const char **.
Hence this hack.
*/
nconv = hack_iconv(cd, &in, &in_len, &nout, &out_len);
if (nconv == (size_t) -1)
{
debug(0, L"Error while converting from utf string");
return 0;
}
*((wchar_t *) nout) = L'\0';
/*
Check for silly iconv behaviour inserting an bytemark in the output
string.
*/
if (*out == L'\xfeff' || *out == L'\xffef' || *out == L'\xefbbbf')
{
wchar_t *out_old = out;
out = wcsdup(out+1);
if (! out)
{
debug(0, L"FNORD!!!!");
free(out_old);
return 0;
}
free(out_old);
}
if (iconv_close(cd) != 0)
wperror(L"iconv_close");
return out;
}
/**
Convert wide string to utf-8
*/
static char *wcs2utf(const wchar_t *in)
{
iconv_t cd=(iconv_t) -1;
int i,j;
char *char_in = (char *)in;
char *out;
/*
Try to convert to wchar_t. If that is not a valid character set,
try various names for ucs-4. We can't be sure that ucs-4 is
really the character set used by wchar_t, but it is the best
assumption we can make.
*/
const char **from_name=0;
switch (sizeof(wchar_t))
{
case 2:
from_name = iconv_wide_names_2;
break;
case 4:
from_name = iconv_wide_names_4;
break;
default:
from_name = iconv_wide_names_unknown;
break;
}
const char **to_name = iconv_utf8_names;
size_t in_len = wcslen(in);
size_t out_len = sizeof(char)*((MAX_UTF8_BYTES*in_len)+1);
size_t nconv;
char *nout;
out = (char *)malloc(out_len);
nout = (char *)out;
in_len *= sizeof(wchar_t);
if (!out)
return 0;
for (i=0; to_name[i]; i++)
{
for (j=0; from_name[j]; j++)
{
cd = iconv_open(to_name[i], from_name[j]);
if (cd != (iconv_t) -1)
{
goto start_conversion;
}
}
}
start_conversion:
if (cd == (iconv_t) -1)
{
/* Something went wrong. */
debug(0, L"Could not perform utf-8 conversion");
if (errno != EINVAL)
wperror(L"iconv_open");
/* Terminate the output string. */
free(out);
return 0;
}
nconv = hack_iconv(cd, &char_in, &in_len, &nout, &out_len);
if (nconv == (size_t) -1)
{
debug(0, L"%d %d", in_len, out_len);
debug(0, L"Error while converting from to string");
/* Terminate the output string. */
free(out);
return 0;
}
*nout = '\0';
if (iconv_close(cd) != 0)
wperror(L"iconv_close");
return out;
}
void env_universal_common_init(void (*cb)(fish_message_type_t type, const wchar_t *key, const wchar_t *val)) void env_universal_common_init(void (*cb)(fish_message_type_t type, const wchar_t *key, const wchar_t *val))
{ {
@ -756,8 +481,9 @@ static wcstring full_escape(const wchar_t *in)
{ {
out.push_back(c); out.push_back(c);
} }
else if (c < 256) else if (c <= ASCII_MAX)
{ {
// See #1225 for discussion of use of ASCII_MAX here
append_format(out, L"\\x%.2x", c); append_format(out, L"\\x%.2x", c);
} }
else if (c < 65536) else if (c < 65536)

18
fish.xcodeproj/project.pbxproj
View file

@ -113,10 +113,11 @@
D08A32B917B446B100F3A533 /* parse_productions.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */; }; D08A32B917B446B100F3A533 /* parse_productions.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */; };
D08A32BA17B446B100F3A533 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; }; D08A32BA17B446B100F3A533 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; };
D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; }; D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; };
D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; };
D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; }; D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; };
D0C9733818DE5449002D7C81 /* utf8.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C9733718DE5449002D7C81 /* utf8.cpp */; };
D0C9733918DE5449002D7C81 /* utf8.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C9733718DE5449002D7C81 /* utf8.cpp */; };
D0CBD587159EF0E10024809C /* launch_fish.scpt in Resources */ = {isa = PBXBuildFile; fileRef = D0CBD586159EF0E10024809C /* launch_fish.scpt */; }; D0CBD587159EF0E10024809C /* launch_fish.scpt in Resources */ = {isa = PBXBuildFile; fileRef = D0CBD586159EF0E10024809C /* launch_fish.scpt */; };
D0D02A67159837AD008E62BD /* complete.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853713B3ACEE0099B651 /* complete.cpp */; }; D0D02A67159837AD008E62BD /* complete.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853713B3ACEE0099B651 /* complete.cpp */; };
D0D02A69159837B2008E62BD /* env.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853A13B3ACEE0099B651 /* env.cpp */; }; D0D02A69159837B2008E62BD /* env.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853A13B3ACEE0099B651 /* env.cpp */; };
@ -152,7 +153,6 @@
D0D02A87159839D5008E62BD /* screen.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855A13B3ACEE0099B651 /* screen.cpp */; }; D0D02A87159839D5008E62BD /* screen.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855A13B3ACEE0099B651 /* screen.cpp */; };
D0D02A88159839D5008E62BD /* signal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855C13B3ACEE0099B651 /* signal.cpp */; }; D0D02A88159839D5008E62BD /* signal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855C13B3ACEE0099B651 /* signal.cpp */; };
D0D02A89159839DF008E62BD /* fish.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854213B3ACEE0099B651 /* fish.cpp */; }; D0D02A89159839DF008E62BD /* fish.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854213B3ACEE0099B651 /* fish.cpp */; };
D0D02A8B15983CDF008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
D0D02A8D15983CFA008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; }; D0D02A8D15983CFA008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
D0D02A8F15983D8F008E62BD /* parser_keywords.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855313B3ACEE0099B651 /* parser_keywords.cpp */; }; D0D02A8F15983D8F008E62BD /* parser_keywords.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855313B3ACEE0099B651 /* parser_keywords.cpp */; };
D0D02AC215985F3F008E62BD /* fishd.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854313B3ACEE0099B651 /* fishd.cpp */; }; D0D02AC215985F3F008E62BD /* fishd.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854313B3ACEE0099B651 /* fishd.cpp */; };
@ -161,7 +161,6 @@
D0D02AC515985F5B008E62BD /* print_help.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855613B3ACEE0099B651 /* print_help.cpp */; }; D0D02AC515985F5B008E62BD /* print_help.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855613B3ACEE0099B651 /* print_help.cpp */; };
D0D02AC615985F65008E62BD /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; }; D0D02AC615985F65008E62BD /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; };
D0D02AC715985F9D008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; }; D0D02AC715985F9D008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
D0D02AC815985F9F008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
D0D02AD615986492008E62BD /* fish_indent.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853F13B3ACEE0099B651 /* fish_indent.cpp */; }; D0D02AD615986492008E62BD /* fish_indent.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853F13B3ACEE0099B651 /* fish_indent.cpp */; };
D0D02AD715986498008E62BD /* print_help.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855613B3ACEE0099B651 /* print_help.cpp */; }; D0D02AD715986498008E62BD /* print_help.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855613B3ACEE0099B651 /* print_help.cpp */; };
D0D02AD81598649E008E62BD /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; }; D0D02AD81598649E008E62BD /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; };
@ -169,7 +168,6 @@
D0D02ADA159864AB008E62BD /* wutil.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0856113B3ACEE0099B651 /* wutil.cpp */; }; D0D02ADA159864AB008E62BD /* wutil.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0856113B3ACEE0099B651 /* wutil.cpp */; };
D0D02ADB159864C2008E62BD /* tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */; }; D0D02ADB159864C2008E62BD /* tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */; };
D0D02ADC159864D5008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; }; D0D02ADC159864D5008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; };
D0D02ADD159864D7008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; };
D0D2694915983772005D9B9C /* function.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854413B3ACEE0099B651 /* function.cpp */; }; D0D2694915983772005D9B9C /* function.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854413B3ACEE0099B651 /* function.cpp */; };
D0D2694A15983779005D9B9C /* builtin.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853513B3ACEE0099B651 /* builtin.cpp */; }; D0D2694A15983779005D9B9C /* builtin.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853513B3ACEE0099B651 /* builtin.cpp */; };
D0F019F115A977140034B3B1 /* fish in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0D2693C159835CA005D9B9C /* fish */; }; D0F019F115A977140034B3B1 /* fish in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0D2693C159835CA005D9B9C /* fish */; };
@ -475,11 +473,12 @@
D0C6FCC914CFA4B0004CE8AD /* autoload.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = autoload.cpp; sourceTree = "<group>"; }; D0C6FCC914CFA4B0004CE8AD /* autoload.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = autoload.cpp; sourceTree = "<group>"; };
D0C6FCCB14CFA4B7004CE8AD /* autoload.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = autoload.h; sourceTree = "<group>"; }; D0C6FCCB14CFA4B7004CE8AD /* autoload.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = autoload.h; sourceTree = "<group>"; };
D0C861EA16CC7054003B5A04 /* builtin_set_color.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_set_color.cpp; sourceTree = "<group>"; }; D0C861EA16CC7054003B5A04 /* builtin_set_color.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_set_color.cpp; sourceTree = "<group>"; };
D0C9733718DE5449002D7C81 /* utf8.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = utf8.cpp; sourceTree = "<group>"; };
D0C9733A18DE5451002D7C81 /* utf8.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = utf8.h; sourceTree = "<group>"; };
D0CA63F316FC275F00093BD4 /* builtin_printf.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_printf.cpp; sourceTree = "<group>"; }; D0CA63F316FC275F00093BD4 /* builtin_printf.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_printf.cpp; sourceTree = "<group>"; };
D0CBD580159EE48F0024809C /* config.fish */ = {isa = PBXFileReference; lastKnownFileType = text; name = config.fish; path = share/config.fish; sourceTree = "<group>"; }; D0CBD580159EE48F0024809C /* config.fish */ = {isa = PBXFileReference; lastKnownFileType = text; name = config.fish; path = share/config.fish; sourceTree = "<group>"; };
D0CBD583159EEE010024809C /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; }; D0CBD583159EEE010024809C /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; };
D0CBD586159EF0E10024809C /* launch_fish.scpt */ = {isa = PBXFileReference; lastKnownFileType = file; name = launch_fish.scpt; path = osx/launch_fish.scpt; sourceTree = "<group>"; }; D0CBD586159EF0E10024809C /* launch_fish.scpt */ = {isa = PBXFileReference; lastKnownFileType = file; name = launch_fish.scpt; path = osx/launch_fish.scpt; sourceTree = "<group>"; };
D0D02A8A15983CDF008E62BD /* libiconv.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libiconv.dylib; path = usr/lib/libiconv.dylib; sourceTree = SDKROOT; };
D0D02A8C15983CFA008E62BD /* libncurses.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libncurses.dylib; path = usr/lib/libncurses.dylib; sourceTree = SDKROOT; }; D0D02A8C15983CFA008E62BD /* libncurses.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libncurses.dylib; path = usr/lib/libncurses.dylib; sourceTree = SDKROOT; };
D0D02A9A15985A75008E62BD /* fish.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = fish.app; sourceTree = BUILT_PRODUCTS_DIR; }; D0D02A9A15985A75008E62BD /* fish.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = fish.app; sourceTree = BUILT_PRODUCTS_DIR; };
D0D02AA915985C0C008E62BD /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = Info.plist; path = osx/Info.plist; sourceTree = "<group>"; }; D0D02AA915985C0C008E62BD /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = Info.plist; path = osx/Info.plist; sourceTree = "<group>"; };
@ -499,7 +498,6 @@
isa = PBXFrameworksBuildPhase; isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647; buildActionMask = 2147483647;
files = ( files = (
D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */,
D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */, D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */,
); );
runOnlyForDeploymentPostprocessing = 0; runOnlyForDeploymentPostprocessing = 0;
@ -509,7 +507,6 @@
buildActionMask = 2147483647; buildActionMask = 2147483647;
files = ( files = (
D0D02AC715985F9D008E62BD /* libncurses.dylib in Frameworks */, D0D02AC715985F9D008E62BD /* libncurses.dylib in Frameworks */,
D0D02AC815985F9F008E62BD /* libiconv.dylib in Frameworks */,
); );
runOnlyForDeploymentPostprocessing = 0; runOnlyForDeploymentPostprocessing = 0;
}; };
@ -518,7 +515,6 @@
buildActionMask = 2147483647; buildActionMask = 2147483647;
files = ( files = (
D0D02ADC159864D5008E62BD /* libncurses.dylib in Frameworks */, D0D02ADC159864D5008E62BD /* libncurses.dylib in Frameworks */,
D0D02ADD159864D7008E62BD /* libiconv.dylib in Frameworks */,
); );
runOnlyForDeploymentPostprocessing = 0; runOnlyForDeploymentPostprocessing = 0;
}; };
@ -527,7 +523,6 @@
buildActionMask = 2147483647; buildActionMask = 2147483647;
files = ( files = (
D0D02A8D15983CFA008E62BD /* libncurses.dylib in Frameworks */, D0D02A8D15983CFA008E62BD /* libncurses.dylib in Frameworks */,
D0D02A8B15983CDF008E62BD /* libiconv.dylib in Frameworks */,
); );
runOnlyForDeploymentPostprocessing = 0; runOnlyForDeploymentPostprocessing = 0;
}; };
@ -583,7 +578,6 @@
isa = PBXGroup; isa = PBXGroup;
children = ( children = (
D0D02A8C15983CFA008E62BD /* libncurses.dylib */, D0D02A8C15983CFA008E62BD /* libncurses.dylib */,
D0D02A8A15983CDF008E62BD /* libiconv.dylib */,
D0CBD583159EEE010024809C /* Foundation.framework */, D0CBD583159EEE010024809C /* Foundation.framework */,
); );
name = Libraries; name = Libraries;
@ -686,6 +680,8 @@
D0A0855C13B3ACEE0099B651 /* signal.cpp */, D0A0855C13B3ACEE0099B651 /* signal.cpp */,
D0A0852513B3ACEE0099B651 /* tokenizer.h */, D0A0852513B3ACEE0099B651 /* tokenizer.h */,
D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */, D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */,
D0C9733A18DE5451002D7C81 /* utf8.h */,
D0C9733718DE5449002D7C81 /* utf8.cpp */,
D0A0852613B3ACEE0099B651 /* util.h */, D0A0852613B3ACEE0099B651 /* util.h */,
D0A0855E13B3ACEE0099B651 /* util.cpp */, D0A0855E13B3ACEE0099B651 /* util.cpp */,
D0A0852713B3ACEE0099B651 /* wgetopt.h */, D0A0852713B3ACEE0099B651 /* wgetopt.h */,
@ -1120,6 +1116,7 @@
files = ( files = (
D0D02AC215985F3F008E62BD /* fishd.cpp in Sources */, D0D02AC215985F3F008E62BD /* fishd.cpp in Sources */,
D0D02AC315985F43008E62BD /* env_universal_common.cpp in Sources */, D0D02AC315985F43008E62BD /* env_universal_common.cpp in Sources */,
D0C9733918DE5449002D7C81 /* utf8.cpp in Sources */,
D0D02AC415985F4D008E62BD /* wutil.cpp in Sources */, D0D02AC415985F4D008E62BD /* wutil.cpp in Sources */,
D0D02AC515985F5B008E62BD /* print_help.cpp in Sources */, D0D02AC515985F5B008E62BD /* print_help.cpp in Sources */,
D0D02AC615985F65008E62BD /* common.cpp in Sources */, D0D02AC615985F65008E62BD /* common.cpp in Sources */,
@ -1157,6 +1154,7 @@
D0D02A86159839D5008E62BD /* postfork.cpp in Sources */, D0D02A86159839D5008E62BD /* postfork.cpp in Sources */,
D0D02A87159839D5008E62BD /* screen.cpp in Sources */, D0D02A87159839D5008E62BD /* screen.cpp in Sources */,
D0D02A88159839D5008E62BD /* signal.cpp in Sources */, D0D02A88159839D5008E62BD /* signal.cpp in Sources */,
D0C9733818DE5449002D7C81 /* utf8.cpp in Sources */,
D0D2694A15983779005D9B9C /* builtin.cpp in Sources */, D0D2694A15983779005D9B9C /* builtin.cpp in Sources */,
D0D2694915983772005D9B9C /* function.cpp in Sources */, D0D2694915983772005D9B9C /* function.cpp in Sources */,
D0D02A67159837AD008E62BD /* complete.cpp in Sources */, D0D02A67159837AD008E62BD /* complete.cpp in Sources */,

259
fish_tests.cpp
View file

@ -62,6 +62,7 @@
#include "parse_util.h" #include "parse_util.h"
#include "pager.h" #include "pager.h"
#include "input.h" #include "input.h"
#include "utf8.h"
static const char * const * s_arguments; static const char * const * s_arguments;
static int s_test_run_count = 0; static int s_test_run_count = 0;
@ -925,6 +926,260 @@ static void test_utils()
if (begin != a + wcslen(L"echo (echo (")) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__); if (begin != a + wcslen(L"echo (echo (")) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__);
} }
/* UTF8 tests taken from Alexey Vatchenko's utf8 library. See http://www.bsdua.org/libbsdua.html */
static void test_utf82wchar(const char *src, size_t slen, const wchar_t *dst, size_t dlen,
int flags, size_t res, const char *descr)
{
size_t size;
wchar_t *mem = NULL;
/* Hack: if wchar is only UCS-2, and the UTF-8 input string contains astral characters, then tweak the expected size to 0 */
if (src != NULL && is_wchar_ucs2())
{
/* A UTF-8 code unit may represent an astral code point if it has 4 or more leading 1s */
const unsigned char astral_mask = 0xF0;
for (size_t i=0; i < slen; i++)
{
if ((src[i] & astral_mask) == astral_mask)
{
/* Astral char. We expect this conversion to just fail. */
res = 0;
break;
}
}
}
if (dst != NULL)
{
mem = (wchar_t *)malloc(dlen * sizeof(*mem));
if (mem == NULL)
{
err(L"u2w: %s: MALLOC FAILED\n", descr);
return;
}
}
do
{
size = utf8_to_wchar(src, slen, mem, dlen, flags);
if (res != size)
{
err(L"u2w: %s: FAILED (rv: %lu, must be %lu)", descr, size, res);
break;
}
if (mem == NULL)
break; /* OK */
if (memcmp(mem, dst, size * sizeof(*mem)) != 0)
{
err(L"u2w: %s: BROKEN", descr);
break;
}
}
while (0);
free(mem);
}
static void test_wchar2utf8(const wchar_t *src, size_t slen, const char *dst, size_t dlen,
int flags, size_t res, const char *descr)
{
size_t size;
char *mem = NULL;
/* Hack: if wchar is simulating UCS-2, and the wchar_t input string contains astral characters, then tweak the expected size to 0 */
if (src != NULL && is_wchar_ucs2())
{
const uint32_t astral_mask = 0xFFFF0000U;
for (size_t i=0; i < slen; i++)
{
if ((src[i] & astral_mask) != 0)
{
/* astral char */
res = 0;
break;
}
}
}
if (dst != NULL)
{
mem = (char *)malloc(dlen);
if (mem == NULL)
{
err(L"w2u: %s: MALLOC FAILED", descr);
return;
}
}
do
{
size = wchar_to_utf8(src, slen, mem, dlen, flags);
if (res != size)
{
err(L"w2u: %s: FAILED (rv: %lu, must be %lu)", descr, size, res);
break;
}
if (mem == NULL)
break; /* OK */
if (memcmp(mem, dst, size) != 0)
{
err(L"w2u: %s: BROKEN", descr);
break;
}
}
while (0);
if (mem != NULL);
free(mem);
}
static void test_utf8()
{
wchar_t w1[] = {0x54, 0x65, 0x73, 0x74};
wchar_t w2[] = {0x0422, 0x0435, 0x0441, 0x0442};
wchar_t w3[] = {0x800, 0x1e80, 0x98c4, 0x9910, 0xff00};
wchar_t w4[] = {0x15555, 0xf7777, 0xa};
wchar_t w5[] = {0x255555, 0x1fa04ff, 0xddfd04, 0xa};
wchar_t w6[] = {0xf255555, 0x1dfa04ff, 0x7fddfd04, 0xa};
wchar_t wb[] = {-2, 0xa, 0xffffffff, 0x0441};
wchar_t wm[] = {0x41, 0x0441, 0x3042, 0xff67, 0x9b0d, 0x2e05da67};
wchar_t wb1[] = {0xa, 0x0422};
wchar_t wb2[] = {0xd800, 0xda00, 0x41, 0xdfff, 0xa};
wchar_t wbom[] = {0xfeff, 0x41, 0xa};
wchar_t wbom2[] = {0x41, 0xa};
wchar_t wbom22[] = {0xfeff, 0x41, 0xa};
char u1[] = {0x54, 0x65, 0x73, 0x74};
char u2[] = {0xd0, 0xa2, 0xd0, 0xb5, 0xd1, 0x81, 0xd1, 0x82};
char u3[] = {0xe0, 0xa0, 0x80, 0xe1, 0xba, 0x80, 0xe9, 0xa3, 0x84,
0xe9, 0xa4, 0x90, 0xef, 0xbc, 0x80
};
char u4[] = {0xf0, 0x95, 0x95, 0x95, 0xf3, 0xb7, 0x9d, 0xb7, 0xa};
char u5[] = {0xf8, 0x89, 0x95, 0x95, 0x95, 0xf9, 0xbe, 0xa0, 0x93,
0xbf, 0xf8, 0xb7, 0x9f, 0xb4, 0x84, 0x0a
};
char u6[] = {0xfc, 0x8f, 0x89, 0x95, 0x95, 0x95, 0xfc, 0x9d, 0xbe,
0xa0, 0x93, 0xbf, 0xfd, 0xbf, 0xb7, 0x9f, 0xb4, 0x84, 0x0a
};
char ub[] = {0xa, 0xd1, 0x81};
char um[] = {0x41, 0xd1, 0x81, 0xe3, 0x81, 0x82, 0xef, 0xbd, 0xa7,
0xe9, 0xac, 0x8d, 0xfc, 0xae, 0x81, 0x9d, 0xa9, 0xa7
};
char ub1[] = {0xa, 0xff, 0xd0, 0xa2, 0xfe, 0x8f, 0xe0, 0x80};
char uc080[] = {0xc0, 0x80};
char ub2[] = {0xed, 0xa1, 0x8c, 0xed, 0xbe, 0xb4, 0xa};
char ubom[] = {0x41, 0xa};
char ubom2[] = {0xef, 0xbb, 0xbf, 0x41, 0xa};
/*
* UTF-8 -> UCS-4 string.
*/
test_utf82wchar(ubom2, sizeof(ubom2), wbom2,
sizeof(wbom2) / sizeof(*wbom2), UTF8_SKIP_BOM,
sizeof(wbom2) / sizeof(*wbom2), "skip BOM");
test_utf82wchar(ubom2, sizeof(ubom2), wbom22,
sizeof(wbom22) / sizeof(*wbom22), 0,
sizeof(wbom22) / sizeof(*wbom22), "BOM");
test_utf82wchar(uc080, sizeof(uc080), NULL, 0, 0, 0,
"c0 80 - forbitten by rfc3629");
test_utf82wchar(ub2, sizeof(ub2), NULL, 0, 0, is_wchar_ucs2() ? 0 : 3,
"resulted in forbitten wchars (len)");
test_utf82wchar(ub2, sizeof(ub2), wb2, sizeof(wb2) / sizeof(*wb2), 0, 0,
"resulted in forbitten wchars");
test_utf82wchar(ub2, sizeof(ub2), L"\x0a", 1, UTF8_IGNORE_ERROR,
1, "resulted in ignored forbitten wchars");
test_utf82wchar(u1, sizeof(u1), w1, sizeof(w1) / sizeof(*w1), 0,
sizeof(w1) / sizeof(*w1), "1 octet chars");
test_utf82wchar(u2, sizeof(u2), w2, sizeof(w2) / sizeof(*w2), 0,
sizeof(w2) / sizeof(*w2), "2 octets chars");
test_utf82wchar(u3, sizeof(u3), w3, sizeof(w3) / sizeof(*w3), 0,
sizeof(w3) / sizeof(*w3), "3 octets chars");
test_utf82wchar(u4, sizeof(u4), w4, sizeof(w4) / sizeof(*w4), 0,
sizeof(w4) / sizeof(*w4), "4 octets chars");
test_utf82wchar(u5, sizeof(u5), w5, sizeof(w5) / sizeof(*w5), 0,
sizeof(w5) / sizeof(*w5), "5 octets chars");
test_utf82wchar(u6, sizeof(u6), w6, sizeof(w6) / sizeof(*w6), 0,
sizeof(w6) / sizeof(*w6), "6 octets chars");
test_utf82wchar("\xff", 1, NULL, 0, 0, 0, "broken utf-8 0xff symbol");
test_utf82wchar("\xfe", 1, NULL, 0, 0, 0, "broken utf-8 0xfe symbol");
test_utf82wchar("\x8f", 1, NULL, 0, 0, 0,
"broken utf-8, start from 10 higher bits");
if (! is_wchar_ucs2()) test_utf82wchar(ub1, sizeof(ub1), wb1, sizeof(wb1) / sizeof(*wb1),
UTF8_IGNORE_ERROR, sizeof(wb1) / sizeof(*wb1), "ignore bad chars");
test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm), 0,
sizeof(wm) / sizeof(*wm), "mixed languages");
test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm) - 1, 0,
0, "boundaries -1");
test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm) + 1, 0,
sizeof(wm) / sizeof(*wm), "boundaries +1");
test_utf82wchar(um, sizeof(um), NULL, 0, 0,
sizeof(wm) / sizeof(*wm), "calculate length");
test_utf82wchar(ub1, sizeof(ub1), NULL, 0, 0,
0, "calculate length of bad chars");
test_utf82wchar(ub1, sizeof(ub1), NULL, 0,
UTF8_IGNORE_ERROR, sizeof(wb1) / sizeof(*wb1),
"calculate length, ignore bad chars");
test_utf82wchar(NULL, 0, NULL, 0, 0, 0, "invalid params, all 0");
test_utf82wchar(u1, 0, NULL, 0, 0, 0,
"invalid params, src buf not NULL");
test_utf82wchar(NULL, 10, NULL, 0, 0, 0,
"invalid params, src length is not 0");
test_utf82wchar(u1, sizeof(u1), w1, 0, 0, 0,
"invalid params, dst is not NULL");
/*
* UCS-4 -> UTF-8 string.
*/
test_wchar2utf8(wbom, sizeof(wbom) / sizeof(*wbom), ubom, sizeof(ubom),
UTF8_SKIP_BOM, sizeof(ubom), "BOM");
test_wchar2utf8(wb2, sizeof(wb2) / sizeof(*wb2), NULL, 0, 0,
0, "prohibited wchars");
test_wchar2utf8(wb2, sizeof(wb2) / sizeof(*wb2), NULL, 0,
UTF8_IGNORE_ERROR, 2, "ignore prohibited wchars");
test_wchar2utf8(w1, sizeof(w1) / sizeof(*w1), u1, sizeof(u1), 0,
sizeof(u1), "1 octet chars");
test_wchar2utf8(w2, sizeof(w2) / sizeof(*w2), u2, sizeof(u2), 0,
sizeof(u2), "2 octets chars");
test_wchar2utf8(w3, sizeof(w3) / sizeof(*w3), u3, sizeof(u3), 0,
sizeof(u3), "3 octets chars");
test_wchar2utf8(w4, sizeof(w4) / sizeof(*w4), u4, sizeof(u4), 0,
sizeof(u4), "4 octets chars");
test_wchar2utf8(w5, sizeof(w5) / sizeof(*w5), u5, sizeof(u5), 0,
sizeof(u5), "5 octets chars");
test_wchar2utf8(w6, sizeof(w6) / sizeof(*w6), u6, sizeof(u6), 0,
sizeof(u6), "6 octets chars");
test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), ub, sizeof(ub), 0,
0, "bad chars");
test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), ub, sizeof(ub),
UTF8_IGNORE_ERROR, sizeof(ub), "ignore bad chars");
test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um), 0,
sizeof(um), "mixed languages");
test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um) - 1, 0,
0, "boundaries -1");
test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um) + 1, 0,
sizeof(um), "boundaries +1");
test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), NULL, 0, 0,
sizeof(um), "calculate length");
test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), NULL, 0, 0,
0, "calculate length of bad chars");
test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), NULL, 0,
UTF8_IGNORE_ERROR, sizeof(ub),
"calculate length, ignore bad chars");
test_wchar2utf8(NULL, 0, NULL, 0, 0, 0, "invalid params, all 0");
test_wchar2utf8(w1, 0, NULL, 0, 0, 0,
"invalid params, src buf not NULL");
test_wchar2utf8(NULL, 10, NULL, 0, 0, 0,
"invalid params, src length is not 0");
test_wchar2utf8(w1, sizeof(w1) / sizeof(*w1), u1, 0, 0, 0,
"invalid params, dst is not NULL");
}
static void test_escape_sequences(void) static void test_escape_sequences(void)
{ {
say(L"Testing escape codes"); say(L"Testing escape codes");
@ -2935,6 +3190,7 @@ int main(int argc, char **argv)
if (should_test_function("cancellation")) test_cancellation(); if (should_test_function("cancellation")) test_cancellation();
if (should_test_function("indents")) test_indents(); if (should_test_function("indents")) test_indents();
if (should_test_function("utils")) test_utils(); if (should_test_function("utils")) test_utils();
if (should_test_function("utf8")) test_utf8();
if (should_test_function("escape_sequences")) test_escape_sequences(); if (should_test_function("escape_sequences")) test_escape_sequences();
if (should_test_function("lru")) test_lru(); if (should_test_function("lru")) test_lru();
if (should_test_function("expand")) test_expand(); if (should_test_function("expand")) test_expand();
@ -2974,7 +3230,8 @@ int main(int argc, char **argv)
event_destroy(); event_destroy();
proc_destroy(); proc_destroy();
if(err_count != 0) { if (err_count != 0)
{
return(1); return(1);
} }
} }

4
highlight.cpp
View file

@ -44,8 +44,6 @@
*/ */
#define VAR_COUNT ( sizeof(highlight_var)/sizeof(wchar_t *) ) #define VAR_COUNT ( sizeof(highlight_var)/sizeof(wchar_t *) )
static void highlight_universal_internal(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos);
/** The environment variables used to specify the color of different tokens. This matches the order in highlight_spec_t */ /** The environment variables used to specify the color of different tokens. This matches the order in highlight_spec_t */
static const wchar_t * const highlight_var[] = static const wchar_t * const highlight_var[] =
{ {
@ -350,7 +348,7 @@ bool plain_statement_get_expanded_command(const wcstring &src, const parse_node_
if (expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES | EXPAND_SKIP_JOBS)) if (expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES | EXPAND_SKIP_JOBS))
{ {
/* Success, return the expanded string by reference */ /* Success, return the expanded string by reference */
std::swap(cmd, *out_cmd); out_cmd->swap(cmd);
result = true; result = true;
} }
} }

5
highlight.h
View file

@ -75,7 +75,6 @@ struct file_detection_context_t;
\param error a list in which a description of each error will be inserted. May be 0, in whcich case no error descriptions will be generated. \param error a list in which a description of each error will be inserted. May be 0, in whcich case no error descriptions will be generated.
*/ */
void highlight_shell(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars); void highlight_shell(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
void highlight_shell_new_parser(const wcstring &buffstr, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
/** /**
Perform syntax highlighting for the text in buff. Matching quotes and paranthesis are highlighted. The result is Perform syntax highlighting for the text in buff. Matching quotes and paranthesis are highlighted. The result is
@ -125,9 +124,5 @@ enum
typedef unsigned int path_flags_t; typedef unsigned int path_flags_t;
bool is_potential_path(const wcstring &const_path, const wcstring_list_t &directories, path_flags_t flags, wcstring *out_path = NULL); bool is_potential_path(const wcstring &const_path, const wcstring_list_t &directories, path_flags_t flags, wcstring *out_path = NULL);
/* For testing */
void highlight_shell_classic(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
void highlight_shell_new_parser(const wcstring &buff, std::vector<highlight_spec_t> &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars);
#endif #endif

2
osx/config.h
View file

@ -218,7 +218,9 @@
#if __GNUC__ >= 3 #if __GNUC__ >= 3
#define __warn_unused __attribute__ ((warn_unused_result)) #define __warn_unused __attribute__ ((warn_unused_result))
#define __sentinel __attribute__ ((sentinel)) #define __sentinel __attribute__ ((sentinel))
#define __packed __attribute__ ((packed))
#else #else
#define __warn_unused #define __warn_unused
#define __sentinel #define __sentinel
#define __packed
#endif #endif

7
parse_constants.h
View file

@ -6,10 +6,11 @@
#ifndef fish_parse_constants_h #ifndef fish_parse_constants_h
#define fish_parse_constants_h #define fish_parse_constants_h
#include "config.h"
#define PARSE_ASSERT(a) assert(a) #define PARSE_ASSERT(a) assert(a)
#define PARSER_DIE() do { fprintf(stderr, "Parser dying!\n"); exit_without_destructors(-1); } while (0) #define PARSER_DIE() do { fprintf(stderr, "Parser dying!\n"); exit_without_destructors(-1); } while (0)
enum parse_token_type_t enum parse_token_type_t
{ {
token_type_invalid, token_type_invalid,
@ -70,7 +71,7 @@ enum parse_token_type_t
LAST_TOKEN_OR_SYMBOL = parse_token_type_terminate, LAST_TOKEN_OR_SYMBOL = parse_token_type_terminate,
FIRST_PARSE_TOKEN_TYPE = parse_token_type_string FIRST_PARSE_TOKEN_TYPE = parse_token_type_string
}; } __packed;
enum parse_keyword_t enum parse_keyword_t
{ {
@ -93,7 +94,7 @@ enum parse_keyword_t
parse_keyword_exec, parse_keyword_exec,
LAST_KEYWORD = parse_keyword_exec LAST_KEYWORD = parse_keyword_exec
}; } __packed;
/* Statement decorations. This matches the order of productions in decorated_statement */ /* Statement decorations. This matches the order of productions in decorated_statement */
enum parse_statement_decoration_t enum parse_statement_decoration_t

5
parse_execution.cpp
View file

@ -72,7 +72,8 @@ node_offset_t parse_execution_context_t::get_offset(const parse_node_t &node) co
const parse_node_t *addr = &node; const parse_node_t *addr = &node;
const parse_node_t *base = &this->tree.at(0); const parse_node_t *base = &this->tree.at(0);
assert(addr >= base); assert(addr >= base);
node_offset_t offset = addr - base; assert(addr - base < SOURCE_OFFSET_INVALID);
node_offset_t offset = static_cast<node_offset_t>(addr - base);
assert(offset < this->tree.size()); assert(offset < this->tree.size());
assert(&tree.at(offset) == &node); assert(&tree.at(offset) == &node);
return offset; return offset;
@ -1100,7 +1101,7 @@ bool parse_execution_context_t::determine_io_chain(const parse_node_t &statement
if (out_chain && ! errored) if (out_chain && ! errored)
{ {
std::swap(*out_chain, result); out_chain->swap(result);
} }
return ! errored; return ! errored;
} }

69
parse_tree.cpp
View file

@ -13,6 +13,13 @@ static bool production_is_empty(const production_t *production)
return (*production)[0] == token_type_invalid; return (*production)[0] == token_type_invalid;
} }
void swap2(parse_node_tree_t &a, parse_node_tree_t &b)
{
fprintf(stderr, "Swapping!\n");
// This uses the base vector implementation
a.swap(b);
}
/** Returns a string description of this parse error */ /** Returns a string description of this parse error */
wcstring parse_error_t::describe_with_prefix(const wcstring &src, const wcstring &prefix, bool is_interactive, bool skip_caret) const wcstring parse_error_t::describe_with_prefix(const wcstring &src, const wcstring &prefix, bool is_interactive, bool skip_caret) const
{ {
@ -422,7 +429,7 @@ static void dump_tree_recursive(const parse_node_tree_t &nodes, const wcstring &
result->push_back(L'\n'); result->push_back(L'\n');
++*line; ++*line;
for (size_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++) for (node_offset_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++)
{ {
dump_tree_recursive(nodes, src, child_idx, indent + 1, result, line, inout_first_node_not_dumped); dump_tree_recursive(nodes, src, child_idx, indent + 1, result, line, inout_first_node_not_dumped);
} }
@ -546,11 +553,18 @@ class parse_ll_t
} }
// Get the parent index. But we can't get the parent parse node yet, since it may be made invalid by adding children // Get the parent index. But we can't get the parent parse node yet, since it may be made invalid by adding children
const size_t parent_node_idx = symbol_stack.back().node_idx; const node_offset_t parent_node_idx = symbol_stack.back().node_idx;
// Add the children. Confusingly, we want our nodes to be in forwards order (last token last, so dumps look nice), but the symbols should be reverse order (last token first, so it's lowest on the stack) // Add the children. Confusingly, we want our nodes to be in forwards order (last token last, so dumps look nice), but the symbols should be reverse order (last token first, so it's lowest on the stack)
const size_t child_start = nodes.size(); const size_t child_start_big = nodes.size();
size_t child_count = 0; assert(child_start_big < NODE_OFFSET_INVALID);
node_offset_t child_start = static_cast<node_offset_t>(child_start_big);
// To avoid constructing multiple nodes, we push_back a single one that we modify
parse_node_t representative_child(token_type_invalid);
representative_child.parent = parent_node_idx;
node_offset_t child_count = 0;
for (size_t i=0; i < MAX_SYMBOLS_PER_PRODUCTION; i++) for (size_t i=0; i < MAX_SYMBOLS_PER_PRODUCTION; i++)
{ {
production_element_t elem = (*production)[i]; production_element_t elem = (*production)[i];
@ -559,16 +573,12 @@ class parse_ll_t
// All done, bail out // All done, bail out
break; break;
} }
else
{ // Append the parse node.
// Generate the parse node. representative_child.type = production_element_type(elem);
parse_token_type_t child_type = production_element_type(elem); nodes.push_back(representative_child);
parse_node_t child = parse_node_t(child_type);
child.parent = parent_node_idx;
nodes.push_back(child);
child_count++; child_count++;
} }
}
// Update the parent // Update the parent
parse_node_t &parent_node = nodes.at(parent_node_idx); parse_node_t &parent_node = nodes.at(parent_node_idx);
@ -583,7 +593,7 @@ class parse_ll_t
// Replace the top of the stack with new stack elements corresponding to our new nodes. Note that these go in reverse order. // Replace the top of the stack with new stack elements corresponding to our new nodes. Note that these go in reverse order.
symbol_stack.pop_back(); symbol_stack.pop_back();
symbol_stack.reserve(symbol_stack.size() + child_count); symbol_stack.reserve(symbol_stack.size() + child_count);
size_t idx = child_count; node_offset_t idx = child_count;
while (idx--) while (idx--)
{ {
production_element_t elem = (*production)[idx]; production_element_t elem = (*production)[idx];
@ -669,18 +679,17 @@ void parse_ll_t::dump_stack(void) const
// Since children always appear after their parents, we can implement this very simply by walking backwards // Since children always appear after their parents, we can implement this very simply by walking backwards
void parse_ll_t::determine_node_ranges(void) void parse_ll_t::determine_node_ranges(void)
{ {
const size_t source_start_invalid = -1;
size_t idx = nodes.size(); size_t idx = nodes.size();
while (idx--) while (idx--)
{ {
parse_node_t *parent = &nodes.at(idx); parse_node_t *parent = &nodes[idx];
// Skip nodes that already have a source range. These are terminal nodes. // Skip nodes that already have a source range. These are terminal nodes.
if (parent->source_start != source_start_invalid) if (parent->source_start != SOURCE_OFFSET_INVALID)
continue; continue;
// Ok, this node needs a source range. Get all of its children, and then set its range. // Ok, this node needs a source range. Get all of its children, and then set its range.
size_t min_start = source_start_invalid, max_end = 0; //note source_start_invalid is huge source_offset_t min_start = SOURCE_OFFSET_INVALID, max_end = 0; //note SOURCE_OFFSET_INVALID is huge
for (node_offset_t i=0; i < parent->child_count; i++) for (node_offset_t i=0; i < parent->child_count; i++)
{ {
const parse_node_t &child = nodes.at(parent->child_offset(i)); const parse_node_t &child = nodes.at(parent->child_offset(i));
@ -691,7 +700,7 @@ void parse_ll_t::determine_node_ranges(void)
} }
} }
if (min_start != source_start_invalid) if (min_start != SOURCE_OFFSET_INVALID)
{ {
assert(max_end >= min_start); assert(max_end >= min_start);
parent->source_start = min_start; parent->source_start = min_start;
@ -704,13 +713,13 @@ void parse_ll_t::acquire_output(parse_node_tree_t *output, parse_error_list_t *e
{ {
if (output != NULL) if (output != NULL)
{ {
std::swap(*output, this->nodes); output->swap(this->nodes);
} }
this->nodes.clear(); this->nodes.clear();
if (errors != NULL) if (errors != NULL)
{ {
std::swap(*errors, this->errors); errors->swap(this->errors);
} }
this->errors.clear(); this->errors.clear();
this->symbol_stack.clear(); this->symbol_stack.clear();
@ -848,7 +857,7 @@ void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token)
void parse_ll_t::reset_symbols(enum parse_token_type_t goal) void parse_ll_t::reset_symbols(enum parse_token_type_t goal)
{ {
/* Add a new goal node, and then reset our symbol list to point at it */ /* Add a new goal node, and then reset our symbol list to point at it */
node_offset_t where = nodes.size(); node_offset_t where = static_cast<node_offset_t>(nodes.size());
nodes.push_back(parse_node_t(goal)); nodes.push_back(parse_node_t(goal));
symbol_stack.clear(); symbol_stack.clear();
@ -1064,14 +1073,10 @@ static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt)
} }
/* Placeholder invalid token */ /* Placeholder invalid token */
static const parse_token_t kInvalidToken = {token_type_invalid, static const parse_token_t kInvalidToken = {token_type_invalid, parse_keyword_none, false, false, SOURCE_OFFSET_INVALID, 0};
parse_keyword_none, false, false, static_cast<size_t>(-1),
static_cast<size_t>(-1)};
/* Terminal token */ /* Terminal token */
static const parse_token_t kTerminalToken = {parse_token_type_terminate, static const parse_token_t kTerminalToken = {parse_token_type_terminate, parse_keyword_none, false, false, SOURCE_OFFSET_INVALID, 0};
parse_keyword_none, false, false, static_cast<size_t>(-1),
static_cast<size_t>(-1)};
static inline bool is_help_argument(const wchar_t *txt) static inline bool is_help_argument(const wchar_t *txt)
{ {
@ -1099,8 +1104,8 @@ static inline parse_token_t next_parse_token(tokenizer_t *tok)
result.keyword = keyword_for_token(tok_type, tok_txt); result.keyword = keyword_for_token(tok_type, tok_txt);
result.has_dash_prefix = (tok_txt[0] == L'-'); result.has_dash_prefix = (tok_txt[0] == L'-');
result.is_help_argument = result.has_dash_prefix && is_help_argument(tok_txt); result.is_help_argument = result.has_dash_prefix && is_help_argument(tok_txt);
result.source_start = (size_t)tok_start; result.source_start = (source_offset_t)tok_start;
result.source_length = tok_extent; result.source_length = (source_offset_t)tok_extent;
tok_next(tok); tok_next(tok);
return result; return result;
@ -1212,7 +1217,7 @@ const parse_node_t *parse_node_tree_t::get_child(const parse_node_t &parent, nod
const parse_node_t &parse_node_tree_t::find_child(const parse_node_t &parent, parse_token_type_t type) const const parse_node_t &parse_node_tree_t::find_child(const parse_node_t &parent, parse_token_type_t type) const
{ {
for (size_t i=0; i < parent.child_count; i++) for (node_offset_t i=0; i < parent.child_count; i++)
{ {
const parse_node_t *child = this->get_child(parent, i); const parse_node_t *child = this->get_child(parent, i);
if (child->type == type) if (child->type == type)
@ -1258,7 +1263,7 @@ static void find_nodes_recursive(const parse_node_tree_t &tree, const parse_node
if (result->size() < max_count) if (result->size() < max_count)
{ {
if (parent.type == type) result->push_back(&parent); if (parent.type == type) result->push_back(&parent);
for (size_t i=0; i < parent.child_count; i++) for (node_offset_t i=0; i < parent.child_count; i++)
{ {
const parse_node_t *child = tree.get_child(parent, i); const parse_node_t *child = tree.get_child(parent, i);
assert(child != NULL); assert(child != NULL);
@ -1495,7 +1500,7 @@ const parse_node_t *parse_node_tree_t::next_node_in_node_list(const parse_node_t
const parse_node_t *next_cursor = NULL; const parse_node_t *next_cursor = NULL;
/* Walk through the children */ /* Walk through the children */
for (size_t i=0; i < list_cursor->child_count; i++) for (node_offset_t i=0; i < list_cursor->child_count; i++)
{ {
const parse_node_t *child = this->get_child(*list_cursor, i); const parse_node_t *child = this->get_child(*list_cursor, i);
if (child->type == entry_type) if (child->type == entry_type)

33
parse_tree.h
View file

@ -18,9 +18,15 @@
class parse_node_t; class parse_node_t;
class parse_node_tree_t; class parse_node_tree_t;
typedef size_t node_offset_t;
typedef uint32_t node_offset_t;
#define NODE_OFFSET_INVALID (static_cast<node_offset_t>(-1)) #define NODE_OFFSET_INVALID (static_cast<node_offset_t>(-1))
typedef uint32_t source_offset_t;
#define SOURCE_OFFSET_INVALID (static_cast<source_offset_t>(-1))
/* Returns a description of a list of parse errors */ /* Returns a description of a list of parse errors */
wcstring parse_errors_description(const parse_error_list_t &errors, const wcstring &src, const wchar_t *prefix = NULL); wcstring parse_errors_description(const parse_error_list_t &errors, const wcstring &src, const wchar_t *prefix = NULL);
@ -31,8 +37,8 @@ struct parse_token_t
enum parse_keyword_t keyword; // Any keyword represented by this token enum parse_keyword_t keyword; // Any keyword represented by this token
bool has_dash_prefix; // Hackish: whether the source contains a dash prefix bool has_dash_prefix; // Hackish: whether the source contains a dash prefix
bool is_help_argument; // Hackish: whether the source looks like '-h' or '--help' bool is_help_argument; // Hackish: whether the source looks like '-h' or '--help'
size_t source_start; source_offset_t source_start;
size_t source_length; source_offset_t source_length;
wcstring describe() const; wcstring describe() const;
wcstring user_presentable_description() const; wcstring user_presentable_description() const;
@ -63,35 +69,36 @@ wcstring parse_dump_tree(const parse_node_tree_t &tree, const wcstring &src);
wcstring token_type_description(parse_token_type_t type); wcstring token_type_description(parse_token_type_t type);
wcstring keyword_description(parse_keyword_t type); wcstring keyword_description(parse_keyword_t type);
/** Class for nodes of a parse tree */ /** Class for nodes of a parse tree. Since there's a lot of these, the size and order of the fields is important. */
class parse_node_t class parse_node_t
{ {
public: public:
/* Type of the node */
enum parse_token_type_t type;
/* Start in the source code */ /* Start in the source code */
size_t source_start; source_offset_t source_start;
/* Length of our range in the source code */ /* Length of our range in the source code */
size_t source_length; source_offset_t source_length;
/* Parent */ /* Parent */
node_offset_t parent; node_offset_t parent;
/* Children */ /* Children */
node_offset_t child_start; node_offset_t child_start;
/* Number of children */
uint8_t child_count; uint8_t child_count;
/* Which production was used */ /* Which production was used */
uint8_t production_idx; uint8_t production_idx;
/* Type of the node */
enum parse_token_type_t type;
/* Description */ /* Description */
wcstring describe(void) const; wcstring describe(void) const;
/* Constructor */ /* Constructor */
explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(-1), source_length(0), parent(NODE_OFFSET_INVALID), child_start(0), child_count(0), production_idx(-1) explicit parse_node_t(parse_token_type_t ty) : source_start(SOURCE_OFFSET_INVALID), source_length(0), parent(NODE_OFFSET_INVALID), child_start(0), child_count(0), production_idx(-1), type(ty)
{ {
} }
@ -104,7 +111,7 @@ public:
/* Indicate if this node has a range of source code associated with it */ /* Indicate if this node has a range of source code associated with it */
bool has_source() const bool has_source() const
{ {
return source_start != (size_t)(-1); return source_start != SOURCE_OFFSET_INVALID;
} }
/* Gets source for the node, or the empty string if it has no source */ /* Gets source for the node, or the empty string if it has no source */
@ -123,7 +130,6 @@ public:
} }
}; };
/* The parse tree itself */ /* The parse tree itself */
class parse_node_tree_t : public std::vector<parse_node_t> class parse_node_tree_t : public std::vector<parse_node_t>
{ {
@ -180,6 +186,7 @@ public:
parse_node_list_t specific_statements_for_job(const parse_node_t &job) const; parse_node_list_t specific_statements_for_job(const parse_node_t &job) const;
}; };
/* The big entry point. Parse a string, attempting to produce a tree for the given goal type */ /* The big entry point. Parse a string, attempting to produce a tree for the given goal type */
bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse_node_tree_t *output, parse_error_list_t *errors, parse_token_type_t goal = symbol_job_list); bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse_node_tree_t *output, parse_error_list_t *errors, parse_token_type_t goal = symbol_job_list);

514
utf8.cpp Normal file
View file

@ -0,0 +1,514 @@
/*
* Copyright (c) 2007 Alexey Vatchenko <av@bsdua.org>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <sys/types.h>
#include <stdint.h>
#include <wchar.h>
#include "utf8.h"
#include <string>
#include <vector>
#include <limits>
#define _NXT 0x80
#define _SEQ2 0xc0
#define _SEQ3 0xe0
#define _SEQ4 0xf0
#define _SEQ5 0xf8
#define _SEQ6 0xfc
#define _BOM 0xfeff
/* We can tweak the following typedef to allow us to simulate Windows-style 16 bit wchar's on Unix */
typedef wchar_t utf8_wchar_t;
#define UTF8_WCHAR_MAX ((size_t)std::numeric_limits<utf8_wchar_t>::max())
bool is_wchar_ucs2()
{
return UTF8_WCHAR_MAX <= 0xFFFF;
}
static size_t utf8_to_wchar_internal(const char *in, size_t insize, utf8_wchar_t *out, size_t outsize, int flags);
static size_t wchar_to_utf8_internal(const utf8_wchar_t *in, size_t insize, char *out, size_t outsize, int flags);
static bool safe_copy_wchar_to_utf8_wchar(const wchar_t *in, utf8_wchar_t *out, size_t count)
{
bool result = true;
for (size_t i=0; i < count; i++)
{
wchar_t c = in[i];
if (c > UTF8_WCHAR_MAX)
{
result = false;
break;
}
out[i] = c;
}
return result;
}
bool utf8_to_wchar_string(const std::string &str, std::wstring *result)
{
result->clear();
const size_t inlen = str.size();
if (inlen == 0)
{
return true;
}
bool success = false;
const char *input = str.c_str();
size_t outlen = utf8_to_wchar(input, inlen, NULL, 0, 0);
if (outlen > 0)
{
wchar_t *tmp = new wchar_t[outlen];
size_t outlen2 = utf8_to_wchar(input, inlen, tmp, outlen, 0);
if (outlen2 > 0)
{
result->assign(tmp, outlen2);
success = true;
}
delete[] tmp;
}
return success;
}
bool wchar_to_utf8_string(const std::wstring &str, std::string *result)
{
result->clear();
const size_t inlen = str.size();
if (inlen == 0)
{
return true;
}
bool success = false;
const wchar_t *input = str.c_str();
size_t outlen = wchar_to_utf8(input, inlen, NULL, 0, 0);
if (outlen > 0)
{
char *tmp = new char[outlen];
size_t outlen2 = wchar_to_utf8(input, inlen, tmp, outlen, 0);
if (outlen2 > 0)
{
result->assign(tmp, outlen2);
success = true;
}
delete[] tmp;
}
return success;
}
size_t utf8_to_wchar(const char *in, size_t insize, wchar_t *out, size_t outsize, int flags)
{
if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
{
return 0;
}
size_t result;
if (sizeof(wchar_t) == sizeof(utf8_wchar_t))
{
result = utf8_to_wchar_internal(in, insize, reinterpret_cast<utf8_wchar_t *>(out), outsize, flags);
}
else
{
// Allocate a temporary buffer to hold the output
// note: outsize may be 0
utf8_wchar_t *tmp_output = new utf8_wchar_t[outsize];
// Invoke the conversion with the temporary
result = utf8_to_wchar_internal(in, insize, tmp_output, outsize, flags);
// Copy back from tmp to the function's output, then clean it up
size_t amount_to_copy = std::min(result, outsize);
std::copy(tmp_output, tmp_output + amount_to_copy, out);
delete[] tmp_output;
}
return result;
}
size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, size_t outsize, int flags)
{
if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
{
return 0;
}
size_t result;
if (sizeof(wchar_t) == sizeof(utf8_wchar_t))
{
result = wchar_to_utf8_internal(reinterpret_cast<const utf8_wchar_t *>(in), insize, out, outsize, flags);
}
else
{
// Allocate a temporary buffer to hold the input
// the std::copy performs the size conversion
// note: insize may be 0
utf8_wchar_t *tmp_input = new utf8_wchar_t[insize];
if (! safe_copy_wchar_to_utf8_wchar(in, tmp_input, insize))
{
// our utf8_wchar_t is UCS-16 and there was an astral character
result = 0;
}
else
{
// Invoke the conversion with the temporary, then clean up the input
result = wchar_to_utf8_internal(tmp_input, insize, out, outsize, flags);
}
delete[] tmp_input;
}
return result;
}
static int __wchar_forbitten(utf8_wchar_t sym);
static int __utf8_forbitten(unsigned char octet);
static int
__wchar_forbitten(utf8_wchar_t sym)
{
/* Surrogate pairs */
if (sym >= 0xd800 && sym <= 0xdfff)
return (-1);
return (0);
}
static int
__utf8_forbitten(unsigned char octet)
{
switch (octet)
{
case 0xc0:
case 0xc1:
case 0xf5:
case 0xff:
return (-1);
}
return (0);
}
/*
* DESCRIPTION
* This function translates UTF-8 string into UCS-2 or UCS-4 string (all symbols
* will be in local machine byte order).
*
* It takes the following arguments:
* in - input UTF-8 string. It can be null-terminated.
* insize - size of input string in bytes.
* out - result buffer for UCS-2/4 string. If out is NULL,
* function returns size of result buffer.
* outsize - size of out buffer in wide characters.
*
* RETURN VALUES
* The function returns size of result buffer (in wide characters).
* Zero is returned in case of error.
*
* CAVEATS
* 1. If UTF-8 string contains zero symbols, they will be translated
* as regular symbols.
* 2. If UTF8_IGNORE_ERROR or UTF8_SKIP_BOM flag is set, sizes may vary
* when `out' is NULL and not NULL. It's because of special UTF-8
* sequences which may result in forbitten (by RFC3629) UNICODE
* characters. So, the caller must check return value every time and
* not prepare buffer in advance (\0 terminate) but after calling this
* function.
*/
static size_t utf8_to_wchar_internal(const char *in, size_t insize, utf8_wchar_t *out, size_t outsize, int flags)
{
unsigned char *p, *lim;
utf8_wchar_t *wlim, high;
size_t n, total, i, n_bits;
if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
return (0);
total = 0;
p = (unsigned char *)in;
lim = p + insize;
wlim = out + outsize;
for (; p < lim; p += n)
{
if (__utf8_forbitten(*p) != 0 &&
(flags & UTF8_IGNORE_ERROR) == 0)
return (0);
/*
* Get number of bytes for one wide character.
*/
n = 1; /* default: 1 byte. Used when skipping bytes. */
if ((*p & 0x80) == 0)
high = (utf8_wchar_t)*p;
else if ((*p & 0xe0) == _SEQ2)
{
n = 2;
high = (utf8_wchar_t)(*p & 0x1f);
}
else if ((*p & 0xf0) == _SEQ3)
{
n = 3;
high = (utf8_wchar_t)(*p & 0x0f);
}
else if ((*p & 0xf8) == _SEQ4)
{
n = 4;
high = (utf8_wchar_t)(*p & 0x07);
}
else if ((*p & 0xfc) == _SEQ5)
{
n = 5;
high = (utf8_wchar_t)(*p & 0x03);
}
else if ((*p & 0xfe) == _SEQ6)
{
n = 6;
high = (utf8_wchar_t)(*p & 0x01);
}
else
{
if ((flags & UTF8_IGNORE_ERROR) == 0)
return (0);
continue;
}
/* does the sequence header tell us truth about length? */
if (lim - p <= n - 1)
{
if ((flags & UTF8_IGNORE_ERROR) == 0)
return (0);
n = 1;
continue; /* skip */
}
/*
* Validate sequence.
* All symbols must have higher bits set to 10xxxxxx
*/
if (n > 1)
{
for (i = 1; i < n; i++)
{
if ((p[i] & 0xc0) != _NXT)
break;
}
if (i != n)
{
if ((flags & UTF8_IGNORE_ERROR) == 0)
return (0);
n = 1;
continue; /* skip */
}
}
total++;
if (out == NULL)
continue;
if (out >= wlim)
return (0); /* no space left */
uint32_t out_val = 0;
*out = 0;
n_bits = 0;
for (i = 1; i < n; i++)
{
out_val |= (utf8_wchar_t)(p[n - i] & 0x3f) << n_bits;
n_bits += 6; /* 6 low bits in every byte */
}
out_val |= high << n_bits;
bool skip = false;
if (__wchar_forbitten(out_val) != 0)
{
if ((flags & UTF8_IGNORE_ERROR) == 0)
{
return 0; /* forbitten character */
}
else
{
skip = true;
}
}
else if (out_val == _BOM && (flags & UTF8_SKIP_BOM) != 0)
{
skip = true;
}
if (skip)
{
total--;
}
else if (out_val > UTF8_WCHAR_MAX)
{
// wchar_t is UCS-2, but the UTF-8 specified an astral character
return 0;
}
else
{
*out++ = out_val;
}
}
return (total);
}
/*
* DESCRIPTION
* This function translates UCS-2/4 symbols (given in local machine
* byte order) into UTF-8 string.
*
* It takes the following arguments:
* in - input unicode string. It can be null-terminated.
* insize - size of input string in wide characters.
* out - result buffer for utf8 string. If out is NULL,
* function returns size of result buffer.
* outsize - size of result buffer.
*
* RETURN VALUES
* The function returns size of result buffer (in bytes). Zero is returned
* in case of error.
*
* CAVEATS
* If UCS-4 string contains zero symbols, they will be translated
* as regular symbols.
*/
static size_t wchar_to_utf8_internal(const utf8_wchar_t *in, size_t insize, char *out, size_t outsize, int flags)
{
const utf8_wchar_t *w, *wlim;
unsigned char *p, *lim;
size_t total, n;
if (in == NULL || insize == 0 || (outsize == 0 && out != NULL))
return (0);
w = in;
wlim = w + insize;
p = (unsigned char *)out;
lim = p + outsize;
total = 0;
for (; w < wlim; w++)
{
if (__wchar_forbitten(*w) != 0)
{
if ((flags & UTF8_IGNORE_ERROR) == 0)
return (0);
else
continue;
}
if (*w == _BOM && (flags & UTF8_SKIP_BOM) != 0)
continue;
const int32_t w_wide = *w;
if (w_wide < 0)
{
if ((flags & UTF8_IGNORE_ERROR) == 0)
return (0);
continue;
}
else if (w_wide <= 0x0000007f)
n = 1;
else if (w_wide <= 0x000007ff)
n = 2;
else if (w_wide <= 0x0000ffff)
n = 3;
else if (w_wide <= 0x001fffff)
n = 4;
else if (w_wide <= 0x03ffffff)
n = 5;
else /* if (w_wide <= 0x7fffffff) */
n = 6;
total += n;
if (out == NULL)
continue;
if (lim - p <= n - 1)
return (0); /* no space left */
/* extract the wchar_t as big-endian. If wchar_t is UCS-16, the first two bytes will be 0 */
unsigned char oc[4];
uint32_t w_tmp = *w;
oc[3] = w_tmp & 0xFF;
w_tmp >>= 8;
oc[2] = w_tmp & 0xFF;
w_tmp >>= 8;
oc[1] = w_tmp & 0xFF;
w_tmp >>= 8;
oc[0] = w_tmp & 0xFF;
switch (n)
{
case 1:
p[0] = oc[3];
break;
case 2:
p[1] = _NXT | (oc[3] & 0x3f);
p[0] = _SEQ2 | (oc[3] >> 6) | ((oc[2] & 0x07) << 2);
break;
case 3:
p[2] = _NXT | (oc[3] & 0x3f);
p[1] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
p[0] = _SEQ3 | ((oc[2] & 0xf0) >> 4);
break;
case 4:
p[3] = _NXT | (oc[3] & 0x3f);
p[2] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
p[1] = _NXT | ((oc[2] & 0xf0) >> 4) |
((oc[1] & 0x03) << 4);
p[0] = _SEQ4 | ((oc[1] & 0x1f) >> 2);
break;
case 5:
p[4] = _NXT | (oc[3] & 0x3f);
p[3] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
p[2] = _NXT | ((oc[2] & 0xf0) >> 4) |
((oc[1] & 0x03) << 4);
p[1] = _NXT | (oc[1] >> 2);
p[0] = _SEQ5 | (oc[0] & 0x03);
break;
case 6:
p[5] = _NXT | (oc[3] & 0x3f);
p[4] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
p[3] = _NXT | (oc[2] >> 4) | ((oc[1] & 0x03) << 4);
p[2] = _NXT | (oc[1] >> 2);
p[1] = _NXT | (oc[0] & 0x3f);
p[0] = _SEQ6 | ((oc[0] & 0x40) >> 6);
break;
}
/*
* NOTE: do not check here for forbitten UTF-8 characters.
* They cannot appear here because we do proper convertion.
*/
p += n;
}
return (total);
}

41
utf8.h Normal file
View file

@ -0,0 +1,41 @@
/*
* Copyright (c) 2007 Alexey Vatchenko <av@bsdua.org>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/*
* utf8: implementation of UTF-8 charset encoding (RFC3629).
*/
#ifndef _UTF8_H_
#define _UTF8_H_
#include <sys/types.h>
#include <string>
#include <wchar.h>
#define UTF8_IGNORE_ERROR 0x01
#define UTF8_SKIP_BOM 0x02
/* Convert a string between UTF8 and UCS-2/4 (depending on size of wchar_t). Returns true if successful, storing the result of the conversion in *result */
bool utf8_to_wchar_string(const std::string &input, std::wstring *result);
bool wchar_to_utf8_string(const std::wstring &input, std::string *result);
/* Variants exposed for testing */
size_t utf8_to_wchar(const char *in, size_t insize, wchar_t *out, size_t outsize, int flags);
size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, size_t outsize, int flags);
bool is_wchar_ucs2();
#endif /* !_UTF8_H_ */