tstUtf8.cpp revision 40d4f06178494f9d0b7eb93819fd5b5d3548def4
/* $Id$ */
/** @file
* IPRT Testcase - UTF-8 and UTF-16 string conversions.
*/
/*
* Copyright (C) 2006-2007 Oracle Corporation
*
* This file is part of VirtualBox Open Source Edition (OSE), as
* available from http://www.virtualbox.org. This file is free software;
* General Public License (GPL) as published by the Free Software
* Foundation, in version 2 as it comes in the "COPYING" file of the
* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
*
* The contents of this file may alternatively be used under the terms
* of the Common Development and Distribution License Version 1.0
* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
* VirtualBox OSE distribution, in which case the provisions of the
* CDDL are applicable instead of those of the GPL.
*
* You may elect to license modified versions of this file under the
* terms and conditions of either the GPL or the CDDL or both.
*/
/*******************************************************************************
* Header Files *
*******************************************************************************/
/**
* Generate a random codepoint for simple UTF-16 encoding.
*/
static RTUTF16 GetRandUtf16(void)
{
do
{
return wc;
}
/**
*
*/
{
static const char s_szBadString1[] = "Bad \xe0\x13\x0";
static const char s_szBadString2[] = "Bad \xef\xbf\xc3";
int rc;
char *pszUtf8;
char *pszCurrent;
/*
* Invalid UTF-8 to UCS-2 test.
*/
(hTest, "Conversion of first bad UTF-8 string to UTF-16 apparently succeeded. It shouldn't. rc=%Rrc\n", rc));
(hTest, "Conversion of second bad UTF-8 strings to UTF-16 apparently succeeded. It shouldn't. rc=%Rrc\n", rc));
/*
* Test current CP conversion.
*/
for (int i = 0; i < 30; i++)
pwszRand[i] = GetRandUtf16();
pwszRand[30] = 0;
if (rc == VINF_SUCCESS)
{
if (rc == VINF_SUCCESS)
{
if (rc == VINF_SUCCESS)
else
RTTestFailed(hTest, "%d: The third part of random UTF-16 -> UTF-8 -> Current -> UTF-8 failed with return value %Rrc.",
}
else if (rc == VERR_NO_TRANSLATION)
RTTestPassed(hTest, "The second part of random UTF-16 -> UTF-8 -> Current -> UTF-8 returned VERR_NO_TRANSLATION. This is probably as it should be.\n");
else
RTTestFailed(hTest, "%d: The second part of random UTF-16 -> UTF-8 -> Current -> UTF-8 failed with return value %Rrc.",
}
else
RTTestFailed(hTest, "%d: The first part of random UTF-16 -> UTF-8 -> Current -> UTF-8 failed with return value %Rrc.",
/*
* Generate a new random string.
*/
for (int i = 0; i < 30; i++)
pwszRand[i] = GetRandUtf16();
pwszRand[30] = 0;
if (rc == VINF_SUCCESS)
{
if (rc == VINF_SUCCESS)
{
int i;
/* nothing */;
else
{
RTTestPrintf(hTest, RTTESTLVL_FAILURE, "First differing character is at position %d and has the value %x.\n", i, pwsz[i]);
}
}
else
RTTestFailed(hTest, "%d: The second part of random UTF-16 -> UTF-8 -> UTF-16 failed with return value %Rrc.",
}
else
RTTestFailed(hTest, "%d: The first part of random UTF-16 -> UTF-8 -> UTF-16 failed with return value %Rrc.",
/*
* Generate yet another random string and convert it to a buffer.
*/
for (int i = 0; i < 30; i++)
pwszRand[i] = GetRandUtf16();
pwszRand[30] = 0;
char szUtf8Array[120];
char *pszUtf8Array = szUtf8Array;
if (rc == 0)
{
if (rc == 0)
{
int i;
;
if (pwsz[i] == 0 && i >= 8)
else
{
RTTestFailed(hTest, "%d: Incorrect conversion of UTF-16 -> fixed length UTF-8 -> UTF-16.\n", __LINE__);
RTTestPrintf(hTest, RTTESTLVL_FAILURE, "First differing character is at position %d and has the value %x.\n", i, pwsz[i]);
}
}
else
RTTestFailed(hTest, "%d: The second part of random UTF-16 -> fixed length UTF-8 -> UTF-16 failed with return value %Rrc.\n", __LINE__, rc);
}
else
RTTestFailed(hTest, "%d: The first part of random UTF-16 -> fixed length UTF-8 -> UTF-16 failed with return value %Rrc.\n", __LINE__, rc);
/*
* And again.
*/
for (int i = 0; i < 30; i++)
pwszRand[i] = GetRandUtf16();
pwszRand[30] = 0;
if (rc == 0)
{
if (rc == 0)
{
int i;
;
else
{
RTTestFailed(hTest, "%d: Incorrect conversion of random UTF-16 -> UTF-8 -> fixed length UTF-16.\n", __LINE__);
RTTestPrintf(hTest, RTTESTLVL_FAILURE, "First differing character is at position %d and has the value %x.\n", i, pwsz2Buf[i]);
}
}
else
RTTestFailed(hTest, "%d: The second part of random UTF-16 -> UTF-8 -> fixed length UTF-16 failed with return value %Rrc.\n", __LINE__, rc);
}
else
RTTestFailed(hTest, "%d: The first part of random UTF-16 -> UTF-8 -> fixed length UTF-16 failed with return value %Rrc.\n",
for (int i = 0; i < 30; i++)
pwszRand[i] = GetRandUtf16();
pwszRand[30] = 0;
if (rc == VERR_BUFFER_OVERFLOW)
RTTestPassed(hTest, "Random UTF-16 -> fixed length UTF-8 with too short buffer successfully rejected.\n");
else
RTTestFailed(hTest, "%d: Random UTF-16 -> fixed length UTF-8 with too small buffer returned value %d instead of VERR_BUFFER_OVERFLOW.\n",
/*
* last time...
*/
for (int i = 0; i < 30; i++)
pwszRand[i] = GetRandUtf16();
pwszRand[30] = 0;
if (rc == VINF_SUCCESS)
{
if (rc == VERR_BUFFER_OVERFLOW)
RTTestPassed(hTest, "Random UTF-16 -> UTF-8 -> fixed length UTF-16 with too short buffer successfully rejected.\n");
else
RTTestFailed(hTest, "%d: The second part of random UTF-16 -> UTF-8 -> fixed length UTF-16 with too short buffer returned value %Rrc instead of VERR_BUFFER_OVERFLOW.\n",
}
else
RTTestFailed(hTest, "%d:The first part of random UTF-16 -> UTF-8 -> fixed length UTF-16 failed with return value %Rrc.\n",
}
static char g_szAll[0x7f + (0x800 - 0x80) * 2 + (0xfffe - 0x800 - (0xe000 - 0xd800))* 3 + (0x110000 - 0x10000) * 4 + 1];
{
if (cBits == 8)
{
if (off < 0x7f)
else if (off < 0xf7f)
else if (off < 0x27f7f)
else if (off < 0x2df79)
else if (off < 0x42df79)
else
}
else if (cBits == 16)
{
else if (off < 0x20f7fd)
else
}
else
{
else
RTTestPrintf(NIL_RTTEST, RTTESTLVL_FAILURE, "RTUNICP U+%#x\n", off / sizeof(RTUNICP) + 0x800 + 1 + 2);
}
}
{
{
{
RTTestPrintf(NIL_RTTEST, RTTESTLVL_FAILURE, " %#x: %02x != %02x!\n", off-1, pb1[off-1], pb2[off-1]);
RTTestPrintf(NIL_RTTEST, RTTESTLVL_FAILURE, " %#x: %02x != %02x!\n", off+1, pb1[off+1], pb2[off+1]);
RTTestPrintf(NIL_RTTEST, RTTESTLVL_FAILURE, " %#x: %02x != %02x!\n", off+2, pb1[off+2], pb2[off+2]);
RTTestPrintf(NIL_RTTEST, RTTESTLVL_FAILURE, " %#x: %02x != %02x!\n", off+3, pb1[off+3], pb2[off+3]);
RTTestPrintf(NIL_RTTEST, RTTESTLVL_FAILURE, " %#x: %02x != %02x!\n", off+4, pb1[off+4], pb2[off+4]);
RTTestPrintf(NIL_RTTEST, RTTESTLVL_FAILURE, " %#x: %02x != %02x!\n", off+5, pb1[off+5], pb2[off+5]);
RTTestPrintf(NIL_RTTEST, RTTESTLVL_FAILURE, " %#x: %02x != %02x!\n", off+6, pb1[off+6], pb2[off+6]);
RTTestPrintf(NIL_RTTEST, RTTESTLVL_FAILURE, " %#x: %02x != %02x!\n", off+7, pb1[off+7], pb2[off+7]);
RTTestPrintf(NIL_RTTEST, RTTESTLVL_FAILURE, " %#x: %02x != %02x!\n", off+8, pb1[off+8], pb2[off+8]);
RTTestPrintf(NIL_RTTEST, RTTESTLVL_FAILURE, " %#x: %02x != %02x!\n", off+9, pb1[off+9], pb2[off+9]);
return 1;
}
}
return 0;
}
void InitStrings()
{
/*
* Generate unicode string containing all the legal UTF-16 codepoints, both UTF-16 and UTF-8 version.
*/
/* the simple code point array first */
unsigned i = 0;
while (uc < 0xd800)
uc = 0xe000;
while (uc < 0xfffe)
uc = 0x10000;
while (uc < 0x110000)
g_uszAll[i++] = 0;
/* the utf-16 one */
i = 0;
uc = 1;
//RTPrintf("tstUtf8: %#x=%#x", i, uc);
while (uc < 0xd800)
uc = 0xe000;
//RTPrintf(" %#x=%#x", i, uc);
while (uc < 0xfffe)
uc = 0x10000;
//RTPrintf(" %#x=%#x", i, uc);
while (uc < 0x110000)
{
uc++;
}
//RTPrintf(" %#x=%#x\n", i, uc);
g_wszAll[i++] = '\0';
/*
* The utf-8 one
*/
i = 0;
uc = 1;
//RTPrintf("tstUtf8: %#x=%#x", i, uc);
while (uc < 0x80)
//RTPrintf(" %#x=%#x", i, uc);
while (uc < 0x800)
{
uc++;
}
//RTPrintf(" %#x=%#x", i, uc);
while (uc < 0xd800)
{
uc++;
}
uc = 0xe000;
//RTPrintf(" %#x=%#x", i, uc);
while (uc < 0xfffe)
{
uc++;
}
uc = 0x10000;
//RTPrintf(" %#x=%#x", i, uc);
while (uc < 0x110000)
{
uc++;
}
//RTPrintf(" %#x=%#x\n", i, uc);
g_szAll[i++] = '\0';
}
{
/*
* Convert to UTF-8 and back.
*/
char *pszUtf8;
if (rc == VINF_SUCCESS)
{
if (rc == VINF_SUCCESS)
{
}
else
}
else
/*
* Convert to UTF-16 and back. (just in case the above test fails)
*/
if (rc == VINF_SUCCESS)
{
if (rc == VINF_SUCCESS)
{
}
else
}
else
/*
* Convert UTF-8 to CPs.
*/
if (rc == VINF_SUCCESS)
{
if (rc == VINF_SUCCESS)
{
}
else
/** @todo RTCpsToUtf8 or something. */
}
else
/*
* Check the various string lengths.
*/
//size_t cuc3 = RTUniLen(g_uszAll);
/*
* Enumerate the strings.
*/
char *pszPut1 = pszPut1Base;
for (;;)
{
/*
* getters
*/
if (RT_FAILURE(rc))
{
break;
}
{
break;
}
{
RTTestFailed(hTest, "RTStrGetCpEx and RTStrGetCp returned different CPs: %RTunicp != %RTunicp", uc2, uc1);
break;
}
{
break;
}
if (RT_FAILURE(rc))
{
break;
}
{
RTTestFailed(hTest, "RTUtf16GetCpEx and RTStrGetCp returned different CPs: %RTunicp != %RTunicp", uc3, uc2);
break;
}
{
RTTestFailed(hTest, "RTUtf16GetCpEx and RTUtf16GetCp returned different CPs: %RTunicp != %RTunicp", uc3, uc4);
break;
}
{
break;
}
/*
* putters
*/
{
break;
}
{
break;
}
/* the end? */
if (!uc1)
break;
}
/* check output if we seems to have made it thru it all. */
{
}
}
/**
* Check case insensitivity.
*/
{
static RTUTF16 s_wszTst1a[] = { 'a', 'B', 'c', 'D', 'E', 'f', 'g', 'h', 'i', 'j', 'K', 'L', 'm', 'N', 'o', 'P', 'q', 'r', 'S', 't', 'u', 'V', 'w', 'x', 'Y', 'Z', 0xc5, 0xc6, 0xf8, 0 };
static RTUTF16 s_wszTst1b[] = { 'A', 'B', 'c', 'd', 'e', 'F', 'G', 'h', 'i', 'J', 'k', 'l', 'M', 'n', 'O', 'p', 'Q', 'R', 's', 't', 'U', 'v', 'w', 'X', 'y', 'z', 0xe5, 0xe6, 0xd8, 0 };
)
)
}
/**
* Test the RTStr*Cmp functions.
*/
{
do \
{ \
} while (0)
/** @todo test the non-ascii bits. */
CHECK_DIFF(RTStrNICmp("AbCdEG", "aBcDef", RTSTR_MAX), > ); /* diff performed on the lower case cp. */
/* We should continue using byte comparison when we hit the invalid CP. Will assert in debug builds. */
// CHECK_DIFF(RTStrNICmp("AbCd\xff""eg", "aBcD\xff""eF", 6), ==);
}
/**
* Check UTF-8 encoding purging.
*/
{
/*
* Test some good strings.
*/
char sz1[] = "1234567890wertyuiopsdfghjklzxcvbnm";
if (pszAll)
{
}
/*
* Test some bad stuff.
*/
struct
{
unsigned char szIn[5];
const char *pszExpect;
} aTests[] =
{
{ 0, { '1', '2', '3', '4', '\0' }, "1234" },
{ 1, { 0x80, '2', '3', '4', '\0' }, "?234" },
{ 1, { '1', 0x80, '3', '4', '\0' }, "1?34" },
{ 1, { '1', '2', 0x80, '4', '\0' }, "12?4" },
{ 1, { '1', '2', '3', 0x80, '\0' }, "123?" },
{ 2, { 0x80, 0x81, '3', '4', '\0' }, "??34" },
{ 2, { '1', 0x80, 0x81, '4', '\0' }, "1??4" },
{ 2, { '1', '2', 0x80, 0x81, '\0' }, "12??" },
};
{
RTTestFailed(hTest, "#%u: %.5Rhxs expected %.5Rhxs (%s)\n", i, aTests[i].szIn, aTests[i].pszExpect, aTests[i].pszExpect);
}
}
/**
* Check string sanitising.
*/
{
'\0' };
struct
{
const char *pcszIn;
const char *pcszOut;
char chReplacement;
}
aTests[] =
{
{ "123654wert\xc2\xa2trew\xe2\x82\xac""4321",
};
enum { MAX_IN_STRING = 256 };
for (unsigned i = 0; i < RT_ELEMENTS(aTests); ++i)
{
char szCopy[MAX_IN_STRING];
aTests[i].chReplacement);
(long long) cReplacements);
}
}
/**
* Check string sanitising.
*/
{
'\0' };
struct
{
const char *pcszIn;
const char *pcszOut;
char chReplacement;
}
aTests[] =
{
{ "123654wert\xc2\xa2trew\xe2\x82\xac""4321",
/* No test for invalid encoding. */
};
enum { MAX_IN_STRING = 256 };
for (unsigned i = 0; i < RT_ELEMENTS(aTests); ++i)
{
aTests[i].chReplacement);
(long long) cReplacements);
}
}
/**
* Benchmark stuff.
*/
{
static union
{
} s_Buf;
/** @todo add RTTest* methods for reporting benchmark results. */
RTTestPrintf(hTest, RTTESTLVL_ALWAYS, "Benchmarking RTStrToUtf16Ex: "); /** @todo figure this stuff into the test framework. */
if (RT_SUCCESS(rc))
{
int i;
for (i = 0; i < 100; i++)
{
if (RT_FAILURE(rc))
{
break;
}
}
}
if (RT_SUCCESS(rc))
{
int i;
for (i = 0; i < 100; i++)
{
if (RT_FAILURE(rc))
{
break;
}
}
}
}
/**
* Tests RTStrEnd
*/
{
}
/**
* Tests RTStrStr and RTStrIStr.
*/
{
#define CHECK_NULL(expr) \
do { \
} while (0)
do { \
) \
} while (0)
}
{
/* Test Utf8 -> Latin1 */
char szShort[256] = { 0 };
if (RT_SUCCESS(rc))
{
if (psz[i] != (char) j)
{
break;
}
}
char sz[512];
&cchActual);
&cchActual);
if (RT_SUCCESS(rc))
{
if (psz2[i] != (char) j)
{
break;
}
}
if (RT_SUCCESS(rc))
{
if ( ((j < 0x100) && (psz[i] != (char) j))
{
break;
}
}
const char *pszBad = "Hello\xDC\xD8";
&cchActual);
/* Test Latin1 -> Utf8 */
const char *pszLat1 = "\x01\x20\x40\x80\x81";
if (RT_SUCCESS(rc))
if (RT_SUCCESS(rc))
char ch[8];
cchActual = 0;
if (RT_SUCCESS(rc))
if (RT_SUCCESS(rc))
{
}
if (RT_SUCCESS(rc))
&cchActual);
if (RT_SUCCESS(rc))
{
}
&cchActual);
if (RT_SUCCESS(rc))
{
}
&cchActual);
}
{
/* Test Utf16 -> Latin1 */
for (unsigned i = 0; i < 255; ++i)
wszShort[i] = i + 1;
if (RT_SUCCESS(rc))
{
if (psz[i] != (char) j)
{
break;
}
}
char sz[512];
&cchActual);
&cchActual);
if (RT_SUCCESS(rc))
{
if (psz2[i] != (char) j)
{
break;
}
}
if (RT_SUCCESS(rc))
{
if ( ((j < 0x100) && (psz[i] != (char) j))
{
break;
}
}
const char *pszBad = "H\0e\0l\0l\0o\0\0\xDC\0\xD8\0";
&cchActual);
/* Test Latin1 -> Utf16 */
const char *pszLat1 = "\x01\x20\x40\x80\x81";
if (RT_SUCCESS(rc))
if (RT_SUCCESS(rc))
if (RT_SUCCESS(rc))
if (RT_SUCCESS(rc))
{
}
if (RT_SUCCESS(rc))
&cwActual);
if (RT_SUCCESS(rc))
{
}
&cwActual);
if (RT_SUCCESS(rc))
{
}
&cwActual);
}
{
/*
* Try trigger a VERR_NO_TRANSLATION error in convert to
* current CP to latin-1.
*/
0x0092, 0x00c1, 0x00f2, 0x1f80, 0x0088, 0x2c38, 0x2c30, 0x0000 };
char *pszTest1;
char *pszOut;
if (RT_SUCCESS(rc))
{
}
else
if (RT_SUCCESS(rc))
}
{
/*
* Test RTStrPutCp, RTStrGetCp and RTStrGetCpEx.
*/
while (uc <= 0x10fffd)
{
/* Figure the range - skip illegal ranges. */
ucFirst = 0xe000;
ucFirst = 0x10000;
ucLast = 0xd7ff;
ucLast = 0xfffd;
/* Encode the range into a string, decode each code point as we go along. */
char sz1[8192];
{
}
/* Decode and re-encode it. */
{
}
/* Decode and wipe it (checking compiler optimizations). */
{
}
/* advance */
}
}
int main()
{
/*
* Init the runtime, test and say hello.
*/
if (rcExit != RTEXITCODE_SUCCESS)
return rcExit;
/*
* Run the tests.
*/
InitStrings();
/*
* Summary
*/
return RTTestSummaryAndDestroy(hTest);
}