uniread.cpp revision c58f1213e628a545081c70e26c6b67a841cff880
/* $Id$ */
/** @file
* IPRT - Unicode Specification Reader.
*/
/*
* Copyright (C) 2006-2012 Oracle Corporation
*
* This file is part of VirtualBox Open Source Edition (OSE), as
* available from http://www.virtualbox.org. This file is free software;
* General Public License (GPL) as published by the Free Software
* Foundation, in version 2 as it comes in the "COPYING" file of the
* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
*
* The contents of this file may alternatively be used under the terms
* of the Common Development and Distribution License Version 1.0
* (CDDL) only, as it comes in the "COPYING.CDDL" file of the
* VirtualBox OSE distribution, in which case the provisions of the
* CDDL are applicable instead of those of the GPL.
*
* You may elect to license modified versions of this file under the
* terms and conditions of either the GPL or the CDDL or both.
*/
/*******************************************************************************
* Header Files *
*******************************************************************************/
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
/*******************************************************************************
* Global Variables *
*******************************************************************************/
/** When set, no output is produced. Very useful when debugging ths code. */
static bool g_fQuiet = false;
/** The file we're currently parsing. */
static const char *g_pszCurFile;
/** The current line number. */
static unsigned g_iLine;
/**
* Exit the program after printing a parse error.
*
* @param pszFormat The message.
* @param ... Format arguments.
*/
static void ParseError(const char *pszFormat, ...)
{
exit(1);
}
/**
* Strip a line.
* @returns pointer to first non-blank char.
* @param pszLine The line string to strip.
*/
{
pszLine++;
if (psz)
*psz = '\0';
else
{
switch (psz[-1])
{
case ' ':
case '\t':
case '\n':
case '\r':
*--psz = '\0';
continue;
}
break;
}
return pszLine;
}
/**
* Checks if the line is blank or a comment line and should be skipped.
* @param pszLine The line to consider.
*/
static bool IsCommentOrBlankLine(const char *pszLine)
{
pszLine++;
}
/**
* Get the first field in the string.
*
* @returns Pointer to the next field.
* @param ppsz Where to store the pointer to the next field.
* @param pszLine The line string. (could also be *ppsz from a FirstNext call)
*/
{
if (!psz)
else
{
*psz = '\0';
}
/* strip */
pszLine++;
{
switch (psz[-1])
{
case ' ':
case '\t':
case '\n':
case '\r':
*--psz = '\0';
continue;
}
break;
}
return pszLine;
}
/**
* Get the next field in a field enumeration.
*
* @returns Pointer to the next field.
* @param ppsz Where to get and store the string position.
*/
{
}
/**
* Splits a decomposition field.
*
* This may start with a type that is enclosed in angle brackets.
*
* @returns Pointer to the mapping values following the type. @a *ppsz if empty.
* @param ppszType Pointer to the type field pointer. On input the type
* field contains the combined type and mapping string. On
* output this should only contain the type, no angle
* brackets. If no type specified, it is replaced with an
* empty string (const).
*/
static char *SplitDecompField(char **ppszType)
{
/* Empty field? */
if (!*psz)
return psz;
/* No type? */
if (*psz != '<')
{
*ppszType = (char *)"";
return psz;
}
/* Split out the type. */
if (!psz)
{
ParseError("Bad Decomposition Type/Mappings\n");
return *ppszType;
}
*psz++ = '\0';
if (!*psz)
ParseError("Missing decomposition mappings\n");
return psz;
}
/**
* Converts a code point field to a number.
* @returns Code point.
* @param psz The field string.
*/
{
}
/**
* Same as ToNum except that if the field is empty the Default is returned.
*/
{
if (*psz)
return Default;
}
/**
* Converts a code point range to numbers.
* @returns The start code point.\
* @returns ~(RTUNICP)0 on failure.
* @param psz The field string.
* @param pLast Where to store the last code point in the range.
*/
{
{
if (*pszEnd == '.')
{
while (*pszEnd == '.')
pszEnd++;
{
return ~(RTUNICP)0;
}
}
else
{
return ~(RTUNICP)0;
}
}
}
/**
* For converting the decomposition mappings field and similar.
*
* @returns Mapping array or NULL if none.
* @param psz The string to convert. Can be empty.
* @param pcEntries Where to store the number of entries.
* @param cMax The max number of entries.
*/
{
unsigned cAlloc = 0;
unsigned i = 0;
/* Convert the code points. */
while (psz)
{
/* skip leading spaces */
while (RT_C_IS_BLANK(*psz))
psz++;
/* the end? */
if (!*psz)
break;
/* room left? */
if (i >= cMax)
{
ParseError("Too many mappings.\n");
break;
}
if (i >= cAlloc)
{
cAlloc += 4;
if (!paCps)
{
exit(1);
}
}
/* Find the end. */
while (RT_C_IS_XDIGIT(*psz))
psz++;
ParseError("Malformed mappings.\n");
if (*psz)
*psz++ = '\0';
/* Convert to number and add it. */
}
*pcEntries = i;
return paCps;
}
/**
* Duplicate a string, optimize certain strings to save memory.
*
* @returns Pointer to string copy.
* @param pszStr The string to duplicate.
*/
{
if (!*pszStr)
return (char*)"";
if (psz)
return psz;
exit(1);
}
/**
* Array of all possible and impossible unicode code points as of 4.1
*/
struct CPINFO
{
unsigned CanonicalCombiningClass;
const char *pszDecompositionType;
unsigned cDecompositionMapping;
const char *pszName;
/** Set if this is an unused entry */
unsigned fNullEntry : 1;
unsigned fAlphabetic : 1;
unsigned fASCIIHexDigit : 1;
unsigned fBidiControl : 1;
unsigned fCaseIgnorable : 1;
unsigned fCased : 1;
unsigned fChangesWhenCasefolded : 1;
unsigned fChangesWhenCasemapped : 1;
unsigned fChangesWhenLowercased : 1;
unsigned fChangesWhenTitlecased : 1;
unsigned fChangesWhenUppercased : 1;
unsigned fDash : 1;
unsigned fDefaultIgnorableCodePoint : 1;
unsigned fDeprecated : 1;
unsigned fDiacritic : 1;
unsigned fExtender : 1;
unsigned fGraphemeBase : 1;
unsigned fGraphemeExtend : 1;
unsigned fGraphemeLink : 1;
unsigned fHexDigit : 1;
unsigned fHyphen : 1;
unsigned fIDContinue : 1;
unsigned fIdeographic : 1;
unsigned fIDSBinaryOperator : 1;
unsigned fIDStart : 1;
unsigned fIDSTrinaryOperator : 1;
unsigned fJoinControl : 1;
unsigned fLogicalOrderException : 1;
unsigned fLowercase : 1;
unsigned fMath : 1;
unsigned fNoncharacterCodePoint : 1;
unsigned fOtherAlphabetic : 1;
unsigned fOtherDefaultIgnorableCodePoint : 1;
unsigned fOtherGraphemeExtend : 1;
unsigned fOtherIDContinue : 1;
unsigned fOtherIDStart : 1;
unsigned fOtherLowercase : 1;
unsigned fOtherMath : 1;
unsigned fOtherUppercase : 1;
unsigned fPatternSyntax : 1;
unsigned fPatternWhiteSpace : 1;
unsigned fQuotationMark : 1;
unsigned fRadical : 1;
unsigned fSoftDotted : 1;
unsigned fSTerm : 1;
unsigned fTerminalPunctuation : 1;
unsigned fUnifiedIdeograph : 1;
unsigned fUppercase : 1;
unsigned fVariationSelector : 1;
unsigned fWhiteSpace : 1;
unsigned fXIDContinue : 1;
unsigned fXIDStart : 1;
/** @name DerivedNormalizationProps.txt
* @{ */
unsigned fFullCompositionExclusion : 1;
unsigned fInvNFC_QC : 2; /**< If 1 (NFC_QC == N) then code point 100% sure not part of NFC string. */
unsigned fInvNFD_QC : 2; /**< If 1 (NFD_QC == N) then code point 100% sure not part of NFD string. */
unsigned fInvNFKC_QC : 2;
unsigned fInvNFKD_QC : 2;
unsigned fExpandsOnNFC : 1;
unsigned fExpandsOnNFD : 1;
unsigned fExpandsOnNFKC : 1;
unsigned fExpandsOnNFKD : 1;
/** @} */
/* unprocessed stuff, so far. */
const char *pszGeneralCategory;
const char *pszBidiClass;
const char *pszNumericType;
const char *pszNumericValueD;
const char *pszNumericValueN;
const char *pszBidiMirrored;
const char *pszUnicode1Name;
const char *pszISOComment;
} g_aCPInfo[0x110000];
/**
* Creates a 'null' entry at i.
* @param i The entry in question.
*/
static void NullEntry(unsigned i)
{
g_aCPInfo[i].SimpleUpperCaseMapping = i;
g_aCPInfo[i].SimpleLowerCaseMapping = i;
g_aCPInfo[i].SimpleTitleCaseMapping = i;
g_aCPInfo[i].cDecompositionMapping = 0;
}
/**
* Open a file for reading, optionally with a base path prefixed.
*
* @returns file stream on success, NULL w/ complaint on failure.
* @param pszBasePath The base path, can be NULL.
* @param pszFilename The name of the file to open.
*/
{
if ( !pszBasePath
|| *pszFilename == '/'
|| *pszFilename == '\\'
#endif
)
{
if (!pFile)
}
else
{
if (!pszFullName)
{
fprintf(stderr, "uniread: failed to allocate %d bytes\n", (int)(cchBasePath + 1 + cchFilename + 1));
return NULL;
}
if (!pFile)
}
g_iLine = 0;
return pFile;
}
/**
* Wrapper around fgets that keep track of the line number.
*
* @returns See fgets.
* @param pszBuf The buffer. See fgets for output definition.
* @param cbBuf The buffer size.
* @param pFile The file to read from.
*/
{
g_iLine++;
}
/**
* Closes a file opened by OpenFile
*
* @param pFile The file to close.
*/
{
g_pszCurFile = NULL;
g_iLine = 0;
}
/**
* Read the UnicodeData.txt file.
* @returns 0 on success.
* @returns !0 on failure.
* @param pszBasePath The base path, can be NULL.
* @param pszFilename The name of the file.
*/
{
/*
* Open input.
*/
if (!pFile)
return 1;
/*
* Parse the input and spit out the output.
*/
char szLine[4096];
RTUNICP i = 0;
{
if (IsCommentOrBlankLine(szLine))
continue;
char *pszCurField;
{
continue;
}
/* catchup? */
while (i < CodePoint)
NullEntry(i++);
if (i != CodePoint)
{
return 1;
}
/* this one */
g_aCPInfo[i].fNullEntry = 0;
g_aCPInfo[i].paDecompositionMapping = ToMapping(pszDecompositionMapping, &g_aCPInfo[i].cDecompositionMapping, 20);
i++;
}
/* catchup? */
while (i < RT_ELEMENTS(g_aCPInfo))
NullEntry(i++);
return 0;
}
/**
* Generates excluded data.
*
* @returns 0 on success, exit code on failure.
*/
static int GenerateExcludedData(void)
{
/*
* Hangul Syllables U+AC00 to U+D7A3.
*/
{
g_aCPInfo[i].fNullEntry = 0;
/** @todo generate the decomposition: http://unicode.org/reports/tr15/#Hangul
* */
}
/** @todo
* CJK Ideographs Extension A (U+3400 - U+4DB5)
* CJK Ideographs (U+4E00 - U+9FA5)
* CJK Ideograph Extension B (U+20000 - U+2A6D6)
* CJK Ideograph Extension C (U+2A700 - U+2B734)
*/
return 0;
}
/**
* Worker for ApplyProperty that handles a yes, no, maybe property value.
*
* @returns 0 (NO), 1 (YES), 2 (MAYBE).
* @param ppszNextField The field cursor, input and output.
*/
static int YesNoMaybePropertyValue(char **ppszNextField)
{
if (!**ppszNextField)
{
ParseError("Missing Y/N/M field\n");
return 0;
}
return 0;
return 1;
return 2;
return 0;
}
/**
* Inverted version of YesNoMaybePropertyValue
*
* @returns 1 (NO), 0 (YES), 2 (MAYBE).
* @param ppszNextField The field cursor, input and output.
*/
static int YesNoMaybePropertyValueInv(char **ppszNextField)
{
switch (rc)
{
case 0: return 1;
case 1: return 0;
default: return rc;
}
}
/**
* Applies a property to a code point.
*
* @param StartCP The code point.
* @param pszProperty The property name.
*/
{
{
return;
}
/* string switch */
else if (!strcmp(pszProperty, "Default_Ignorable_Code_Point")) pCPInfo->fDefaultIgnorableCodePoint = 1;
else if (!strcmp(pszProperty, "Other_Default_Ignorable_Code_Point")) pCPInfo->fOtherDefaultIgnorableCodePoint = 1;
/* DerivedNormalizationProps: */
else if (!strcmp(pszProperty, "Full_Composition_Exclusion")) pCPInfo->fFullCompositionExclusion = 1;
else if (!strcmp(pszProperty, "NFC_QC")) pCPInfo->fInvNFC_QC = YesNoMaybePropertyValueInv(&pszNextField);
else if (!strcmp(pszProperty, "NFD_QC")) pCPInfo->fInvNFD_QC = YesNoMaybePropertyValueInv(&pszNextField);
else if (!strcmp(pszProperty, "NFKC_QC")) pCPInfo->fInvNFKC_QC = YesNoMaybePropertyValueInv(&pszNextField);
else if (!strcmp(pszProperty, "NFKD_QC")) pCPInfo->fInvNFKD_QC = YesNoMaybePropertyValueInv(&pszNextField);
else
{
return;
}
if (pszNextField && *pszNextField)
}
/**
* Reads a property file.
*
* There are several property files, this code can read all
* of those but will only make use of the properties it recognizes.
*
* @returns 0 on success.
* @returns !0 on failure.
* @param pszBasePath The base path, can be NULL.
* @param pszFilename The name of the file.
*/
{
/*
* Open input.
*/
if (!pFile)
return 1;
/*
* Parse the input and spit out the output.
*/
char szLine[4096];
{
if (IsCommentOrBlankLine(szLine))
continue;
char *pszCurField;
if (!*pszProperty)
{
ParseError("no property field.\n");
continue;
}
continue;
}
return 0;
}
/**
* Append a flag to the string.
*/
{
{
*pszEnd++ = ' ';
*pszEnd++ = '|';
*pszEnd++ = ' ';
}
return psz;
}
/**
* Calcs the flags for a code point.
* @returns true if there is a flag.
* @returns false if the isn't.
*/
{
pszFlags[0] = '\0';
/** @todo read the specs on this other vs standard stuff, and check out the finer points */
if (pInfo->fWhiteSpace)
//if (pInfo->???)
// AppendFlag(pszFlags, "RTUNI_BSPACE");
{
}
if (!*pszFlags)
{
pszFlags[0] = '0';
return false;
}
return true;
}
/**
* printf wrapper for the primary output stream.
*
* @returns See vfprintf.
* @param pszFormat The vfprintf format string.
* @param ... The format arguments.
*/
static int Stream1Printf(const char *pszFormat, ...)
{
int cch;
if (!g_fQuiet)
else
return cch;
}
/** the data store for stream two. */
static char g_szStream2[10240];
static unsigned volatile g_offStream2 = 0;
/**
* Initializes the 2nd steam.
*/
static void Stream2Init(void)
{
g_szStream2[0] = '\0';
g_offStream2 = 0;
}
/**
* Flushes the 2nd stream to stdout.
*/
static int Stream2Flush(void)
{
Stream2Init();
return 0;
}
/**
* printf to the 2nd stream.
*/
static int Stream2Printf(const char *pszFormat, ...)
{
unsigned offStream2 = g_offStream2;
offStream2 += cch;
if (offStream2 >= sizeof(g_szStream2))
{
exit(1);
}
return cch;
}
/**
* Print the unidata.cpp file header and include list.
*/
int PrintHeader(const char *argv0)
{
Stream1Printf("/** @file\n"
" *\n"
" * IPRT - Unicode Tables.\n"
" *\n"
" */\n"
"\n"
"/*\n"
" * Copyright (C) 2006-2010 Oracle Corporation\n"
" *\n"
" * This file is part of VirtualBox Open Source Edition (OSE), as\n"
" * available from http://www.virtualbox.org. This file is free software;\n"
" * General Public License (GPL) as published by the Free Software\n"
" * Foundation, in version 2 as it comes in the \"COPYING\" file of the\n"
" * VirtualBox OSE distribution. VirtualBox OSE is distributed in the\n"
" * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.\n"
" *\n"
" * The contents of this file may alternatively be used under the terms\n"
" * of the Common Development and Distribution License Version 1.0\n"
" * (CDDL) only, as it comes in the \"COPYING.CDDL\" file of the\n"
" * VirtualBox OSE distribution, in which case the provisions of the\n"
" * CDDL are applicable instead of those of the GPL.\n"
" *\n"
" * You may elect to license modified versions of this file under the\n"
" * terms and conditions of either the GPL or the CDDL or both.\n"
" */\n"
"\n"
"\n",
argv0);
return 0;
}
/**
* Print the flag tables.
*/
int PrintFlags(void)
{
/*
* Print flags table.
*/
Stream2Init();
Stream2Printf("const RTUNIFLAGSRANGE g_aRTUniFlagsRanges[] =\n"
"{\n");
RTUNICP i = 0;
int iStart = -1;
while (i < RT_ELEMENTS(g_aCPInfo))
{
/* figure how far off the next chunk is */
char szFlags[256];
unsigned iNonNull = i;
&& iNonNull >= 256
iNonNull++;
{
if (iStart >= 0)
{
Stream1Printf("};\n\n");
iStart = -1;
}
i = iNonNull;
}
else
{
if (iStart < 0)
{
Stream1Printf("static const uint8_t g_afRTUniFlags0x%06x[] = \n"
"{\n", i);
iStart = i;
}
i++;
}
}
Stream2Printf(" { ~(RTUNICP)0, ~(RTUNICP)0, NULL }\n"
"};\n\n\n");
Stream1Printf("\n");
return Stream2Flush();
}
/**
* Prints the upper case tables.
*/
static int PrintUpper(void)
{
Stream2Init();
Stream2Printf("const RTUNICASERANGE g_aRTUniUpperRanges[] =\n"
"{\n");
RTUNICP i = 0;
int iStart = -1;
while (i < RT_ELEMENTS(g_aCPInfo))
{
/* figure how far off the next chunk is */
unsigned iSameCase = i;
&& iSameCase >= 256)
iSameCase++;
{
if (iStart >= 0)
{
Stream1Printf("};\n\n");
iStart = -1;
}
i = iSameCase;
}
else
{
if (iStart < 0)
{
Stream1Printf("static const RTUNICP g_afRTUniUpper0x%06x[] = \n"
"{\n", i);
iStart = i;
}
Stream1Printf(" 0x%02x, /* U+%06x: %s*/\n", g_aCPInfo[i].SimpleUpperCaseMapping, g_aCPInfo[i].CodePoint, g_aCPInfo[i].pszName);
i++;
}
}
Stream2Printf(" { ~(RTUNICP)0, ~(RTUNICP)0, NULL }\n"
"};\n\n\n");
Stream1Printf("\n");
return Stream2Flush();
}
/**
* Prints the lowercase tables.
*/
static int PrintLower(void)
{
Stream2Init();
Stream2Printf("const RTUNICASERANGE g_aRTUniLowerRanges[] =\n"
"{\n");
RTUNICP i = 0;
int iStart = -1;
while (i < RT_ELEMENTS(g_aCPInfo))
{
/* figure how far off the next chunk is */
unsigned iSameCase = i;
&& iSameCase >= 256)
iSameCase++;
{
if (iStart >= 0)
{
Stream1Printf("};\n\n");
iStart = -1;
}
i = iSameCase;
}
else
{
if (iStart < 0)
{
Stream1Printf("static const RTUNICP g_afRTUniLower0x%06x[] = \n"
"{\n", i);
iStart = i;
}
Stream1Printf(" 0x%02x, /* U+%06x: %s*/\n",
i++;
}
}
Stream2Printf(" { ~(RTUNICP)0, ~(RTUNICP)0, NULL }\n"
"};\n\n\n");
Stream1Printf("\n");
return Stream2Flush();
}
{
/*
* Parse args.
*/
if (argc <= 1)
{
printf("usage: %s [-C|--dir <UCD-dir>] [UnicodeData.txt [DerivedCoreProperties.txt [PropList.txt] [DerivedNormalizationProps.txt]]]\n",
argv[0]);
return 1;
}
const char *pszBaseDir = NULL;
const char *pszUnicodeData = "UnicodeData.txt";
const char *pszDerivedCoreProperties = "DerivedCoreProperties.txt";
const char *pszPropList = "PropList.txt";
const char *pszDerivedNormalizationProps = "DerivedNormalizationProps.txt";
int iFile = 0;
{
{
switch (iFile++)
{
default:
return 1;
}
}
{
{
return 1;
}
argi++;
}
g_fQuiet = true;
else
{
return 1;
}
}
/*
* Read the data.
*/
if (rc)
return rc;
rc = GenerateExcludedData();
if (rc)
return rc;
if (rc)
return rc;
if (rc)
return rc;
if (rc)
return rc;
/*
* Print stuff.
*/
if (rc)
return rc;
rc = PrintFlags();
if (rc)
return rc;
rc = PrintUpper();
if (rc)
return rc;
rc = PrintLower();
if (rc)
return rc;
/* done */
return rc;
}