2N/A/* -*- mode: c; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2N/A/*
2N/A * util/support/utf8.c
2N/A *
2N/A * Copyright 2008 by the Massachusetts Institute of Technology.
2N/A * All Rights Reserved.
2N/A *
2N/A * Export of this software from the United States of America may
2N/A * require a specific license from the United States Government.
2N/A * It is the responsibility of any person or organization contemplating
2N/A * export to obtain such a license before exporting.
2N/A *
2N/A * WITHIN THAT CONSTRAINT, permission to use, copy, modify, and
2N/A * distribute this software and its documentation for any purpose and
2N/A * without fee is hereby granted, provided that the above copyright
2N/A * notice appear in all copies and that both that copyright notice and
2N/A * this permission notice appear in supporting documentation, and that
2N/A * the name of M.I.T. not be used in advertising or publicity pertaining
2N/A * to distribution of the software without specific, written prior
2N/A * permission. Furthermore if you modify this software you must label
2N/A * your software as modified software and not distribute it in such a
2N/A * fashion that it might be confused with the original M.I.T. software.
2N/A * M.I.T. makes no representations about the suitability of
2N/A * this software for any purpose. It is provided "as is" without express
2N/A * or implied warranty.
2N/A */
2N/A/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
2N/A *
2N/A * Copyright 1998-2008 The OpenLDAP Foundation.
2N/A * All rights reserved.
2N/A *
2N/A * Redistribution and use in source and binary forms, with or without
2N/A * modification, are permitted only as authorized by the OpenLDAP
2N/A * Public License.
2N/A *
2N/A * A copy of this license is available in the file LICENSE in the
2N/A * top-level directory of the distribution or, alternatively, at
2N/A * <http://www.OpenLDAP.org/license.html>.
2N/A */
2N/A/* Basic UTF-8 routines
2N/A *
2N/A * These routines are "dumb". Though they understand UTF-8,
2N/A * they don't grok Unicode. That is, they can push bits,
2N/A * but don't have a clue what the bits represent. That's
2N/A * good enough for use with the KRB5 Client SDK.
2N/A *
2N/A * These routines are not optimized.
2N/A */
2N/A
2N/A#include "k5-platform.h"
2N/A#include "k5-utf8.h"
2N/A#include "supp-int.h"
2N/A
2N/A/*
2N/A * return the number of bytes required to hold the
2N/A * NULL-terminated UTF-8 string NOT INCLUDING the
2N/A * termination.
2N/A */
2N/Asize_t krb5int_utf8_bytes(const char *p)
2N/A{
2N/A size_t bytes;
2N/A
2N/A for (bytes = 0; p[bytes]; bytes++)
2N/A ;
2N/A
2N/A return bytes;
2N/A}
2N/A
2N/Asize_t krb5int_utf8_chars(const char *p)
2N/A{
2N/A /* could be optimized and could check for invalid sequences */
2N/A size_t chars = 0;
2N/A
2N/A for ( ; *p ; KRB5_UTF8_INCR(p))
2N/A chars++;
2N/A
2N/A return chars;
2N/A}
2N/A
2N/Asize_t krb5int_utf8c_chars(const char *p, size_t length)
2N/A{
2N/A /* could be optimized and could check for invalid sequences */
2N/A size_t chars = 0;
2N/A const char *end = p + length;
2N/A
2N/A for ( ; p < end; KRB5_UTF8_INCR(p))
2N/A chars++;
2N/A
2N/A return chars;
2N/A}
2N/A
2N/A/* return offset to next character */
2N/Aint krb5int_utf8_offset(const char *p)
2N/A{
2N/A return KRB5_UTF8_NEXT(p) - p;
2N/A}
2N/A
2N/A/*
2N/A * Returns length indicated by first byte.
2N/A */
2N/Aconst char krb5int_utf8_lentab[] = {
2N/A 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2N/A 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2N/A 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2N/A 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2N/A 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2N/A 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2N/A 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
2N/A 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0 };
2N/A
2N/Aint krb5int_utf8_charlen(const char *p)
2N/A{
2N/A if (!(*p & 0x80))
2N/A return 1;
2N/A
2N/A return krb5int_utf8_lentab[*(const unsigned char *)p ^ 0x80];
2N/A}
2N/A
2N/A/*
2N/A * Make sure the UTF-8 char used the shortest possible encoding
2N/A * returns charlen if valid, 0 if not.
2N/A *
2N/A * Here are the valid UTF-8 encodings, taken from RFC 2279 page 4.
2N/A * The table is slightly modified from that of the RFC.
2N/A *
2N/A * UCS-4 range (hex) UTF-8 sequence (binary)
2N/A * 0000 0000-0000 007F 0.......
2N/A * 0000 0080-0000 07FF 110++++. 10......
2N/A * 0000 0800-0000 FFFF 1110++++ 10+..... 10......
2N/A * 0001 0000-001F FFFF 11110+++ 10++.... 10...... 10......
2N/A * 0020 0000-03FF FFFF 111110++ 10+++... 10...... 10...... 10......
2N/A * 0400 0000-7FFF FFFF 1111110+ 10++++.. 10...... 10...... 10...... 10......
2N/A *
2N/A * The '.' bits are "don't cares". When validating a UTF-8 sequence,
2N/A * at least one of the '+' bits must be set, otherwise the character
2N/A * should have been encoded in fewer octets. Note that in the two-octet
2N/A * case, only the first octet needs to be validated, and this is done
2N/A * in the krb5int_utf8_lentab[] above.
2N/A */
2N/A
2N/A/* mask of required bits in second octet */
2N/A#undef c
2N/A#define c const char
2N/Ac krb5int_utf8_mintab[] = {
2N/A (c)0x20, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80,
2N/A (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80,
2N/A (c)0x30, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80, (c)0x80,
2N/A (c)0x38, (c)0x80, (c)0x80, (c)0x80, (c)0x3c, (c)0x80, (c)0x00, (c)0x00 };
2N/A#undef c
2N/A
2N/Aint krb5int_utf8_charlen2(const char *p)
2N/A{
2N/A int i = KRB5_UTF8_CHARLEN(p);
2N/A
2N/A if (i > 2) {
2N/A if (!(krb5int_utf8_mintab[*p & 0x1f] & p[1]))
2N/A i = 0;
2N/A }
2N/A
2N/A return i;
2N/A}
2N/A
2N/A/*
2N/A * Convert a UTF8 character to a UCS4 character. Return 0 on success,
2N/A * -1 on failure.
2N/A */
2N/Aint krb5int_utf8_to_ucs4(const char *p, krb5_ucs4 *out)
2N/A{
2N/A const unsigned char *c = (const unsigned char *) p;
2N/A krb5_ucs4 ch;
2N/A int len, i;
2N/A static unsigned char mask[] = {
2N/A 0, 0x7f, 0x1f, 0x0f, 0x07, 0x03, 0x01 };
2N/A
2N/A *out = 0;
2N/A len = KRB5_UTF8_CHARLEN2(p, len);
2N/A
2N/A if (len == 0)
2N/A return -1;
2N/A
2N/A ch = c[0] & mask[len];
2N/A
2N/A for (i = 1; i < len; i++) {
2N/A if ((c[i] & 0xc0) != 0x80)
2N/A return -1;
2N/A
2N/A ch <<= 6;
2N/A ch |= c[i] & 0x3f;
2N/A }
2N/A
2N/A *out = ch;
2N/A return 0;
2N/A}
2N/A
2N/Aint krb5int_utf8_to_ucs2(const char *p, krb5_ucs2 *out)
2N/A{
2N/A krb5_ucs4 ch;
2N/A
2N/A *out = 0;
2N/A if (krb5int_utf8_to_ucs4(p, &ch) == -1 || ch > 0xFFFF)
2N/A return -1;
2N/A *out = (krb5_ucs2) ch;
2N/A return 0;
2N/A}
2N/A
2N/A/* conv UCS-2 to UTF-8, not used */
2N/Asize_t krb5int_ucs4_to_utf8(krb5_ucs4 c, char *buf)
2N/A{
2N/A size_t len = 0;
2N/A unsigned char *p = (unsigned char *) buf;
2N/A
2N/A /* not a valid Unicode character */
2N/A if (c < 0)
2N/A return 0;
2N/A
2N/A /* Just return length, don't convert */
2N/A if (buf == NULL) {
2N/A if (c < 0x80) return 1;
2N/A else if (c < 0x800) return 2;
2N/A else if (c < 0x10000) return 3;
2N/A else if (c < 0x200000) return 4;
2N/A else if (c < 0x4000000) return 5;
2N/A else return 6;
2N/A }
2N/A
2N/A if (c < 0x80) {
2N/A p[len++] = c;
2N/A } else if (c < 0x800) {
2N/A p[len++] = 0xc0 | ( c >> 6 );
2N/A p[len++] = 0x80 | ( c & 0x3f );
2N/A } else if (c < 0x10000) {
2N/A p[len++] = 0xe0 | ( c >> 12 );
2N/A p[len++] = 0x80 | ( (c >> 6) & 0x3f );
2N/A p[len++] = 0x80 | ( c & 0x3f );
2N/A } else if (c < 0x200000) {
2N/A p[len++] = 0xf0 | ( c >> 18 );
2N/A p[len++] = 0x80 | ( (c >> 12) & 0x3f );
2N/A p[len++] = 0x80 | ( (c >> 6) & 0x3f );
2N/A p[len++] = 0x80 | ( c & 0x3f );
2N/A } else if (c < 0x4000000) {
2N/A p[len++] = 0xf8 | ( c >> 24 );
2N/A p[len++] = 0x80 | ( (c >> 18) & 0x3f );
2N/A p[len++] = 0x80 | ( (c >> 12) & 0x3f );
2N/A p[len++] = 0x80 | ( (c >> 6) & 0x3f );
2N/A p[len++] = 0x80 | ( c & 0x3f );
2N/A } else /* if( c < 0x80000000 ) */ {
2N/A p[len++] = 0xfc | ( c >> 30 );
2N/A p[len++] = 0x80 | ( (c >> 24) & 0x3f );
2N/A p[len++] = 0x80 | ( (c >> 18) & 0x3f );
2N/A p[len++] = 0x80 | ( (c >> 12) & 0x3f );
2N/A p[len++] = 0x80 | ( (c >> 6) & 0x3f );
2N/A p[len++] = 0x80 | ( c & 0x3f );
2N/A }
2N/A
2N/A return len;
2N/A}
2N/A
2N/Asize_t krb5int_ucs2_to_utf8(krb5_ucs2 c, char *buf)
2N/A{
2N/A return krb5int_ucs4_to_utf8((krb5_ucs4)c, buf);
2N/A}
2N/A
2N/A#define KRB5_UCS_UTF8LEN(c) \
2N/A c < 0 ? 0 : (c < 0x80 ? 1 : (c < 0x800 ? 2 : (c < 0x10000 ? 3 : \
2N/A (c < 0x200000 ? 4 : (c < 0x4000000 ? 5 : 6)))))
2N/A
2N/A/*
2N/A * Advance to the next UTF-8 character
2N/A *
2N/A * Ignores length of multibyte character, instead rely on
2N/A * continuation markers to find start of next character.
2N/A * This allows for "resyncing" of when invalid characters
2N/A * are provided provided the start of the next character
2N/A * is appears within the 6 bytes examined.
2N/A */
2N/Achar *krb5int_utf8_next(const char *p)
2N/A{
2N/A int i;
2N/A const unsigned char *u = (const unsigned char *) p;
2N/A
2N/A if (KRB5_UTF8_ISASCII(u)) {
2N/A return (char *) &p[1];
2N/A }
2N/A
2N/A for (i = 1; i < 6; i++) {
2N/A if ((u[i] & 0xc0) != 0x80) {
2N/A return (char *) &p[i];
2N/A }
2N/A }
2N/A
2N/A return (char *) &p[i];
2N/A}
2N/A
2N/A/*
2N/A * Advance to the previous UTF-8 character
2N/A *
2N/A * Ignores length of multibyte character, instead rely on
2N/A * continuation markers to find start of next character.
2N/A * This allows for "resyncing" of when invalid characters
2N/A * are provided provided the start of the next character
2N/A * is appears within the 6 bytes examined.
2N/A */
2N/Achar *krb5int_utf8_prev(const char *p)
2N/A{
2N/A int i;
2N/A const unsigned char *u = (const unsigned char *) p;
2N/A
2N/A for (i = -1; i>-6 ; i--) {
2N/A if ((u[i] & 0xc0 ) != 0x80) {
2N/A return (char *) &p[i];
2N/A }
2N/A }
2N/A
2N/A return (char *) &p[i];
2N/A}
2N/A
2N/A/*
2N/A * Copy one UTF-8 character from src to dst returning
2N/A * number of bytes copied.
2N/A *
2N/A * Ignores length of multibyte character, instead rely on
2N/A * continuation markers to find start of next character.
2N/A * This allows for "resyncing" of when invalid characters
2N/A * are provided provided the start of the next character
2N/A * is appears within the 6 bytes examined.
2N/A */
2N/Aint krb5int_utf8_copy(char* dst, const char *src)
2N/A{
2N/A int i;
2N/A const unsigned char *u = (const unsigned char *) src;
2N/A
2N/A dst[0] = src[0];
2N/A
2N/A if (KRB5_UTF8_ISASCII(u)) {
2N/A return 1;
2N/A }
2N/A
2N/A for (i=1; i<6; i++) {
2N/A if ((u[i] & 0xc0) != 0x80) {
2N/A return i;
2N/A }
2N/A dst[i] = src[i];
2N/A }
2N/A
2N/A return i;
2N/A}
2N/A
2N/A#ifndef UTF8_ALPHA_CTYPE
2N/A/*
2N/A * UTF-8 ctype routines
2N/A * Only deals with characters < 0x80 (ie: US-ASCII)
2N/A */
2N/A
2N/Aint krb5int_utf8_isascii(const char * p)
2N/A{
2N/A unsigned c = * (const unsigned char *) p;
2N/A
2N/A return KRB5_ASCII(c);
2N/A}
2N/A
2N/Aint krb5int_utf8_isdigit(const char * p)
2N/A{
2N/A unsigned c = * (const unsigned char *) p;
2N/A
2N/A if (!KRB5_ASCII(c))
2N/A return 0;
2N/A
2N/A return KRB5_DIGIT( c );
2N/A}
2N/A
2N/Aint krb5int_utf8_isxdigit(const char * p)
2N/A{
2N/A unsigned c = * (const unsigned char *) p;
2N/A
2N/A if (!KRB5_ASCII(c))
2N/A return 0;
2N/A
2N/A return KRB5_HEX(c);
2N/A}
2N/A
2N/Aint krb5int_utf8_isspace(const char * p)
2N/A{
2N/A unsigned c = * (const unsigned char *) p;
2N/A
2N/A if (!KRB5_ASCII(c))
2N/A return 0;
2N/A
2N/A switch(c) {
2N/A case ' ':
2N/A case '\t':
2N/A case '\n':
2N/A case '\r':
2N/A case '\v':
2N/A case '\f':
2N/A return 1;
2N/A }
2N/A
2N/A return 0;
2N/A}
2N/A
2N/A/*
2N/A * These are not needed by the C SDK and are
2N/A * not "good enough" for general use.
2N/A */
2N/Aint krb5int_utf8_isalpha(const char * p)
2N/A{
2N/A unsigned c = * (const unsigned char *) p;
2N/A
2N/A if (!KRB5_ASCII(c))
2N/A return 0;
2N/A
2N/A return KRB5_ALPHA(c);
2N/A}
2N/A
2N/Aint krb5int_utf8_isalnum(const char * p)
2N/A{
2N/A unsigned c = * (const unsigned char *) p;
2N/A
2N/A if (!KRB5_ASCII(c))
2N/A return 0;
2N/A
2N/A return KRB5_ALNUM(c);
2N/A}
2N/A
2N/A#if 0
2N/Aint krb5int_utf8_islower(const char * p)
2N/A{
2N/A unsigned c = * (const unsigned char *) p;
2N/A
2N/A if (!KRB5_ASCII(c))
2N/A return 0;
2N/A
2N/A return KRB5_LOWER(c);
2N/A}
2N/A
2N/Aint krb5int_utf8_isupper(const char * p)
2N/A{
2N/A unsigned c = * (const unsigned char *) p;
2N/A
2N/A if (!KRB5_ASCII(c))
2N/A return 0;
2N/A
2N/A return KRB5_UPPER(c);
2N/A}
2N/A#endif
2N/A#endif
2N/A
2N/A
2N/A/*
2N/A * UTF-8 string routines
2N/A */
2N/A
2N/A/* like strchr() */
2N/Achar *krb5int_utf8_strchr(const char *str, const char *chr)
2N/A{
2N/A krb5_ucs4 chs, ch;
2N/A
2N/A if (krb5int_utf8_to_ucs4(chr, &ch) == -1)
2N/A return NULL;
2N/A for ( ; *str != '\0'; KRB5_UTF8_INCR(str)) {
2N/A if (krb5int_utf8_to_ucs4(str, &chs) == 0 && chs == ch)
2N/A return (char *)str;
2N/A }
2N/A
2N/A return NULL;
2N/A}
2N/A
2N/A/* like strcspn() but returns number of bytes, not characters */
2N/Asize_t krb5int_utf8_strcspn(const char *str, const char *set)
2N/A{
2N/A const char *cstr, *cset;
2N/A krb5_ucs4 chstr, chset;
2N/A
2N/A for (cstr = str; *cstr != '\0'; KRB5_UTF8_INCR(cstr)) {
2N/A for (cset = set; *cset != '\0'; KRB5_UTF8_INCR(cset)) {
2N/A if (krb5int_utf8_to_ucs4(cstr, &chstr) == 0
2N/A && krb5int_utf8_to_ucs4(cset, &chset) == 0 && chstr == chset)
2N/A return cstr - str;
2N/A }
2N/A }
2N/A
2N/A return cstr - str;
2N/A}
2N/A
2N/A/* like strspn() but returns number of bytes, not characters */
2N/Asize_t krb5int_utf8_strspn(const char *str, const char *set)
2N/A{
2N/A const char *cstr, *cset;
2N/A krb5_ucs4 chstr, chset;
2N/A
2N/A for (cstr = str; *cstr != '\0'; KRB5_UTF8_INCR(cstr)) {
2N/A for (cset = set; ; KRB5_UTF8_INCR(cset)) {
2N/A if (*cset == '\0')
2N/A return cstr - str;
2N/A if (krb5int_utf8_to_ucs4(cstr, &chstr) == 0
2N/A && krb5int_utf8_to_ucs4(cset, &chset) == 0 && chstr == chset)
2N/A break;
2N/A }
2N/A }
2N/A
2N/A return cstr - str;
2N/A}
2N/A
2N/A/* like strpbrk(), replaces strchr() as well */
2N/Achar *krb5int_utf8_strpbrk(const char *str, const char *set)
2N/A{
2N/A const char *cset;
2N/A krb5_ucs4 chstr, chset;
2N/A
2N/A for ( ; *str != '\0'; KRB5_UTF8_INCR(str)) {
2N/A for (cset = set; *cset != '\0'; KRB5_UTF8_INCR(cset)) {
2N/A if (krb5int_utf8_to_ucs4(str, &chstr) == 0
2N/A && krb5int_utf8_to_ucs4(cset, &chset) == 0 && chstr == chset)
2N/A return (char *)str;
2N/A }
2N/A }
2N/A
2N/A return NULL;
2N/A}
2N/A
2N/A/* like strtok_r(), not strtok() */
2N/Achar *krb5int_utf8_strtok(char *str, const char *sep, char **last)
2N/A{
2N/A char *begin;
2N/A char *end;
2N/A
2N/A if (last == NULL)
2N/A return NULL;
2N/A
2N/A begin = str ? str : *last;
2N/A
2N/A begin += krb5int_utf8_strspn(begin, sep);
2N/A
2N/A if (*begin == '\0') {
2N/A *last = NULL;
2N/A return NULL;
2N/A }
2N/A
2N/A end = &begin[krb5int_utf8_strcspn(begin, sep)];
2N/A
2N/A if (*end != '\0') {
2N/A char *next = KRB5_UTF8_NEXT(end);
2N/A *end = '\0';
2N/A end = next;
2N/A }
2N/A
2N/A *last = end;
2N/A
2N/A return begin;
2N/A}