#pragma ident "%Z%%M% %I% %E% SMI"
/*
** 2001 September 15
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** An tokenizer for SQL
**
** This file contains C code that splits an SQL input string up into
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
**
** $Id: tokenize.c,v 1.68 2004/02/14 23:59:58 drh Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <ctype.h>
#include <stdlib.h>
/*
** All the keywords of the SQL language are stored as in a hash
** table composed of instances of the following structure.
*/
struct Keyword {
};
/*
** These are the keywords
*/
{ "ABORT", TK_ABORT, },
{ "AFTER", TK_AFTER, },
{ "ALL", TK_ALL, },
{ "AND", TK_AND, },
{ "AS", TK_AS, },
{ "ASC", TK_ASC, },
{ "ATTACH", TK_ATTACH, },
{ "BEFORE", TK_BEFORE, },
{ "BEGIN", TK_BEGIN, },
{ "BETWEEN", TK_BETWEEN, },
{ "BY", TK_BY, },
{ "CASCADE", TK_CASCADE, },
{ "CASE", TK_CASE, },
{ "CHECK", TK_CHECK, },
{ "CLUSTER", TK_CLUSTER, },
{ "COLLATE", TK_COLLATE, },
{ "COMMIT", TK_COMMIT, },
{ "CONFLICT", TK_CONFLICT, },
{ "CONSTRAINT", TK_CONSTRAINT, },
{ "COPY", TK_COPY, },
{ "CREATE", TK_CREATE, },
{ "CROSS", TK_JOIN_KW, },
{ "DATABASE", TK_DATABASE, },
{ "DEFAULT", TK_DEFAULT, },
{ "DEFERRED", TK_DEFERRED, },
{ "DEFERRABLE", TK_DEFERRABLE, },
{ "DELETE", TK_DELETE, },
{ "DELIMITERS", TK_DELIMITERS, },
{ "DESC", TK_DESC, },
{ "DETACH", TK_DETACH, },
{ "DISTINCT", TK_DISTINCT, },
{ "DROP", TK_DROP, },
{ "END", TK_END, },
{ "EACH", TK_EACH, },
{ "ELSE", TK_ELSE, },
{ "EXCEPT", TK_EXCEPT, },
{ "EXPLAIN", TK_EXPLAIN, },
{ "FAIL", TK_FAIL, },
{ "FOR", TK_FOR, },
{ "FOREIGN", TK_FOREIGN, },
{ "FROM", TK_FROM, },
{ "FULL", TK_JOIN_KW, },
{ "GLOB", TK_GLOB, },
{ "GROUP", TK_GROUP, },
{ "HAVING", TK_HAVING, },
{ "IGNORE", TK_IGNORE, },
{ "IMMEDIATE", TK_IMMEDIATE, },
{ "IN", TK_IN, },
{ "INDEX", TK_INDEX, },
{ "INITIALLY", TK_INITIALLY, },
{ "INNER", TK_JOIN_KW, },
{ "INSERT", TK_INSERT, },
{ "INSTEAD", TK_INSTEAD, },
{ "INTERSECT", TK_INTERSECT, },
{ "INTO", TK_INTO, },
{ "IS", TK_IS, },
{ "ISNULL", TK_ISNULL, },
{ "JOIN", TK_JOIN, },
{ "KEY", TK_KEY, },
{ "LEFT", TK_JOIN_KW, },
{ "LIKE", TK_LIKE, },
{ "LIMIT", TK_LIMIT, },
{ "MATCH", TK_MATCH, },
{ "NATURAL", TK_JOIN_KW, },
{ "NOT", TK_NOT, },
{ "NOTNULL", TK_NOTNULL, },
{ "NULL", TK_NULL, },
{ "OF", TK_OF, },
{ "OFFSET", TK_OFFSET, },
{ "ON", TK_ON, },
{ "OR", TK_OR, },
{ "ORDER", TK_ORDER, },
{ "OUTER", TK_JOIN_KW, },
{ "PRAGMA", TK_PRAGMA, },
{ "PRIMARY", TK_PRIMARY, },
{ "RAISE", TK_RAISE, },
{ "REFERENCES", TK_REFERENCES, },
{ "REPLACE", TK_REPLACE, },
{ "RESTRICT", TK_RESTRICT, },
{ "RIGHT", TK_JOIN_KW, },
{ "ROLLBACK", TK_ROLLBACK, },
{ "ROW", TK_ROW, },
{ "SELECT", TK_SELECT, },
{ "SET", TK_SET, },
{ "STATEMENT", TK_STATEMENT, },
{ "TABLE", TK_TABLE, },
{ "TEMP", TK_TEMP, },
{ "TEMPORARY", TK_TEMP, },
{ "THEN", TK_THEN, },
{ "TRANSACTION", TK_TRANSACTION, },
{ "TRIGGER", TK_TRIGGER, },
{ "UNION", TK_UNION, },
{ "UNIQUE", TK_UNIQUE, },
{ "UPDATE", TK_UPDATE, },
{ "USING", TK_USING, },
{ "VACUUM", TK_VACUUM, },
{ "VALUES", TK_VALUES, },
{ "VIEW", TK_VIEW, },
{ "WHEN", TK_WHEN, },
{ "WHERE", TK_WHERE, },
};
/*
** This is the hash table
*/
/*
** This function looks up an identifier to determine if it is a
** keyword. If it is a keyword, the token code of that keyword is
** returned. If the input is not a keyword, TK_ID is returned.
*/
int sqliteKeywordCode(const char *z, int n){
int h, i;
Keyword *p;
if( needInit ){
/* Initialize the keyword hash table */
if( needInit ){
int nk;
for(i=0; i<nk; i++){
h %= KEY_HASH_SIZE;
aiHashTable[h] = i+1;
}
needInit = 0;
}
}
h = sqliteHashNoCase(z, n) % KEY_HASH_SIZE;
for(i=aiHashTable[h]; i; i=p->iNext){
p = &aKeywordTable[i-1];
return p->tokenType;
}
}
return TK_ID;
}
/*
** If X is a character that can be used in an identifier and
** X&0x80==0 then isIdChar[X] will be 1. If X&0x80==0x80 then
** X is always an identifier character. (Hence all UTF-8
** characters can be part of an identifier). isIdChar[X] will
** be 0 for every character in the lower 128 ASCII characters
** that cannot be used as part of an identifier.
**
** In this implementation, an identifier can be a string of
** alphabetic characters, digits, and "_" plus any character
** with the high-order bit set. The latter rule means that
** any sequence of UTF-8 characters or characters taken from
** an extended ISO8859 character set can form an identifier.
*/
static const char isIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
};
/*
** Return the length of the token that begins at z[0].
** Store the token type in *tokenType before returning.
*/
int i;
switch( *z ){
case ' ': case '\t': case '\n': case '\f': case '\r': {
for(i=1; isspace(z[i]); i++){}
return i;
}
case '-': {
if( z[1]=='-' ){
for(i=2; z[i] && z[i]!='\n'; i++){}
*tokenType = TK_COMMENT;
return i;
}
return 1;
}
case '(': {
return 1;
}
case ')': {
return 1;
}
case ';': {
return 1;
}
case '+': {
return 1;
}
case '*': {
return 1;
}
case '/': {
if( z[1]!='*' || z[2]==0 ){
return 1;
}
for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
if( z[i] ) i++;
*tokenType = TK_COMMENT;
return i;
}
case '%': {
return 1;
}
case '=': {
return 1 + (z[1]=='=');
}
case '<': {
if( z[1]=='=' ){
return 2;
}else if( z[1]=='>' ){
return 2;
}else if( z[1]=='<' ){
return 2;
}else{
return 1;
}
}
case '>': {
if( z[1]=='=' ){
return 2;
}else if( z[1]=='>' ){
return 2;
}else{
return 1;
}
}
case '!': {
if( z[1]!='=' ){
*tokenType = TK_ILLEGAL;
return 2;
}else{
return 2;
}
}
case '|': {
if( z[1]!='|' ){
return 1;
}else{
return 2;
}
}
case ',': {
return 1;
}
case '&': {
return 1;
}
case '~': {
return 1;
}
case '\'': case '"': {
int delim = z[0];
for(i=1; z[i]; i++){
if( z[i]==delim ){
if( z[i+1]==delim ){
i++;
}else{
break;
}
}
}
if( z[i] ) i++;
return i;
}
case '.': {
return 1;
}
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9': {
*tokenType = TK_INTEGER;
for(i=1; isdigit(z[i]); i++){}
i += 2;
while( isdigit(z[i]) ){ i++; }
}
if( (z[i]=='e' || z[i]=='E') &&
( isdigit(z[i+1])
)
){
i += 2;
while( isdigit(z[i]) ){ i++; }
}
return i;
}
case '[': {
for(i=1; z[i] && z[i-1]!=']'; i++){}
return i;
}
case '?': {
*tokenType = TK_VARIABLE;
return 1;
}
default: {
if( (*z&0x80)==0 && !isIdChar[*z] ){
break;
}
*tokenType = sqliteKeywordCode((char*)z, i);
return i;
}
}
*tokenType = TK_ILLEGAL;
return 1;
}
/*
** Run the parser on the given SQL string. The parser structure is
** passed in. An SQLITE_ status code is returned. If an error occurs
** and pzErrMsg!=NULL then an error message might be written into
** memory obtained from malloc() and *pzErrMsg made to point to that
** error message. Or maybe not.
*/
int nErr = 0;
int i;
void *pEngine;
int tokenType;
extern void *sqliteParserAlloc(void*(*)(int));
extern void sqliteParserFree(void*, void(*)(void*));
i = 0;
if( pEngine==0 ){
return 1;
}
while( sqlite_malloc_failed==0 && zSql[i]!=0 ){
assert( i>=0 );
i += pParse->sLastToken.n;
switch( tokenType ){
case TK_SPACE:
case TK_COMMENT: {
goto abort_parse;
}
break;
}
case TK_ILLEGAL: {
nErr++;
goto abort_parse;
}
case TK_SEMI: {
/* Fall thru into the default case */
}
default: {
goto abort_parse;
}
break;
}
}
}
if( lastTokenParsed!=TK_SEMI ){
}
}
(char*)0);
}
}else{
}
}
}
}
if( pParse->pNewTrigger ){
pParse->pNewTrigger = 0;
}
}
return nErr;
}
/*
** Token types used by the sqlite_complete() routine. See the header
** comments on that procedure for additional information.
*/
#define tkEXPLAIN 0
/*
** Return TRUE if the given SQL string ends in a semicolon.
**
** Special handling is require for CREATE TRIGGER statements.
** Whenever the CREATE TRIGGER keywords are seen, the statement
** must end with ";END;".
**
** This implementation uses a state machine with 7 states:
**
** (0) START At the beginning or end of an SQL statement. This routine
** returns 1 if it ends in the START state and 0 if it ends
** in any other state.
**
** (1) EXPLAIN The keyword EXPLAIN has been seen at the beginning of
** a statement.
**
** (2) CREATE The keyword CREATE has been seen at the beginning of a
** TEMP or TEMPORARY
**
** (3) NORMAL We are in the middle of statement which ends with a single
** semicolon.
**
** (4) TRIGGER We are in the middle of a trigger definition that must be
** ended by a semicolon, the keyword END, and another semicolon.
**
** (5) SEMI We've seen the first semicolon in the ";END;" that occurs at
** the end of a trigger definition.
**
** (6) END We've seen the ";END" of the ";END;" that occurs at the end
** of a trigger difinition.
**
** Transitions between states above are determined by tokens extracted
** from the input. The following tokens are significant:
**
** (0) tkEXPLAIN The "explain" keyword.
** (1) tkCREATE The "create" keyword.
** (2) tkTEMP The "temp" or "temporary" keyword.
** (3) tkTRIGGER The "trigger" keyword.
** (4) tkEND The "end" keyword.
** (5) tkSEMI A semicolon.
** (6) tkWS Whitespace
** (7) tkOTHER Any other SQL token.
**
** Whitespace never causes a state transition and is always ignored.
*/
/* The following matrix defines the transition from one state to another
** according to what token is seen. trans[state][token] returns the
** next state.
*/
/* Token: */
/* State: ** EXPLAIN CREATE TEMP TRIGGER END SEMI WS OTHER */
/* 0 START: */ { 1, 2, 3, 3, 3, 0, 0, 3, },
/* 1 EXPLAIN: */ { 3, 2, 3, 3, 3, 0, 1, 3, },
/* 2 CREATE: */ { 3, 3, 2, 4, 3, 0, 2, 3, },
/* 3 NORMAL: */ { 3, 3, 3, 3, 3, 0, 3, 3, },
/* 4 TRIGGER: */ { 4, 4, 4, 4, 4, 5, 4, 4, },
/* 5 SEMI: */ { 4, 4, 4, 4, 6, 5, 5, 4, },
/* 6 END: */ { 4, 4, 4, 4, 4, 0, 6, 4, },
};
while( *zSql ){
switch( *zSql ){
case ';': { /* A semicolon */
break;
}
case ' ':
case '\r':
case '\t':
case '\n':
case '\f': { /* White space is ignored */
break;
}
case '/': { /* C-style comments */
break;
}
zSql += 2;
if( zSql[0]==0 ) return 0;
zSql++;
break;
}
case '-': { /* SQL-style comments from "--" to end of line */
break;
}
break;
}
case '[': { /* Microsoft-style identifiers in [...] */
zSql++;
if( *zSql==0 ) return 0;
break;
}
case '"': /* single- and double-quoted strings */
case '\'': {
int c = *zSql;
zSql++;
if( *zSql==0 ) return 0;
break;
}
default: {
/* Keywords and unquoted identifiers */
int nId;
switch( *zSql ){
case 'c': case 'C': {
}else{
}
break;
}
case 't': case 'T': {
}else{
}
break;
}
case 'e': case 'E': {
}else{
}
break;
}
default: {
break;
}
}
}else{
/* Operators and special symbols */
}
break;
}
}
zSql++;
}
return state==0;
}