使用者:Antigng/AF/AFTokenizer

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "AFTokenizer.h"
#include "AFParser.h"
#include "mem.h"
#include "struct.h"

const unsigned char hex_decoding[]=
{
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	1, 2, 3, 4, 5, 6, 7, 8, 9,10, 0, 0, 0, 0, 0, 0,
	0,11,12,13,14,15,16, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0,11,12,13,14,15,16, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
#define isValidForHex(ch) (hex_decoding[(unsigned char)(ch)])
const unsigned char idvalid[]=
{
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
	0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,
	0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
#define isValidForID(ch) (idvalid[(unsigned char)(ch)])
const unsigned char spacevalid[]=
{
	0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
#define isSpace(ch) (spacevalid[(unsigned char)(ch)])
#define forwardRead(s,c,i,o) (c=s[++i]+o)

static void throwError(int count,unsigned char symbol,const char *reason)
{
	char symText[16];
	if(!symbol)
	{
		sprintf(symText,"EOL");
	}
	else if((symbol>=32)&&(symbol<=126))
	{
		sprintf(symText,"'%c'",symbol);
	}
	else
	{
		sprintf(symText,"'\\u%02x'",symbol);
	}
	fprintf(stderr,"Lexer: Error at char number %d: %s - %s.\n",count,symText,reason);
	return;
}

static char *text=NULL;
static int top=0;
static int numLexer(const char *source,int start,struct _AFToken *tok_p)
{
	unsigned char ch;
	int count=0;
	int type=0;
	int ivalue;
	double fvalue,mul;
	switch(start)
	{
	case 0:
		ivalue=0;
		goto _DFAState_2;
		break;
	case 1:
		goto _DFAState_4;
		break;
	case 2:
		ivalue=source[0]-'0';
		break;
	default:
		goto _DFAFailure;
	}
_DFAState_1:
	forwardRead((unsigned char)source,ch,count,(-'0'));
	if(ch<10)
	{
		ivalue=ivalue*10+ch;
		goto _DFAState_1;
	}
	else if(ch!=(unsigned char)('.'-'0'))
	{
		goto _DFAWrapup;
	}
_DFAState_2:
	forwardRead((unsigned char)source,ch,count,(-'0'));
	if(ch<10)
	{
		type=1;
		fvalue=(double)ivalue+(double)ch*0.1;
		mul=0.1;
	}
	else
	{
		goto _DFAFailure;
	}
_DFAState_3:
	forwardRead((unsigned char)source,ch,count,(-'0'));
	if(ch<10)
	{
		mul=mul*0.1;
		fvalue+=mul*(double)ch;
		goto _DFAState_3;
	}
	else
	{
		goto _DFAWrapup;
	}
_DFAState_4:
	forwardRead(source,ch,count,0);
	switch(ch)
	{
	case '.':
		ivalue=0;
		goto _DFAState_2;
		break;
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
		ivalue=ch-'0';
		goto _DFAState_1;
		break;
	case 'b':
		ivalue=0;
		goto _DFAState_7;
		break;
	case 'o':
		ivalue=0;
		break;
	case 'x':
		ivalue=0;
		goto _DFAState_6;
		break;
	default:
		ivalue=0;
		goto _DFAWrapup;
	}
_DFAState_5:
	forwardRead((unsigned char)source,ch,count,(-'0'));
	if(ch<8)
	{
		ivalue=ivalue*8+ch;
		goto _DFAState_5;
	}
	else
	{
		goto _DFAWrapup;
	}
_DFAState_6:
	forwardRead(source,ch,count,0);
	if((ch=isValidForHex(ch))>0)
	{
		ivalue=ivalue*16+ch-1;
		goto _DFAState_6;
	}
	else
	{
		goto _DFAWrapup;
	}
_DFAState_7:
	forwardRead((unsigned char)source,ch,count,(-'0'));
	if(ch<2)
	{
		ivalue=ivalue*2+ch;
		goto _DFAState_7;
	}
	else
	{
		goto _DFAWrapup;
	}
_DFAWrapup:
	switch(source[count])
	{
	case 0:
	case '\t':
	case '\n':
	case '\v':
	case '\f':
	case '\r':
	case ' ':
	case '!':
	case '%':
	case '&':
	case '(':
	case ')':
	case '*':
	case '+':
	case ',':
	case '-':
	case '/':
	case ':':
	case ';':
	case '<':
	case '=':
	case '>':
	case '?':
	case '[':
	case ']':
	case '^':
	case '|':
		if(type)
		{
			tok_p->type=T_FLOAT;
			tok_p->value.f=fvalue;
		}
		else
		{
			tok_p->type=T_INT;
			tok_p->value.i=ivalue;
		}
		return count;
		break;
	}
_DFAFailure:
	return -1;
}
static int idLexer(const char *source,unsigned int *hash_p)
{
	int count=0;
	unsigned int hash=0;
	char ch=source[0];
	do
	{
		str_update_hash(hash,ch);
		text[top++]=ch;
		count++;
	}while(isValidForID(ch=source[count]));
	text[top++]=0;
	*hash_p=hash;
	return count;
}
static int stringLexer(const char *source,char start_ch)
{
	int count=-1;
	char num=0;
	char ch;
_DFAState_0:
	forwardRead(source,ch,count,0);
	if(!ch)
	{
		return -1;
	}
	else if(ch=='\\')
	{
	}
	else if(ch==start_ch)
	{
		text[top++]=0;
		return count;
	}
	else
	{
		text[top++]=ch;
		goto _DFAState_0;
	}
	forwardRead(source,ch,count,0);
	switch(ch)
	{
	case 0:
		return -1;
		break;
	case '\\':
	case '\'':
	case '\"':
		text[top++]=ch;
		goto _DFAState_0;
		break;
	case 'n':
		text[top++]='\n';
		goto _DFAState_0;
		break;
	case 'r':
		text[top++]='\r';
		goto _DFAState_0;
		break;
	case 't':
		text[top++]='\t';
		goto _DFAState_0;
		break;
	case 'x':
		text[top]='\\';
		text[top+1]='x';
		break;
	default:
		text[top]='\\';
		text[top+1]=ch;
		top+=2;
		goto _DFAState_0;
		break;
	}
	forwardRead(source,ch,count,0);
	if(!ch)
	{
		return -1;
	}
	text[top+2]=ch;
	if((ch=isValidForHex(ch))!=0)
	{
		num=ch-1;
	}
	else
	{
		top+=3;
		goto _DFAState_0;
	}
	forwardRead(source,ch,count,0);
	if(!ch)
	{
		return -1;
	}
	if((ch=isValidForHex(ch))>0)
	{
		text[top++]=num*16+ch-1;			
	}
	else
	{
		text[top+3]=ch;
		top+=4;
	}
	goto _DFAState_0;
}
static int skipOverSpaces(const char *source,char *x)
{
	int count=1;
	while(isSpace(source[count]))
	{
		count++;
	}
	*x=source[count];
	return count;
}
static int skipOverComments(const char *source)
{
	int count=0;
_DFAState_0:
	switch(source[count])
	{
	case 0:
		return -1;
		break;
	case '*':
		break;
	default:
		count++;
		goto _DFAState_0;
	}
	count++;
	switch(source[count])
	{
	case 0:
		return -1;
		break;
	case '/':
		return count+1;
		break;
	default:
		count++;
		goto _DFAState_0;
	}
}
static struct hashlist *AFKeywords=NULL;
static enum _AFKeywordType AFKeywordType[]=
{
	K_like,
	K_in,
	K_contains,
	K_rlike,
	K_irlike,
	K_if,
	K_then,
	K_else,
	K_end,
	K_true,
	K_false,
	K_null
};

/* Get next token*/
static const char *AFSource=NULL;
static int AFCount=0;
static char AFCur=0;
#define getIdWithKeywords() \
{\
	enum _AFKeywordType *key;\
	unsigned int hash;\
	int basecount=top;\
	int len=idLexer(AFSource+AFCount,&hash);\
	AFCount+=len-1;\
	if(str_hashquery_withhash(hash,AFKeywords,text+basecount,(void **)&key))\
	{\
		tok_p->type=T_KEYWORD;\
		tok_p->value.key=*key;\
	}\
	else\
	{\
		tok_p->type=T_ID;\
		tok_p->value.s.begin=text+basecount;\
		tok_p->value.s.offset=len;\
		tok_p->value.s.hash=hash;\
	}\
}
#define getIdWithoutKeywords() \
{\
	unsigned int hash;\
	int basecount=top;\
	int len=idLexer(AFSource+AFCount,&hash);\
	AFCount+=len-1;\
	tok_p->type=T_ID;\
	tok_p->value.s.begin=text+basecount;\
	tok_p->value.s.offset=len;\
	tok_p->value.s.hash=hash;\
}
#define setTokenWithOP(OP) \
{\
	tok_p->type=T_OP;\
	tok_p->value.op=OP;\
}
#define setTokenWithString() \
{\
	AFCount++;\
	{\
		int basetop=top;\
		int len=stringLexer(AFSource+AFCount,AFCur);\
		if(len>=0)\
		{\
			AFCount+=len;\
			tok_p->type=T_STRING;\
			tok_p->value.s.begin=text+basetop;\
			tok_p->value.s.offset=top-basetop-1;\
		}\
		else\
		{\
			throwError(AFCount,AFCur,"unclosed string");\
			return 1;\
		}\
	}\
}
#define setTokenWithNum(start) \
{\
	int len=numLexer(AFSource+AFCount,start,tok_p);\
	if(len>0)\
	{\
		AFCount+=len-1;\
	}\
	else\
	{\
		throwError(AFCount,AFCur,"invalid character in number");\
		return 1;\
	}\
}
int AFGetNextToken(struct _AFToken *tok_p)
{
_Tokenizer_Start:
	switch(AFCur)
	{
		/* EOF here */
	case 0:
		tok_p->type=T_NONE;
		return 0;
		break;

		/* spaces here */
	case '\t':
	case '\n':
	case '\v':
	case '\f':
	case '\r':
	case ' ':
		AFCount+=skipOverSpaces(AFSource+AFCount,&AFCur);
		goto _Tokenizer_Start;
		break;

		/* separators & operators here */
	case '!':
		forwardRead(AFSource,AFCur,AFCount,0);
		if(AFCur=='=')
		{
			forwardRead(AFSource,AFCur,AFCount,0);
			if(AFCur=='=')
			{
				setTokenWithOP(O_SINEQ);
			}
			else
			{
				setTokenWithOP(O_INEQ);
				return 0;
			}
		}
		else
		{
			setTokenWithOP(O_NOT);
			return 0;
		}
		break;
	case '%':
		setTokenWithOP(O_REM);
		break;
	case '&':
		setTokenWithOP(O_AND);
		break;
	case '(':
		tok_p->type=T_BRA;
		break;
	case ')':
		tok_p->type=T_KET;
		break;
	case '*':
		forwardRead(AFSource,AFCur,AFCount,0);
		if(AFCur=='*')
		{
			setTokenWithOP(O_EXP);
		}
		else
		{
			setTokenWithOP(O_MUL);
			return 0;
		}
		break;
	case '+':
		setTokenWithOP(O_PLUS);
		break;
	case ',':
		tok_p->type=T_COMMA;
		break;
	case '-':
		setTokenWithOP(O_MINUS);
		break;
	case '/':
		forwardRead(AFSource,AFCur,AFCount,0);
		if(AFCur=='*')
		{
			int len;
			len=skipOverComments(AFSource+AFCount);
			if(len>=0)
			{
				AFCount+=len;
				AFCur=AFSource[AFCount];
				goto _Tokenizer_Start;
			}
			else
			{
				throwError(AFCount,AFCur,"unclosed comments");
				return 1;
			}
		}
		else
		{
			setTokenWithOP(O_DIV);
		}
		return 0;
		break;
	case ':':
		forwardRead(AFSource,AFCur,AFCount,0);
		if(AFCur=='=')
		{
			setTokenWithOP(O_SET);
		}
		else
		{
			setTokenWithOP(O_TER_S);
			return 0;
		}
		break;
	case ';':
		tok_p->type=T_STATEMENT_SEPARATOR;
		break;
	case '<':
		forwardRead(AFSource,AFCur,AFCount,0);
		if(AFCur=='=')
		{
			setTokenWithOP(O_LE);
		}
		else
		{
			setTokenWithOP(O_L);
			return 0;
		}
		break;
	case '=':
		forwardRead(AFSource,AFCur,AFCount,0);
		if(AFCur=='=')
		{
			forwardRead(AFSource,AFCur,AFCount,0);
			if(AFCur=='=')
			{
				setTokenWithOP(O_SEQ);
			}
			else
			{
				setTokenWithOP(O_EQ);
				return 0;
			}
		}
		else
		{
			setTokenWithOP(O_EQ);
			return 0;
		}
		break;
	case '>':
		forwardRead(AFSource,AFCur,AFCount,0);
		if(AFCur=='=')
		{
			setTokenWithOP(O_GE);
		}
		else
		{
			setTokenWithOP(O_G);
			return 0;
		}
		break;
	case '?':
		setTokenWithOP(O_TER_Q);
		break;
	case '[':
		tok_p->type=T_SQUARE_BRA;
		break;
	case ']':
		tok_p->type=T_SQUARE_KET;
		break;
	case '^':
		setTokenWithOP(O_XOR);
		break;
	case '|':
		setTokenWithOP(O_OR);
		break;

		/* strings here */
	case '\"':
	case '\'':
		setTokenWithString();
		break;

		/* numbers here */
	case '.':
		setTokenWithNum(0);
		break;
	case '0':
		setTokenWithNum(1);
		break;
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
		setTokenWithNum(2);
		break;

		/* ids & tokens here */
	case 'A':
	case 'B':
	case 'C':
	case 'D':
	case 'E':
	case 'F':
	case 'G':
	case 'H':
	case 'I':
	case 'J':
	case 'K':
	case 'L':
	case 'M':
	case 'N':
	case 'O':
	case 'P':
	case 'Q':
	case 'R':
	case 'S':
	case 'T':
	case 'U':
	case 'V':
	case 'W':
	case 'X':
	case 'Y':
	case 'Z':
	case '_':
	case 'a':
	case 'b':
		getIdWithoutKeywords();
		break;
	case 'c':
	case 'd':
	case 'e':
	case 'f':
	case 'g':
	case 'h':
	case 'i':
	case 'j':
	case 'k':
	case 'l':
	case 'm':
	case 'n':
	case 'o':
	case 'p':
	case 'q':
	case 'r':
	case 's':
	case 't':
	case 'u':
	case 'v':
	case 'w':
	case 'x':
	case 'y':
	case 'z':
		getIdWithKeywords();
		break;
	default:
		throwError(AFCount,AFCur,"invalid character");
		return 1;
		break;
	}
	forwardRead(AFSource,AFCur,AFCount,0);
	return 0;
}
void AFTokenizerIni()
{
	AFKeywords=hashini();
	str_hashadd(AFKeywords,"like",AFKeywordType+K_like);
	str_hashadd(AFKeywords,"matches",AFKeywordType+K_like);
	str_hashadd(AFKeywords,"in",AFKeywordType+K_in);
	str_hashadd(AFKeywords,"contains",AFKeywordType+K_contains);
	str_hashadd(AFKeywords,"rlike",AFKeywordType+K_rlike);
	str_hashadd(AFKeywords,"regex",AFKeywordType+K_rlike);
	str_hashadd(AFKeywords,"irlike",AFKeywordType+K_irlike);
	str_hashadd(AFKeywords,"if",AFKeywordType+K_if);
	str_hashadd(AFKeywords,"then",AFKeywordType+K_then);
	str_hashadd(AFKeywords,"else",AFKeywordType+K_else);
	str_hashadd(AFKeywords,"end",AFKeywordType+K_end);
	str_hashadd(AFKeywords,"true",AFKeywordType+K_true);
	str_hashadd(AFKeywords,"false",AFKeywordType+K_false);
	str_hashadd(AFKeywords,"null",AFKeywordType+K_null);
	return;
}
void AFTokenizerReset(const char *source,unsigned int source_len)
{
	static unsigned int text_limit=0;
	/* Reset text. Text cannot be longer than the source code. */
	if(text_limit<source_len)
	{
		s_free(text);
		text=(char *)s_malloc(sizeof(char)*source_len);
		text_limit=source_len;
	}
	top=0;
	/* Reset the pointer to the source code. */
	AFSource=source;
	AFCount=0;
	AFCur=*source;
	return ;
}