gered/picoc
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

More compact storage of lex token values

Browse source 
git-svn-id: http://picoc.googlecode.com/svn/trunk@57 21eae674-98b7-11dd-bd71-f92a316d2d60
This commit is contained in:
zik.saleeba 2009-02-03 00:13:50 +00:00
parent 44117cde04
commit 5b1d4a611f
1 changed files with 34 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
lex.c
View file

@ -11,8 +11,6 @@
#define isCidstart(c) (isalpha(c) || (c)=='_' || (c)=='#') #define isCidstart(c) (isalpha(c) || (c)=='_' || (c)=='#')
#define isCident(c) (isalnum(c) || (c)=='_') #define isCident(c) (isalnum(c) || (c)=='_')
#define ISVALUETOKEN(t) ((t) >= TokenIdentifier && (t) <= TokenCharacterConstant)
#define NEXTIS(c,x,y) { if (NextChar == (c)) { Lexer->Pos++; GotToken = (x); } else GotToken = (y); } #define NEXTIS(c,x,y) { if (NextChar == (c)) { Lexer->Pos++; GotToken = (x); } else GotToken = (y); }
#define NEXTIS3(c,x,d,y,z) { if (NextChar == (c)) { Lexer->Pos++; GotToken = (x); } else NEXTIS(d,y,z) } #define NEXTIS3(c,x,d,y,z) { if (NextChar == (c)) { Lexer->Pos++; GotToken = (x); } else NEXTIS(d,y,z) }
#define NEXTIS4(c,x,d,y,e,z,a) { if (NextChar == (c)) { Lexer->Pos++; GotToken = (x); } else NEXTIS3(d,y,e,z,a) } #define NEXTIS4(c,x,d,y,e,z,a) { if (NextChar == (c)) { Lexer->Pos++; GotToken = (x); } else NEXTIS3(d,y,e,z,a) }
@ -273,6 +271,18 @@ enum LexToken LexScanGetToken(struct LexState *Lexer, struct Value **Value)
return GotToken; return GotToken;
} }
/* what size value goes with each token */
int LexTokenSize(enum LexToken Token)
{
switch (Token)
{
case TokenIdentifier: case TokenStringConstant: return sizeof(char *);
case TokenIntegerConstant: case TokenCharacterConstant: return sizeof(int);
case TokenFPConstant: return sizeof(double);
default: return 0;
}
}
/* produce tokens from the lexer and return a heap buffer with the result - used for scanning */ /* produce tokens from the lexer and return a heap buffer with the result - used for scanning */
void *LexTokenise(struct LexState *Lexer) void *LexTokenise(struct LexState *Lexer)
{ {
@ -281,6 +291,7 @@ void *LexTokenise(struct LexState *Lexer)
struct Value *GotValue; struct Value *GotValue;
int MemAvailable; int MemAvailable;
int MemUsed = 0; int MemUsed = 0;
int ValueSize;
void *TokenSpace = HeapStackGetFreeSpace(&MemAvailable); void *TokenSpace = HeapStackGetFreeSpace(&MemAvailable);
do do
@ -290,13 +301,13 @@ void *LexTokenise(struct LexState *Lexer)
TokenSpace++; TokenSpace++;
MemUsed++; MemUsed++;
if (ISVALUETOKEN(Token)) ValueSize = LexTokenSize(Token);
if (ValueSize > 0)
{ /* store a value as well */ { /* store a value as well */
int ValueSize = sizeof(struct Value) + GotValue->Typ->Sizeof;
if (MemAvailable - MemUsed <= ValueSize) if (MemAvailable - MemUsed <= ValueSize)
LexFail(Lexer, "out of memory while lexing"); LexFail(Lexer, "out of memory while lexing");
memcpy(TokenSpace, GotValue, ValueSize); memcpy(TokenSpace, GotValue->Val, ValueSize);
TokenSpace += ValueSize; TokenSpace += ValueSize;
MemUsed += ValueSize; MemUsed += ValueSize;
} }
@ -338,6 +349,7 @@ void LexInitParser(struct ParseState *Parser, void *TokenSource, const char *Fil
enum LexToken LexGetToken(struct ParseState *Parser, struct Value **Value, int IncPos) enum LexToken LexGetToken(struct ParseState *Parser, struct Value **Value, int IncPos)
{ {
enum LexToken Token; enum LexToken Token;
int ValueSize;
while ((enum LexToken)*(unsigned char *)Parser->Pos == TokenEndOfLine) while ((enum LexToken)*(unsigned char *)Parser->Pos == TokenEndOfLine)
{ /* skip leading newlines */ { /* skip leading newlines */
@ -346,17 +358,28 @@ enum LexToken LexGetToken(struct ParseState *Parser, struct Value **Value, int I
} }
Token = (enum LexToken)*(unsigned char *)Parser->Pos; Token = (enum LexToken)*(unsigned char *)Parser->Pos;
if (ISVALUETOKEN(Token)) ValueSize = LexTokenSize(Token);
{ /* this token requires a value */ if (ValueSize > 0)
int ValueLen = sizeof(struct Value) + ((struct Value *)Parser->Pos)->Typ->Sizeof; { /* this token requires a value - unpack it */
if (Value != NULL) if (Value != NULL)
{ /* copy the value out (aligns it in the process) */ {
memcpy(&LexValue, (struct Value *)Parser->Pos, ValueLen); switch (Token)
{
case TokenStringConstant: case TokenIdentifier: LexValue.Typ = &StringType; break;
case TokenIntegerConstant: LexValue.Typ = &IntType; break;
case TokenCharacterConstant: LexValue.Typ = &CharType; break;
case TokenFPConstant: LexValue.Typ = &FPType; break;
default: break;
}
memcpy(LexValue.Val, Parser->Pos, ValueSize);
LexValue.ValOnHeap = FALSE;
LexValue.ValOnStack = FALSE;
*Value = &LexValue; *Value = &LexValue;
} }
if (IncPos) if (IncPos)
Parser->Pos += ValueLen + 1; Parser->Pos += ValueSize + 1;
} }
else else
{ {