gered/picoc
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

tokenize

Browse source
This commit is contained in:
Joseph Poirier 2015-06-19 05:06:51 -05:00
parent 9316233d16
commit 57190f3366
1 changed files with 5 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
lex.c
View file

@ -38,7 +38,7 @@ static void LexSkipLineCont(struct LexState *Lexer, char NextChar);
static enum LexToken LexScanGetToken(Picoc *pc, struct LexState *Lexer, static enum LexToken LexScanGetToken(Picoc *pc, struct LexState *Lexer,
struct Value **Value); struct Value **Value);
static int LexTokenSize(enum LexToken Token); static int LexTokenSize(enum LexToken Token);
static void *LexTokenise(Picoc *pc, struct LexState *Lexer, int *TokenLen); static void *LexTokenize(Picoc *pc, struct LexState *Lexer, int *TokenLen);
static enum LexToken LexGetRawToken(struct ParseState *Parser, struct Value **Value, static enum LexToken LexGetRawToken(struct ParseState *Parser, struct Value **Value,
int IncPos); int IncPos);
static void LexHashIncPos(struct ParseState *Parser, int IncPos); static void LexHashIncPos(struct ParseState *Parser, int IncPos);
@ -622,7 +622,7 @@ int LexTokenSize(enum LexToken Token)
/* produce tokens from the lexer and return a heap buffer with /* produce tokens from the lexer and return a heap buffer with
the result - used for scanning */ the result - used for scanning */
void *LexTokenise(Picoc *pc, struct LexState *Lexer, int *TokenLen) void *LexTokenize(Picoc *pc, struct LexState *Lexer, int *TokenLen)
{ {
int MemUsed = 0; int MemUsed = 0;
int ValueSize; int ValueSize;
@ -635,7 +635,7 @@ void *LexTokenise(Picoc *pc, struct LexState *Lexer, int *TokenLen)
char *TokenPos = (char*)TokenSpace; char *TokenPos = (char*)TokenSpace;
if (TokenSpace == NULL) if (TokenSpace == NULL)
LexFail(pc, Lexer, "(LexTokenise TokenSpace == NULL) out of memory"); LexFail(pc, Lexer, "(LexTokenize TokenSpace == NULL) out of memory");
do { do {
/* store the token at the end of the stack area */ /* store the token at the end of the stack area */
@ -666,7 +666,7 @@ void *LexTokenise(Picoc *pc, struct LexState *Lexer, int *TokenLen)
HeapMem = HeapAllocMem(pc, MemUsed); HeapMem = HeapAllocMem(pc, MemUsed);
if (HeapMem == NULL) if (HeapMem == NULL)
LexFail(pc, Lexer, "(LexTokenise HeapMem == NULL) out of memory"); LexFail(pc, Lexer, "(LexTokenize HeapMem == NULL) out of memory");
assert(ReserveSpace >= MemUsed); assert(ReserveSpace >= MemUsed);
memcpy(HeapMem, TokenSpace, MemUsed); memcpy(HeapMem, TokenSpace, MemUsed);
@ -701,7 +701,7 @@ void *LexAnalyse(Picoc *pc, const char *FileName, const char *Source,
Lexer.CharacterPos = 1; Lexer.CharacterPos = 1;
Lexer.SourceText = Source; Lexer.SourceText = Source;
return LexTokenise(pc, &Lexer, TokenLen); return LexTokenize(pc, &Lexer, TokenLen);
} }
/* prepare to parse a pre-tokenised buffer */ /* prepare to parse a pre-tokenised buffer */