gered/picoc
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Now handling multi-line string constants correctly.

Browse source 
Also handling either LF newlines or CR/LF newlines in multi-line string constants correctly.
Changed handling of multi-line comments to use the new, simpler Lexer->EmitExtraNewlines field.
Made FANCY_ERROR_REPORTING standard to reduce code obfuscation.
Fixed issue 98.


git-svn-id: http://picoc.googlecode.com/svn/trunk@473 21eae674-98b7-11dd-bd71-f92a316d2d60
This commit is contained in:
zik.saleeba 2010-07-27 10:35:54 +00:00
parent dc0b17c1a4
commit e45e455b9b
5 changed files with 30 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
lex.c
View file

@ -19,15 +19,9 @@
#define NEXTIS3PLUS(c,x,d,y,e,z,a) { if (NextChar == (c)) { LEXER_INC(Lexer); GotToken = (x); } else if (NextChar == (d)) { if (Lexer->Pos[1] == (e)) { LEXER_INCN(Lexer, 2); GotToken = (z); } else { LEXER_INC(Lexer); GotToken = (y); } } else GotToken = (a); } #define NEXTIS3PLUS(c,x,d,y,e,z,a) { if (NextChar == (c)) { LEXER_INC(Lexer); GotToken = (x); } else if (NextChar == (d)) { if (Lexer->Pos[1] == (e)) { LEXER_INCN(Lexer, 2); GotToken = (z); } else { LEXER_INC(Lexer); GotToken = (y); } } else GotToken = (a); }
#define NEXTISEXACTLY3(c,d,y,z) { if (NextChar == (c) && Lexer->Pos[1] == (d)) { LEXER_INCN(Lexer, 2); GotToken = (y); } else GotToken = (z); } #define NEXTISEXACTLY3(c,d,y,z) { if (NextChar == (c) && Lexer->Pos[1] == (d)) { LEXER_INCN(Lexer, 2); GotToken = (y); } else GotToken = (z); }
#ifdef FANCY_ERROR_REPORTING
#define LEXER_INC(l) ( (l)->Pos++, (l)->CharacterPos++ ) #define LEXER_INC(l) ( (l)->Pos++, (l)->CharacterPos++ )
#define LEXER_INCN(l, n) ( (l)->Pos+=(n), (l)->CharacterPos+=(n) ) #define LEXER_INCN(l, n) ( (l)->Pos+=(n), (l)->CharacterPos+=(n) )
#define TOKEN_DATA_OFFSET 2 #define TOKEN_DATA_OFFSET 2
#else
#define LEXER_INC(l) (l)->Pos++
#define LEXER_INCN(l, n) (l)->Pos+=(n)
#define TOKEN_DATA_OFFSET 1
#endif
#define MAX_CHAR_VALUE 255 /* maximum value which can be represented by a "char" data type */ #define MAX_CHAR_VALUE 255 /* maximum value which can be represented by a "char" data type */
@ -260,8 +254,12 @@ unsigned char LexUnEscapeCharacter(const char **From, const char *End)
unsigned char ThisChar; unsigned char ThisChar;
while ( *From != End && **From == '\\' && while ( *From != End && **From == '\\' &&
&(*From)[1] != End && (*From)[1] == '\n') &(*From)[1] != End && (*From)[1] == '\n' )
(*From) += 2; /* skip escaped end of lines */ (*From) += 2; /* skip escaped end of lines with LF line termination */
while ( *From != End && **From == '\\' &&
&(*From)[1] != End && &(*From)[2] != End && (*From)[1] == '\r' && (*From)[2] == '\n')
(*From) += 3; /* skip escaped end of lines with CR/LF line termination */
if (*From == End) if (*From == End)
return '\\'; return '\\';
@ -310,7 +308,20 @@ enum LexToken LexGetStringConstant(struct LexState *Lexer, struct Value *Value,
{ {
/* find the end */ /* find the end */
if (Escape) if (Escape)
{
if (*Lexer->Pos == '\r' && Lexer->Pos+1 != Lexer->End)
Lexer->Pos++;
if (*Lexer->Pos == '\n' && Lexer->Pos+1 != Lexer->End)
{
Lexer->Line++;
Lexer->Pos++;
Lexer->CharacterPos = 0;
Lexer->EmitExtraNewlines++;
}
Escape = FALSE; Escape = FALSE;
}
else if (*Lexer->Pos == '\\') else if (*Lexer->Pos == '\\')
Escape = TRUE; Escape = TRUE;
@ -368,13 +379,8 @@ void LexSkipComment(struct LexState *Lexer, char NextChar, enum LexToken *Return
while (Lexer->Pos != Lexer->End && (*(Lexer->Pos-1) != '*' || *Lexer->Pos != '/')) while (Lexer->Pos != Lexer->End && (*(Lexer->Pos-1) != '*' || *Lexer->Pos != '/'))
{ {
if (*Lexer->Pos == '\n') if (*Lexer->Pos == '\n')
{ Lexer->EmitExtraNewlines++;
LEXER_INC(Lexer);
Lexer->Mode = LexModeMultiLineComment;
*ReturnToken = TokenEndOfLine;
return;
}
else
LEXER_INC(Lexer); LEXER_INC(Lexer);
} }
@ -398,12 +404,11 @@ enum LexToken LexScanGetToken(struct LexState *Lexer, struct Value **Value)
char NextChar; char NextChar;
enum LexToken GotToken = TokenNone; enum LexToken GotToken = TokenNone;
/* handle the end of multi-line comments */ /* handle cases line multi-line comments or string constants which mess up the line count */
if (Lexer->Mode == LexModeMultiLineComment) if (Lexer->EmitExtraNewlines > 0)
{ {
LexSkipComment(Lexer, '*', &GotToken); Lexer->EmitExtraNewlines--;
if (GotToken != TokenNone) return TokenEndOfLine;
return GotToken;
} }
/* scan for a token */ /* scan for a token */
@ -417,9 +422,7 @@ enum LexToken LexScanGetToken(struct LexState *Lexer, struct Value **Value)
Lexer->Line++; Lexer->Line++;
Lexer->Pos++; Lexer->Pos++;
Lexer->Mode = LexModeNormal; Lexer->Mode = LexModeNormal;
#ifdef FANCY_ERROR_REPORTING
Lexer->CharacterPos = 0; Lexer->CharacterPos = 0;
#endif
return TokenEndOfLine; return TokenEndOfLine;
} }
else if (Lexer->Mode == LexModeHashDefine || Lexer->Mode == LexModeHashDefineSpace) else if (Lexer->Mode == LexModeHashDefine || Lexer->Mode == LexModeHashDefineSpace)
@ -499,16 +502,10 @@ void *LexTokenise(struct LexState *Lexer, int *TokenLen)
struct Value *GotValue; struct Value *GotValue;
int MemUsed = 0; int MemUsed = 0;
int ValueSize; int ValueSize;
#ifdef FANCY_ERROR_REPORTING
int ReserveSpace = (Lexer->End - Lexer->Pos) * 4 + 16; int ReserveSpace = (Lexer->End - Lexer->Pos) * 4 + 16;
#else
int ReserveSpace = (Lexer->End - Lexer->Pos) * 3 + 16;
#endif
void *TokenSpace = HeapAllocStack(ReserveSpace); void *TokenSpace = HeapAllocStack(ReserveSpace);
char *TokenPos = (char *)TokenSpace; char *TokenPos = (char *)TokenSpace;
#ifdef FANCY_ERROR_REPORTING
int LastCharacterPos = 0; int LastCharacterPos = 0;
#endif
if (TokenSpace == NULL) if (TokenSpace == NULL)
LexFail(Lexer, "out of memory"); LexFail(Lexer, "out of memory");
@ -525,11 +522,9 @@ void *LexTokenise(struct LexState *Lexer, int *TokenLen)
TokenPos++; TokenPos++;
MemUsed++; MemUsed++;
#ifdef FANCY_ERROR_REPORTING
*(unsigned char *)TokenPos = (unsigned char)LastCharacterPos; *(unsigned char *)TokenPos = (unsigned char)LastCharacterPos;
TokenPos++; TokenPos++;
MemUsed++; MemUsed++;
#endif
ValueSize = LexTokenSize(Token); ValueSize = LexTokenSize(Token);
if (ValueSize > 0) if (ValueSize > 0)
@ -540,9 +535,7 @@ void *LexTokenise(struct LexState *Lexer, int *TokenLen)
MemUsed += ValueSize; MemUsed += ValueSize;
} }
#ifdef FANCY_ERROR_REPORTING
LastCharacterPos = Lexer->CharacterPos; LastCharacterPos = Lexer->CharacterPos;
#endif
} while (Token != TokenEOF); } while (Token != TokenEOF);
@ -578,10 +571,9 @@ void *LexAnalyse(const char *FileName, const char *Source, int SourceLen, int *T
Lexer.Line = 1; Lexer.Line = 1;
Lexer.FileName = FileName; Lexer.FileName = FileName;
Lexer.Mode = LexModeNormal; Lexer.Mode = LexModeNormal;
#ifdef FANCY_ERROR_REPORTING Lexer.EmitExtraNewlines = 0;
Lexer.CharacterPos = 1; Lexer.CharacterPos = 1;
Lexer.SourceText = Source; Lexer.SourceText = Source;
#endif
return LexTokenise(&Lexer, TokenLen); return LexTokenise(&Lexer, TokenLen);
} }
@ -596,10 +588,8 @@ void LexInitParser(struct ParseState *Parser, const char *SourceText, void *Toke
Parser->SearchLabel = 0; Parser->SearchLabel = 0;
Parser->HashIfLevel = 0; Parser->HashIfLevel = 0;
Parser->HashIfEvaluateToLevel = 0; Parser->HashIfEvaluateToLevel = 0;
#ifdef FANCY_ERROR_REPORTING
Parser->CharacterPos = 0; Parser->CharacterPos = 0;
Parser->SourceText = SourceText; Parser->SourceText = SourceText;
#endif
} }
/* get the next token, without pre-processing */ /* get the next token, without pre-processing */
@ -657,9 +647,7 @@ enum LexToken LexGetRawToken(struct ParseState *Parser, struct Value **Value, in
/* start a new list */ /* start a new list */
InteractiveHead = LineNode; InteractiveHead = LineNode;
Parser->Line = 1; Parser->Line = 1;
#ifdef FANCY_ERROR_REPORTING
Parser->CharacterPos = 0; Parser->CharacterPos = 0;
#endif
} }
else else
InteractiveTail->Next = LineNode; InteractiveTail->Next = LineNode;
@ -688,10 +676,7 @@ enum LexToken LexGetRawToken(struct ParseState *Parser, struct Value **Value, in
} }
} while ((Parser->FileName == StrEmpty && Token == TokenEOF) || Token == TokenEndOfLine); } while ((Parser->FileName == StrEmpty && Token == TokenEOF) || Token == TokenEndOfLine);
#ifdef FANCY_ERROR_REPORTING
Parser->CharacterPos = *((unsigned char *)Parser->Pos + 1); Parser->CharacterPos = *((unsigned char *)Parser->Pos + 1);
#endif
ValueSize = LexTokenSize(Token); ValueSize = LexTokenSize(Token);
if (ValueSize > 0) if (ValueSize > 0)
{ {

2
parse.c
View file

@ -315,9 +315,7 @@ void ParserCopyPos(struct ParseState *To, struct ParseState *From)
To->Line = From->Line; To->Line = From->Line;
To->HashIfLevel = From->HashIfLevel; To->HashIfLevel = From->HashIfLevel;
To->HashIfEvaluateToLevel = From->HashIfEvaluateToLevel; To->HashIfEvaluateToLevel = From->HashIfEvaluateToLevel;
#ifdef FANCY_ERROR_REPORTING
To->CharacterPos = From->CharacterPos; To->CharacterPos = From->CharacterPos;
#endif
} }
/* parse a "for" statement */ /* parse a "for" statement */

10
picoc.h
View file

@ -109,10 +109,8 @@ struct ParseState
int SearchLabel; /* what case label we're searching for */ int SearchLabel; /* what case label we're searching for */
int HashIfLevel; int HashIfLevel;
int HashIfEvaluateToLevel; int HashIfEvaluateToLevel;
#ifdef FANCY_ERROR_REPORTING
int CharacterPos; int CharacterPos;
const char *SourceText; const char *SourceText;
#endif
}; };
/* values */ /* values */
@ -249,8 +247,7 @@ enum LexMode
LexModeHashInclude, LexModeHashInclude,
LexModeHashDefine, LexModeHashDefine,
LexModeHashDefineSpace, LexModeHashDefineSpace,
LexModeHashDefineSpaceIdent, LexModeHashDefineSpaceIdent
LexModeMultiLineComment
}; };
struct LexState struct LexState
@ -259,11 +256,10 @@ struct LexState
const char *End; const char *End;
const char *FileName; const char *FileName;
int Line; int Line;
enum LexMode Mode;
#ifdef FANCY_ERROR_REPORTING
int CharacterPos; int CharacterPos;
const char *SourceText; const char *SourceText;
#endif enum LexMode Mode;
int EmitExtraNewlines;
}; };
/* library function definition */ /* library function definition */

11
platform.c
View file

@ -1,6 +1,5 @@
#include "picoc.h" #include "picoc.h"
#ifdef FANCY_ERROR_REPORTING
void PrintSourceTextErrorLine(const char *FileName, const char *SourceText, int Line, int CharacterPos) void PrintSourceTextErrorLine(const char *FileName, const char *SourceText, int Line, int CharacterPos)
{ {
int LineCount; int LineCount;
@ -40,17 +39,12 @@ void PrintSourceTextErrorLine(const char *FileName, const char *SourceText, int
PlatformPrintf("^\n%s:%d: ", FileName, Line, CharacterPos); PlatformPrintf("^\n%s:%d: ", FileName, Line, CharacterPos);
} }
#endif
/* display the source line and line number to identify an error */ /* display the source line and line number to identify an error */
void PlatformErrorPrefix(struct ParseState *Parser) void PlatformErrorPrefix(struct ParseState *Parser)
{ {
if (Parser != NULL) if (Parser != NULL)
#ifdef FANCY_ERROR_REPORTING
PrintSourceTextErrorLine(Parser->FileName, Parser->SourceText, Parser->Line, Parser->CharacterPos); PrintSourceTextErrorLine(Parser->FileName, Parser->SourceText, Parser->Line, Parser->CharacterPos);
#else
PlatformPrintf("%s:%d: ", Parser->FileName, Parser->Line);
#endif
} }
/* exit with a message */ /* exit with a message */
@ -88,12 +82,7 @@ void LexFail(struct LexState *Lexer, const char *Message, ...)
{ {
va_list Args; va_list Args;
#ifdef FANCY_ERROR_REPORTING
PrintSourceTextErrorLine(Lexer->FileName, Lexer->SourceText, Lexer->Line, Lexer->CharacterPos); PrintSourceTextErrorLine(Lexer->FileName, Lexer->SourceText, Lexer->Line, Lexer->CharacterPos);
#else
PlatformPrintf("%s:%d: ", Lexer->FileName, Lexer->Line);
#endif
va_start(Args, Message); va_start(Args, Message);
PlatformVPrintf(Message, Args); PlatformVPrintf(Message, Args);
va_end(Args); va_end(Args);

3
platform.h
View file

@ -26,8 +26,6 @@
#define LOCAL_TABLE_SIZE 11 /* size of local variable table (can expand) */ #define LOCAL_TABLE_SIZE 11 /* size of local variable table (can expand) */
#define STRUCT_TABLE_SIZE 11 /* size of struct/union member table (can expand) */ #define STRUCT_TABLE_SIZE 11 /* size of struct/union member table (can expand) */
#define FANCY_ERROR_REPORTING /* optional feature - gives more detailed error messages but uses more memory */
#define INTERACTIVE_PROMPT_START "starting picoc\n" #define INTERACTIVE_PROMPT_START "starting picoc\n"
#define INTERACTIVE_PROMPT_STATEMENT "picoc> " #define INTERACTIVE_PROMPT_STATEMENT "picoc> "
#define INTERACTIVE_PROMPT_LINE " > " #define INTERACTIVE_PROMPT_LINE " > "
@ -95,7 +93,6 @@ extern jmp_buf ExitBuf;
# define assert(x) # define assert(x)
# define BUILTIN_MINI_STDLIB # define BUILTIN_MINI_STDLIB
# undef BIG_ENDIAN # undef BIG_ENDIAN
# undef FANCY_ERROR_REPORTING
# else # else
# ifdef SURVEYOR_HOST # ifdef SURVEYOR_HOST