gered/picoc
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

formatting, initial multiline macro support

Browse source
This commit is contained in:
Joseph Poirier 2015-06-19 04:45:51 -05:00
parent 27b73e91de
commit 9316233d16
3 changed files with 152 additions and 126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
interpreter.h
View file

@ -151,7 +151,8 @@ enum LexToken {
/* 0x5b */ TokenOpenMacroBracket, /* 0x5b */ TokenOpenMacroBracket,
/* 0x5c */ TokenEOF, /* 0x5c */ TokenEOF,
TokenEndOfLine, TokenEndOfLine,
TokenEndOfFunction TokenEndOfFunction,
TokenBackSlash
}; };
/* used in dynamic memory allocation */ /* used in dynamic memory allocation */
@ -512,7 +513,7 @@ extern void LexInitParser(struct ParseState *Parser, Picoc *pc,
extern enum LexToken LexGetToken(struct ParseState *Parser, struct Value **Value, extern enum LexToken LexGetToken(struct ParseState *Parser, struct Value **Value,
int IncPos); int IncPos);
extern enum LexToken LexRawPeekToken(struct ParseState *Parser); extern enum LexToken LexRawPeekToken(struct ParseState *Parser);
extern void LexToEndOfLine(struct ParseState *Parser); extern void LexToEndOfMacro(struct ParseState *Parser);
extern void *LexCopyTokens(struct ParseState *StartParser, struct ParseState *EndParser); extern void *LexCopyTokens(struct ParseState *StartParser, struct ParseState *EndParser);
extern void LexInteractiveClear(Picoc *pc, struct ParseState *Parser); extern void LexInteractiveClear(Picoc *pc, struct ParseState *Parser);
extern void LexInteractiveCompleted(Picoc *pc, struct ParseState *Parser); extern void LexInteractiveCompleted(Picoc *pc, struct ParseState *Parser);

73
lex.c
View file

@ -20,7 +20,8 @@
#define LEXER_INCN(l, n) ( (l)->Pos+=(n), (l)->CharacterPos+=(n) ) #define LEXER_INCN(l, n) ( (l)->Pos+=(n), (l)->CharacterPos+=(n) )
#define TOKEN_DATA_OFFSET (2) #define TOKEN_DATA_OFFSET (2)
#define MAX_CHAR_VALUE (255) /* maximum value which can be represented by a "char" data type */ /* maximum value which can be represented by a "char" data type */
#define MAX_CHAR_VALUE (255)
static enum LexToken LexCheckReservedWord(Picoc *pc, const char *Word); static enum LexToken LexCheckReservedWord(Picoc *pc, const char *Word);
static enum LexToken LexGetNumber(Picoc *pc, struct LexState *Lexer, struct Value *Value); static enum LexToken LexGetNumber(Picoc *pc, struct LexState *Lexer, struct Value *Value);
@ -32,8 +33,8 @@ static enum LexToken LexGetStringConstant(Picoc *pc, struct LexState *Lexer,
struct Value *Value, char EndChar); struct Value *Value, char EndChar);
static enum LexToken LexGetCharacterConstant(Picoc *pc, struct LexState *Lexer, static enum LexToken LexGetCharacterConstant(Picoc *pc, struct LexState *Lexer,
struct Value *Value); struct Value *Value);
static void LexSkipComment(struct LexState *Lexer, char NextChar, static void LexSkipComment(struct LexState *Lexer, char NextChar);
enum LexToken *ReturnToken); static void LexSkipLineCont(struct LexState *Lexer, char NextChar);
static enum LexToken LexScanGetToken(Picoc *pc, struct LexState *Lexer, static enum LexToken LexScanGetToken(Picoc *pc, struct LexState *Lexer,
struct Value **Value); struct Value **Value);
static int LexTokenSize(enum LexToken Token); static int LexTokenSize(enum LexToken Token);
@ -103,7 +104,7 @@ void LexInit(Picoc *pc)
int Count; int Count;
TableInitTable(&pc->ReservedWordTable, &pc->ReservedWordHashTable[0], TableInitTable(&pc->ReservedWordTable, &pc->ReservedWordHashTable[0],
sizeof(ReservedWords) / sizeof(struct ReservedWord) * 2, true); sizeof(ReservedWords) / sizeof(struct ReservedWord)*2, true);
for (Count = 0; Count < sizeof(ReservedWords) / sizeof(struct ReservedWord); for (Count = 0; Count < sizeof(ReservedWords) / sizeof(struct ReservedWord);
Count++) { Count++) {
@ -290,11 +291,11 @@ unsigned char LexUnEscapeCharacter(const char **From, const char *End)
{ {
unsigned char ThisChar; unsigned char ThisChar;
while ( *From != End && **From == '\\' && while (*From != End && **From == '\\' &&
&(*From)[1] != End && (*From)[1] == '\n' ) &(*From)[1] != End && (*From)[1] == '\n' )
(*From) += 2; /* skip escaped end of lines with LF line termination */ (*From) += 2; /* skip escaped end of lines with LF line termination */
while ( *From != End && **From == '\\' && while (*From != End && **From == '\\' &&
&(*From)[1] != End && &(*From)[1] != End &&
&(*From)[2] != End && (*From)[1] == '\r' && (*From)[2] == '\n') &(*From)[2] != End && (*From)[1] == '\r' && (*From)[2] == '\n')
(*From) += 3; /* skip escaped end of lines with CR/LF line termination */ (*From) += 3; /* skip escaped end of lines with CR/LF line termination */
@ -420,8 +421,7 @@ enum LexToken LexGetCharacterConstant(Picoc *pc, struct LexState *Lexer,
} }
/* skip a comment - used while scanning */ /* skip a comment - used while scanning */
void LexSkipComment(struct LexState *Lexer, char NextChar, void LexSkipComment(struct LexState *Lexer, char NextChar)
enum LexToken *ReturnToken)
{ {
if (NextChar == '*') { if (NextChar == '*') {
/* conventional C comment */ /* conventional C comment */
@ -429,7 +429,6 @@ void LexSkipComment(struct LexState *Lexer, char NextChar,
(*(Lexer->Pos-1) != '*' || *Lexer->Pos != '/')) { (*(Lexer->Pos-1) != '*' || *Lexer->Pos != '/')) {
if (*Lexer->Pos == '\n') if (*Lexer->Pos == '\n')
Lexer->EmitExtraNewlines++; Lexer->EmitExtraNewlines++;
LEXER_INC(Lexer); LEXER_INC(Lexer);
} }
@ -444,6 +443,14 @@ void LexSkipComment(struct LexState *Lexer, char NextChar,
} }
} }
/* skip a line continuation - used while scanning */
void LexSkipLineCont(struct LexState *Lexer, char NextChar)
{
while (Lexer->Pos != Lexer->End && *Lexer->Pos != '\n') {
LEXER_INC(Lexer);
}
}
/* get a single token from the source - used while scanning */ /* get a single token from the source - used while scanning */
enum LexToken LexScanGetToken(Picoc *pc, struct LexState *Lexer, enum LexToken LexScanGetToken(Picoc *pc, struct LexState *Lexer,
struct Value **Value) struct Value **Value)
@ -522,7 +529,7 @@ enum LexToken LexScanGetToken(Picoc *pc, struct LexState *Lexer,
case '/': case '/':
if (NextChar == '/' || NextChar == '*') { if (NextChar == '/' || NextChar == '*') {
LEXER_INC(Lexer); LEXER_INC(Lexer);
LexSkipComment(Lexer, NextChar, &GotToken); LexSkipComment(Lexer, NextChar);
} else } else
NEXTIS('=', TokenDivideAssign, TokenSlash); NEXTIS('=', TokenDivideAssign, TokenSlash);
break; break;
@ -584,6 +591,14 @@ enum LexToken LexScanGetToken(Picoc *pc, struct LexState *Lexer,
case ':': case ':':
GotToken = TokenColon; GotToken = TokenColon;
break; break;
// XXX: multiline support
// case '\\':
// if (NextChar == ' ' || NextChar == '\n') {
// LEXER_INC(Lexer);
// LexSkipLineCont(Lexer, NextChar);
// } else
// LexFail(pc, Lexer, "xx illegal character '%c'", ThisChar);
// break;
default: default:
LexFail(pc, Lexer, "illegal character '%c'", ThisChar); LexFail(pc, Lexer, "illegal character '%c'", ThisChar);
break; break;
@ -629,18 +644,18 @@ void *LexTokenise(Picoc *pc, struct LexState *Lexer, int *TokenLen)
#ifdef DEBUG_LEXER #ifdef DEBUG_LEXER
printf("Token: %02x\n", Token); printf("Token: %02x\n", Token);
#endif #endif
*(unsigned char *)TokenPos = Token; *(unsigned char*)TokenPos = Token;
TokenPos++; TokenPos++;
MemUsed++; MemUsed++;
*(unsigned char *)TokenPos = (unsigned char)LastCharacterPos; *(unsigned char*)TokenPos = (unsigned char)LastCharacterPos;
TokenPos++; TokenPos++;
MemUsed++; MemUsed++;
ValueSize = LexTokenSize(Token); ValueSize = LexTokenSize(Token);
if (ValueSize > 0) { if (ValueSize > 0) {
/* store a value as well */ /* store a value as well */
memcpy((void *)TokenPos, (void *)GotValue->Val, ValueSize); memcpy((void*)TokenPos, (void*)GotValue->Val, ValueSize);
TokenPos += ValueSize; TokenPos += ValueSize;
MemUsed += ValueSize; MemUsed += ValueSize;
} }
@ -661,7 +676,7 @@ void *LexTokenise(Picoc *pc, struct LexState *Lexer, int *TokenLen)
int Count; int Count;
printf("Tokens: "); printf("Tokens: ");
for (Count = 0; Count < MemUsed; Count++) for (Count = 0; Count < MemUsed; Count++)
printf("%02x ", *((unsigned char *)HeapMem+Count)); printf("%02x ", *((unsigned char*)HeapMem+Count));
printf("\n"); printf("\n");
} }
#endif #endif
@ -784,12 +799,12 @@ enum LexToken LexGetRawToken(struct ParseState *Parser, struct Value **Value,
Parser->Pos = pc->InteractiveCurrentLine->Tokens; Parser->Pos = pc->InteractiveCurrentLine->Tokens;
} }
Token = (enum LexToken)*(unsigned char *)Parser->Pos; Token = (enum LexToken)*(unsigned char*)Parser->Pos;
} }
} while ((Parser->FileName == pc->StrEmpty && } while ((Parser->FileName == pc->StrEmpty &&
Token == TokenEOF) || Token == TokenEndOfLine); Token == TokenEOF) || Token == TokenEndOfLine);
Parser->CharacterPos = *((unsigned char *)Parser->Pos + 1); Parser->CharacterPos = *((unsigned char*)Parser->Pos + 1);
ValueSize = LexTokenSize(Token); ValueSize = LexTokenSize(Token);
if (ValueSize > 0) { if (ValueSize > 0) {
/* this token requires a value - unpack it */ /* this token requires a value - unpack it */
@ -814,8 +829,8 @@ enum LexToken LexGetRawToken(struct ParseState *Parser, struct Value **Value,
break; break;
} }
memcpy((void *)pc->LexValue.Val, memcpy((void*)pc->LexValue.Val,
(void *)((char *)Parser->Pos+TOKEN_DATA_OFFSET), ValueSize); (void*)((char*)Parser->Pos+TOKEN_DATA_OFFSET), ValueSize);
pc->LexValue.ValOnHeap = false; pc->LexValue.ValOnHeap = false;
pc->LexValue.ValOnStack = false; pc->LexValue.ValOnStack = false;
pc->LexValue.IsLValue = false; pc->LexValue.IsLValue = false;
@ -1083,17 +1098,27 @@ enum LexToken LexGetToken(struct ParseState *Parser, struct Value **Value,
/* take a quick peek at the next token, skipping any pre-processing */ /* take a quick peek at the next token, skipping any pre-processing */
enum LexToken LexRawPeekToken(struct ParseState *Parser) enum LexToken LexRawPeekToken(struct ParseState *Parser)
{ {
return (enum LexToken)*(unsigned char *)Parser->Pos; return (enum LexToken)*(unsigned char*)Parser->Pos;
} }
/* find the end of the line */ /* find the end of the line */
void LexToEndOfLine(struct ParseState *Parser) void LexToEndOfMacro(struct ParseState *Parser)
{ {
// bool isContinued = false;
while (true) { while (true) {
enum LexToken Token = (enum LexToken)*(unsigned char *)Parser->Pos; enum LexToken Token = (enum LexToken)*(unsigned char*)Parser->Pos;
if (Token == TokenEndOfLine || Token == TokenEOF) if (Token == TokenEndOfLine || Token == TokenEOF)
return; return;
// XXX: multiline support
// else if (Token == TokenEndOfLine) {
// if (!isContinued)
// return;
// else
// isContinued = false;
else else
// XXX: multiline support
// if (Token == TokenBackSlash)
// isContinued = true;
LexGetRawToken(Parser, NULL, true); LexGetRawToken(Parser, NULL, true);
} }
} }
@ -1104,7 +1129,7 @@ void *LexCopyTokens(struct ParseState *StartParser, struct ParseState *EndParser
{ {
int MemSize = 0; int MemSize = 0;
int CopySize; int CopySize;
unsigned char *Pos = (unsigned char *)StartParser->Pos; unsigned char *Pos = (unsigned char*)StartParser->Pos;
unsigned char *NewTokens; unsigned char *NewTokens;
unsigned char *NewTokenPos; unsigned char *NewTokenPos;
struct TokenLine *ILine; struct TokenLine *ILine;
@ -1114,7 +1139,7 @@ void *LexCopyTokens(struct ParseState *StartParser, struct ParseState *EndParser
/* non-interactive mode - copy the tokens */ /* non-interactive mode - copy the tokens */
MemSize = EndParser->Pos - StartParser->Pos; MemSize = EndParser->Pos - StartParser->Pos;
NewTokens = VariableAlloc(pc, StartParser, MemSize + TOKEN_DATA_OFFSET, true); NewTokens = VariableAlloc(pc, StartParser, MemSize + TOKEN_DATA_OFFSET, true);
memcpy(NewTokens, (void *)StartParser->Pos, MemSize); memcpy(NewTokens, (void*)StartParser->Pos, MemSize);
} else { } else {
/* we're in interactive mode - add up line by line */ /* we're in interactive mode - add up line by line */
for (pc->InteractiveCurrentLine = pc->InteractiveHead; for (pc->InteractiveCurrentLine = pc->InteractiveHead;
@ -1129,7 +1154,7 @@ void *LexCopyTokens(struct ParseState *StartParser, struct ParseState *EndParser
/* all on a single line */ /* all on a single line */
MemSize = EndParser->Pos - StartParser->Pos; MemSize = EndParser->Pos - StartParser->Pos;
NewTokens = VariableAlloc(pc, StartParser, MemSize + TOKEN_DATA_OFFSET, true); NewTokens = VariableAlloc(pc, StartParser, MemSize + TOKEN_DATA_OFFSET, true);
memcpy(NewTokens, (void *)StartParser->Pos, MemSize); memcpy(NewTokens, (void*)StartParser->Pos, MemSize);
} else { } else {
/* it's spread across multiple lines */ /* it's spread across multiple lines */
MemSize = &pc->InteractiveCurrentLine->Tokens[pc->InteractiveCurrentLine->NumBytes-TOKEN_DATA_OFFSET] - Pos; MemSize = &pc->InteractiveCurrentLine->Tokens[pc->InteractiveCurrentLine->NumBytes-TOKEN_DATA_OFFSET] - Pos;

6
parse.c
View file

@ -397,7 +397,7 @@ void ParseMacroDefinition(struct ParseState *Parser)
MacroNameStr = MacroName->Val->Identifier; MacroNameStr = MacroName->Val->Identifier;
if (LexRawPeekToken(Parser) == TokenOpenMacroBracket) { if (LexRawPeekToken(Parser) == TokenOpenMacroBracket) {
/* it's a parameterised macro, read the parameters */ /* it's a parameterized macro, read the parameters */
enum LexToken Token = LexGetToken(Parser, NULL, true); enum LexToken Token = LexGetToken(Parser, NULL, true);
struct ParseState ParamParser; struct ParseState ParamParser;
int NumParams; int NumParams;
@ -431,7 +431,7 @@ void ParseMacroDefinition(struct ParseState *Parser)
if (Token != TokenCloseBracket) if (Token != TokenCloseBracket)
ProgramFail(Parser, "close bracket expected"); ProgramFail(Parser, "close bracket expected");
} else { } else {
/* allocate a simple unparameterised macro */ /* allocate a simple unparameterized macro */
MacroValue = VariableAllocValueAndData(Parser->pc, Parser, MacroValue = VariableAllocValueAndData(Parser->pc, Parser,
sizeof(struct MacroDef), false, NULL, true); sizeof(struct MacroDef), false, NULL, true);
MacroValue->Val->MacroDef.NumParams = 0; MacroValue->Val->MacroDef.NumParams = 0;
@ -440,7 +440,7 @@ void ParseMacroDefinition(struct ParseState *Parser)
/* copy the body of the macro to execute later */ /* copy the body of the macro to execute later */
ParserCopy(&MacroValue->Val->MacroDef.Body, Parser); ParserCopy(&MacroValue->Val->MacroDef.Body, Parser);
MacroValue->Typ = &Parser->pc->MacroType; MacroValue->Typ = &Parser->pc->MacroType;
LexToEndOfLine(Parser); LexToEndOfMacro(Parser);
MacroValue->Val->MacroDef.Body.Pos = MacroValue->Val->MacroDef.Body.Pos =
LexCopyTokens(&MacroValue->Val->MacroDef.Body, Parser); LexCopyTokens(&MacroValue->Val->MacroDef.Body, Parser);