From ed26e6716b4875ee44a94180219b5643f6ae4ca3 Mon Sep 17 00:00:00 2001 From: "zik.saleeba" Date: Sat, 31 Oct 2009 14:18:46 +0000 Subject: [PATCH] Fixed an annoying problem with entering functions interactively. Includes a heap of debug. git-svn-id: http://picoc.googlecode.com/svn/trunk@362 21eae674-98b7-11dd-bd71-f92a316d2d60 --- lex.c | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/lex.c b/lex.c index a857558..de7bb55 100644 --- a/lex.c +++ b/lex.c @@ -494,11 +494,11 @@ void *LexTokenise(struct LexState *Lexer, int *TokenLen) TokenPos += ValueSize; MemUsed += ValueSize; } - + #ifdef FANCY_ERROR_REPORTING LastCharacterPos = Lexer->CharacterPos; #endif - + } while (Token != TokenEOF); HeapMem = HeapAllocMem(MemUsed); @@ -511,6 +511,7 @@ void *LexTokenise(struct LexState *Lexer, int *TokenLen) #ifdef DEBUG_LEXER { int Count; + printf("Tokens: "); for (Count = 0; Count < MemUsed; Count++) printf("%02x ", *((unsigned char *)HeapMem+Count)); printf("\n"); @@ -527,6 +528,7 @@ void *LexAnalyse(const char *FileName, const char *Source, int SourceLen, int *T { struct LexState Lexer; + printf("LexAnalyse(\"%s\")\n", Source); Lexer.Pos = Source; Lexer.End = Source + SourceLen; Lexer.Line = 1; @@ -535,6 +537,7 @@ void *LexAnalyse(const char *FileName, const char *Source, int SourceLen, int *T Lexer.CharacterPos = 1; Lexer.SourceText = Source; #endif + return LexTokenise(&Lexer, TokenLen); } @@ -558,12 +561,13 @@ enum LexToken LexGetToken(struct ParseState *Parser, struct Value **Value, int I enum LexToken Token = TokenNone; int ValueSize; + printf("Parser->Pos == %08lx\n", (unsigned long)Parser->Pos); do { /* get the next token */ if (Parser->Pos == NULL && InteractiveHead != NULL) Parser->Pos = InteractiveHead->Tokens; - + if (Parser->FileName != StrEmpty || InteractiveHead != NULL) { /* skip leading newlines */ @@ -581,7 +585,7 @@ enum LexToken LexGetToken(struct ParseState *Parser, struct Value **Value, int I void *LineTokens; int LineBytes; struct TokenLine *LineNode; - + if (InteractiveHead == NULL || (unsigned char *)Parser->Pos == &InteractiveTail->Tokens[InteractiveTail->NumBytes-TOKEN_DATA_OFFSET]) { /* get interactive input */ @@ -598,6 +602,7 @@ enum LexToken LexGetToken(struct ParseState *Parser, struct Value **Value, int I /* put the new line at the end of the linked list of interactive lines */ LineTokens = LexAnalyse(StrEmpty, &LineBuffer[0], strlen(LineBuffer), &LineBytes); + printf("Added new line node at %08lx-%08lx with text %s\n", (unsigned long)LineTokens, (unsigned long)LineTokens+LineBytes-TOKEN_DATA_OFFSET, &LineBuffer[0]); LineNode = VariableAlloc(Parser, sizeof(struct TokenLine), TRUE); LineNode->Tokens = LineTokens; LineNode->NumBytes = LineBytes; @@ -618,14 +623,21 @@ enum LexToken LexGetToken(struct ParseState *Parser, struct Value **Value, int I else { /* go to the next token line */ + printf("Next token line...\n"); if (Parser->Pos != &InteractiveCurrentLine->Tokens[InteractiveCurrentLine->NumBytes-TOKEN_DATA_OFFSET]) { /* scan for the line */ for (InteractiveCurrentLine = InteractiveHead; Parser->Pos != &InteractiveCurrentLine->Tokens[InteractiveCurrentLine->NumBytes-TOKEN_DATA_OFFSET]; InteractiveCurrentLine = InteractiveCurrentLine->Next) - {} + { + printf("InteractiveCurrentLine = %08lx-%08lx, Pos = %08lx, NumBytes = %d\n", (unsigned long)&InteractiveCurrentLine->Tokens[0], (unsigned long)&InteractiveCurrentLine->Tokens[InteractiveCurrentLine->NumBytes-TOKEN_DATA_OFFSET], (unsigned long)Parser->Pos, InteractiveCurrentLine->NumBytes); + assert(InteractiveCurrentLine->Next != NULL); + printf(" next InteractiveCurrentLine = %08lx-%08lx, Pos = %08lx, NumBytes = %d\n", (unsigned long)&InteractiveCurrentLine->Next->Tokens[0], (unsigned long)&InteractiveCurrentLine->Next->Tokens[InteractiveCurrentLine->Next->NumBytes-TOKEN_DATA_OFFSET], (unsigned long)Parser->Pos, InteractiveCurrentLine->Next->NumBytes); + } } + assert(InteractiveCurrentLine != NULL); InteractiveCurrentLine = InteractiveCurrentLine->Next; + assert(InteractiveCurrentLine != NULL); Parser->Pos = InteractiveCurrentLine->Tokens; } @@ -679,6 +691,7 @@ enum LexToken LexGetToken(struct ParseState *Parser, struct Value **Value, int I #ifdef DEBUG_LEXER printf("Got token=%02x inc=%d pos=%d\n", Token, IncPos, Parser->CharacterPos); #endif + assert(Token >= TokenNone && Token <= TokenEndOfFunction); return Token; } @@ -695,7 +708,7 @@ void LexToEndOfLine(struct ParseState *Parser) } } -/* copy the tokens from StartParser to EndParser into new memory and terminate with a TokenEOF */ +/* copy the tokens from StartParser to EndParser into new memory, removing TokenEOFs and terminate with a TokenEndOfFunction */ void *LexCopyTokens(struct ParseState *StartParser, struct ParseState *EndParser) { int MemSize = 0; @@ -721,35 +734,88 @@ void *LexCopyTokens(struct ParseState *StartParser, struct ParseState *EndParser if (EndParser->Pos >= StartParser->Pos && EndParser->Pos < &InteractiveCurrentLine->Tokens[InteractiveCurrentLine->NumBytes]) { /* all on a single line */ - MemSize = EndParser->Pos - StartParser->Pos; + MemSize = EndParser->Pos - StartParser->Pos - TOKEN_DATA_OFFSET; NewTokens = VariableAlloc(StartParser, MemSize + 1, TRUE); memcpy(NewTokens, (void *)StartParser->Pos, MemSize); } else { /* it's spread across multiple lines */ - MemSize = &InteractiveCurrentLine->Tokens[InteractiveCurrentLine->NumBytes-1] - Pos; + MemSize = &InteractiveCurrentLine->Tokens[InteractiveCurrentLine->NumBytes-TOKEN_DATA_OFFSET] - Pos; + { + int Count; + printf("Adding up: %d bytes - ", MemSize); + for (Count = 0; Count < MemSize; Count++) + printf("%02x ", Pos[Count]); + printf("\n"); + } + for (ILine = InteractiveCurrentLine->Next; ILine != NULL && (EndParser->Pos < &ILine->Tokens[0] || EndParser->Pos >= &ILine->Tokens[ILine->NumBytes]); ILine = ILine->Next) - MemSize += ILine->NumBytes - 1; + { + { + int Count; + printf("Adding up: %d bytes - ", ILine->NumBytes - TOKEN_DATA_OFFSET); + for (Count = 0; Count < ILine->NumBytes - TOKEN_DATA_OFFSET; Count++) + printf("%02x ", ILine->Tokens[Count]); + printf("\n"); + } + MemSize += ILine->NumBytes - TOKEN_DATA_OFFSET; + } assert(ILine != NULL); MemSize += EndParser->Pos - &ILine->Tokens[0]; + { + int Count; + printf("Adding up: %d bytes - ", EndParser->Pos - &ILine->Tokens[0]); + for (Count = 0; Count < EndParser->Pos - &ILine->Tokens[0]; Count++) + printf("%02x ", ILine->Tokens[Count]); + printf("\n"); + } NewTokens = VariableAlloc(StartParser, MemSize + 1, TRUE); - CopySize = &InteractiveCurrentLine->Tokens[InteractiveCurrentLine->NumBytes-1] - Pos; + CopySize = &InteractiveCurrentLine->Tokens[InteractiveCurrentLine->NumBytes-TOKEN_DATA_OFFSET] - Pos; memcpy(NewTokens, Pos, CopySize); + { + int Count; + printf("Copying - "); + for (Count = 0; Count < CopySize; Count++) + printf("%02x ", Pos[Count]); + printf("\n"); + } NewTokenPos = NewTokens + CopySize; for (ILine = InteractiveCurrentLine->Next; ILine != NULL && (EndParser->Pos < &ILine->Tokens[0] || EndParser->Pos >= &ILine->Tokens[ILine->NumBytes]); ILine = ILine->Next) { - memcpy(NewTokenPos, &ILine->Tokens[0], ILine->NumBytes-1); - NewTokenPos += ILine->NumBytes-1; + { + int Count; + printf("Copying - "); + for (Count = 0; Count < ILine->NumBytes - TOKEN_DATA_OFFSET; Count++) + printf("%02x ", ILine->Tokens[Count]); + printf("\n"); + } + memcpy(NewTokenPos, &ILine->Tokens[0], ILine->NumBytes - TOKEN_DATA_OFFSET); + NewTokenPos += ILine->NumBytes-TOKEN_DATA_OFFSET; } assert(ILine != NULL); + { + int Count; + printf("Copying - "); + for (Count = 0; Count < EndParser->Pos - &ILine->Tokens[0]; Count++) + printf("%02x ", ILine->Tokens[Count]); + printf("\n"); + } memcpy(NewTokenPos, &ILine->Tokens[0], EndParser->Pos - &ILine->Tokens[0]); } } NewTokens[MemSize] = (unsigned char)TokenEndOfFunction; + printf("Copied tokens to address range %08lx-%08lx, size %d\n", (unsigned long)NewTokens, (unsigned long)NewTokens+MemSize+1, MemSize+1); + { + int Count; + for (Count = 0; Count <= MemSize; Count++) + printf("%02x ", NewTokens[Count]); + printf("\n"); + } + return NewTokens; }