lex: Refactor the tokenizer into a lexer module

diff --git a/src/Makefile b/src/Makefile
index 95609902aebe8faec6d03d601d25def63b769dd7..1a0b10f3d651be8997e01743c6dd7f90fefd33f7 100644 (file)
--- a/src/Makefile
+++ b/src/Makefile
@@ -1,4 +1,4 @@
-OBJECTS = str.o token.o list.o lex.o
+OBJECTS = str.o token.o list.o lex.o main.o
 OUTPUT = c3
 PHONY = clean
 

diff --git a/src/lex.c b/src/lex.c
index c70f5e3611ba8a5392a7d245144ff394b754f025..6175fd68d33e10dcb3ea87902a765fca2bdd08ba 100644 (file)
--- a/src/lex.c
+++ b/src/lex.c
@@ -48,9 +48,9 @@ static int _line = 1;
 static int _col = 0;
 static int _pline = 1;
 static int _pcol = 1;
-list_t *tokens = NULL;
+static struct token *_next_token = NULL;
 
-int getnext(void)
+static int _getnextchar(void)
 {
        int ret_val;
 
@@ -90,7 +90,7 @@ int getnext(void)
        return(ret_val);
 }
 
-int putnext(const char c)
+static int _putnextchar(const char c)
 {
        int ret_val;
 
@@ -107,16 +107,20 @@ int putnext(const char c)
        return(ret_val);
 }
 
-struct token *nexttoken(void)
+struct token *lex_gettoken(void)
 {
        struct token *tok;
        int state;
        char c;
 
+       if(_next_token) {
+               return(_next_token);
+       }
+
        state = STATE_NONE;
        tok = NULL;
 
-       while((c = getnext())) {
+       while((c = _getnextchar())) {
                switch(state) {
                case STATE_NONE:
                        switch(c) {
@@ -236,7 +240,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, "/="));
 
                        default:
-                               putnext(c);
+                               _putnextchar(c);
                                return(token_new2(_line, _col, "/"));
                        }
 
@@ -277,7 +281,7 @@ struct token *nexttoken(void)
                                break;
 
                        default:
-                               putnext(c);
+                               _putnextchar(c);
                                return(token_new2(_line, _col, "<"));
                        }
 
@@ -288,7 +292,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, "<<="));
                        }
 
-                       putnext(c);
+                       _putnextchar(c);
                        return(token_new2(_line, _col, "<<"));
 
                case STATE_GT:
@@ -301,7 +305,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, ">="));
 
                        default:
-                               putnext(c);
+                               _putnextchar(c);
                                return(token_new2(_line, _col, ">"));
                        }
 
@@ -312,7 +316,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, ">>="));
                        }
 
-                       putnext(c);
+                       _putnextchar(c);
                        return(token_new2(_line, _col, ">>"));
 
                case STATE_NOT:
@@ -320,7 +324,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, "!="));
                        }
 
-                       putnext(c);
+                       _putnextchar(c);
                        return(token_new2(_line, _col, "!"));
 
                case STATE_MOD:
@@ -328,7 +332,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, "%="));
                        }
 
-                       putnext(c);
+                       _putnextchar(c);
                        return(token_new2(_line, _col, "%"));
 
                case STATE_AND:
@@ -340,7 +344,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, "&="));
 
                        default:
-                               putnext(c);
+                               _putnextchar(c);
                                return(token_new2(_line, _col, "&"));
                        }
 
@@ -353,7 +357,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, "|="));
 
                        default:
-                               putnext(c);
+                               _putnextchar(c);
                                return(token_new2(_line, _col, "|"));
                        }
 
@@ -362,7 +366,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, "^="));
                        }
 
-                       putnext(c);
+                       _putnextchar(c);
                        return(token_new2(_line, _col, "^"));
 
                case STATE_ASSIGN:
@@ -370,7 +374,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, "=="));
                        }
 
-                       putnext(c);
+                       _putnextchar(c);
                        return(token_new2(_line, _col, "="));
 
                case STATE_STRING:
@@ -419,7 +423,7 @@ struct token *nexttoken(void)
                        if(identifier_chr(c)) {
                                str_appendc(tok->value, c);
                        } else {
-                               putnext(c);
+                               _putnextchar(c);
                                return(tok);
                        }
 
@@ -431,7 +435,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, "*="));
 
                        default:
-                               putnext(c);
+                               _putnextchar(c);
                                return(token_new2(_line, _col, "*"));
                        }
 
@@ -444,7 +448,7 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, "+="));
 
                        default:
-                               putnext(c);
+                               _putnextchar(c);
                                return(token_new2(_line, _col, "+"));
                        }
 
@@ -460,12 +464,12 @@ struct token *nexttoken(void)
                                return(token_new2(_line, _col, "->"));
 
                        default:
-                               putnext(c);
+                               _putnextchar(c);
                                return(token_new2(_line, _col, "-"));
                        }
 
                case STATE_DOT:
-                       putnext(c);
+                       _putnextchar(c);
 
                        if(c >= '0' && c <= '9') {
                                state = STATE_NUM;
@@ -480,12 +484,12 @@ struct token *nexttoken(void)
                                str_appendc(tok->value, 'x');
                                state = STATE_NUM;
                        } else if(c >= '0' && c <= '9') {
-                               putnext(c);
+                               _putnextchar(c);
                                state = STATE_NUM;
                        } else if(c == '.') {
 
                        } else {
-                               putnext(c);
+                               _putnextchar(c);
                                return(tok);
                        }
 
@@ -496,7 +500,7 @@ struct token *nexttoken(void)
                        if(c >= '0' && c <= '9' || c == '.') {
                                str_appendc(tok->value, c);
                        } else {
-                               putnext(c);
+                               _putnextchar(c);
                                return(tok);
                        }
 
@@ -507,15 +511,15 @@ struct token *nexttoken(void)
        return(tok);
 }
 
-int main(int argc, char *argv[])
+int lex_puttoken(struct token *tok)
 {
        int ret_val;
-       struct token *tok;
-
-       ret_val = 0;
 
-       while((tok = nexttoken())) {
-               printf("Token at %4d:%3d: \"%s\"\n", tok->line, tok->column, str_value(tok->value));
+       if(_next_token) {
+               ret_val = -EALREADY;
+       } else {
+               _next_token = tok;
+               ret_val = 0;
        }
 
        return(ret_val);

diff --git a/src/lex.h b/src/lex.h
new file mode 100644 (file)
index 0000000..a4fc46e
--- /dev/null
+++ b/src/lex.h
@@ -0,0 +1,9 @@
+#ifndef LEX_H
+#define LEX_H
+
+#include "token.h"
+
+struct token *lex_gettoken(void);
+int lex_puttoken(struct token*);
+
+#endif /* LEX_H */

diff --git a/src/main.c b/src/main.c
new file mode 100644 (file)
index 0000000..88032c0
--- /dev/null
+++ b/src/main.c
@@ -0,0 +1,15 @@
+#include <stdio.h>
+#include "lex.h"
+#include "str.h"
+
+int main(int argc, char *argv[])
+{
+       struct token *tok;
+
+       while((tok = lex_gettoken())) {
+               printf("Token at %4d:%3d: \"%s\"\n", tok->line, tok->column, str_value(tok->value));
+               token_free(tok);
+       }
+
+       return(0);
+}