lex: Assign tokens a type during the lexing step

diff --git a/src/lex.c b/src/lex.c
index 6175fd68d33e10dcb3ea87902a765fca2bdd08ba..655509857b5391e8699b0373a81b385856e677c6 100644 (file)
--- a/src/lex.c
+++ b/src/lex.c
@@ -185,17 +185,37 @@ struct token *lex_gettoken(void)
                                break;
 
                        case '~':
+                               return(token_new_from_char(TOKEN_COMPLEMENT, _line, _col, c));
+
                        case '(':
+                               return(token_new_from_char(TOKEN_LPAREN, _line, _col, c));
+
                        case ')':
+                               return(token_new_from_char(TOKEN_RPAREN, _line, _col, c));
+
                        case '{':
+                               return(token_new_from_char(TOKEN_LBRACE, _line, _col, c));
+
                        case '}':
+                               return(token_new_from_char(TOKEN_RBRACE, _line, _col, c));
+
                        case '[':
+                               return(token_new_from_char(TOKEN_LBRACKET, _line, _col, c));
+
                        case ']':
+                               return(token_new_from_char(TOKEN_RBRACKET, _line, _col, c));
+
                        case ':':
+                               return(token_new_from_char(TOKEN_COLON, _line, _col, c));
+
                        case ';':
+                               return(token_new_from_char(TOKEN_SEMICOLON, _line, _col, c));
+
                        case ',':
+                               return(token_new_from_char(TOKEN_COMMA, _line, _col, c));
+
                        case '?':
-                               return(token_new_from_char(_line, _col, c));
+                               return(token_new_from_char(TOKEN_QMARK, _line, _col, c));
 
                        case '\r':
                                fprintf(stderr, "DOS user, eh?\n");
@@ -207,17 +227,17 @@ struct token *lex_gettoken(void)
                        default:
                                if(identifier_firstchr(c)) {
                                        /* looks like an identifier */
-                                       tok = token_new_from_char(_line, _col, c);
+                                       tok = token_new_from_char(TOKEN_IDENTIFIER, _line, _col, c);
                                        assert(tok);
                                        state = STATE_ID;
                                        break;
                                } else if(c == '0') {
-                                       tok = token_new_from_char(_line, _col, c);
+                                       tok = token_new_from_char(TOKEN_NUMBER, _line, _col, c);
                                        assert(tok);
                                        state = STATE_ZERO;
                                        break;
                                } else if(c > '0' && c <= '9') {
-                                       tok = token_new_from_char(_line, _col, c);
+                                       tok = token_new_from_char(TOKEN_NUMBER, _line, _col, c);
                                        assert(tok);
                                        state = STATE_NUM;
                                        break;
@@ -237,22 +257,22 @@ struct token *lex_gettoken(void)
                                break;
 
                        case '=':
-                               return(token_new2(_line, _col, "/="));
+                               return(token_new_from_str(TOKEN_DIV_ASSIGN, _line, _col, "/="));
 
                        default:
                                _putnextchar(c);
-                               return(token_new2(_line, _col, "/"));
+                               return(token_new_from_str(TOKEN_DIV, _line, _col, "/"));
                        }
 
                        break;
 
                case STATE_COMMENT:
                        if(!tok) {
-                               tok = token_new2(_line, _col, "/*");
+                               tok = token_new_from_str(TOKEN_COMMENT, _line, _col, "/*");
                                assert(tok);
                        }
 
-                       str_appendc(tok->value, c);
+                       token_append_char(tok, c);
 
                        if(c == '*') {
                                state = STATE_COMMENT_END;
@@ -261,7 +281,7 @@ struct token *lex_gettoken(void)
                        break;
 
                case STATE_COMMENT_END:
-                       str_appendc(tok->value, c);
+                       token_append_char(tok, c);
 
                        if(c == '/') {
                                return(tok);
@@ -277,23 +297,23 @@ struct token *lex_gettoken(void)
                                break;
 
                        case '=':
-                               return(token_new2(_line, _col, "<="));
+                               return(token_new_from_str(TOKEN_LE, _line, _col, "<="));
                                break;
 
                        default:
                                _putnextchar(c);
-                               return(token_new2(_line, _col, "<"));
+                               return(token_new_from_str(TOKEN_LT, _line, _col, "<"));
                        }
 
                        break;
 
                case STATE_SHL:
                        if(c == '=') {
-                               return(token_new2(_line, _col, "<<="));
+                               return(token_new_from_str(TOKEN_ASSIGN_SHL, _line, _col, "<<="));
                        }
 
                        _putnextchar(c);
-                       return(token_new2(_line, _col, "<<"));
+                       return(token_new_from_str(TOKEN_SHL, _line, _col, "<<"));
 
                case STATE_GT:
                        switch(c) {
@@ -302,88 +322,88 @@ struct token *lex_gettoken(void)
                                break;
 
                        case '=':
-                               return(token_new2(_line, _col, ">="));
+                               return(token_new_from_str(TOKEN_GE, _line, _col, ">="));
 
                        default:
                                _putnextchar(c);
-                               return(token_new2(_line, _col, ">"));
+                               return(token_new_from_str(TOKEN_GT, _line, _col, ">"));
                        }
 
                        break;
 
                case STATE_SHR:
                        if(c == '=') {
-                               return(token_new2(_line, _col, ">>="));
+                               return(token_new_from_str(TOKEN_ASSIGN_SHR, _line, _col, ">>="));
                        }
 
                        _putnextchar(c);
-                       return(token_new2(_line, _col, ">>"));
+                       return(token_new_from_str(TOKEN_SHR, _line, _col, ">>"));
 
                case STATE_NOT:
                        if(c == '=') {
-                               return(token_new2(_line, _col, "!="));
+                               return(token_new_from_str(TOKEN_NEQ, _line, _col, "!="));
                        }
 
                        _putnextchar(c);
-                       return(token_new2(_line, _col, "!"));
+                       return(token_new_from_str(TOKEN_NOT, _line, _col, "!"));
 
                case STATE_MOD:
                        if(c == '=') {
-                               return(token_new2(_line, _col, "%="));
+                               return(token_new_from_str(TOKEN_ASSIGN_MOD, _line, _col, "%="));
                        }
 
                        _putnextchar(c);
-                       return(token_new2(_line, _col, "%"));
+                       return(token_new_from_str(TOKEN_MOD, _line, _col, "%"));
 
                case STATE_AND:
                        switch(c) {
                        case '&':
-                               return(token_new2(_line, _col, "&&"));
+                               return(token_new_from_str(TOKEN_LAND, _line, _col, "&&"));
 
                        case '=':
-                               return(token_new2(_line, _col, "&="));
+                               return(token_new_from_str(TOKEN_ASSIGN_AND, _line, _col, "&="));
 
                        default:
                                _putnextchar(c);
-                               return(token_new2(_line, _col, "&"));
+                               return(token_new_from_str(TOKEN_AND, _line, _col, "&"));
                        }
 
                case STATE_OR:
                        switch(c) {
                        case '|':
-                               return(token_new2(_line, _col, "||"));
+                               return(token_new_from_str(TOKEN_LOR, _line, _col, "||"));
 
                        case '=':
-                               return(token_new2(_line, _col, "|="));
+                               return(token_new_from_str(TOKEN_ASSIGN_OR, _line, _col, "|="));
 
                        default:
                                _putnextchar(c);
-                               return(token_new2(_line, _col, "|"));
+                               return(token_new_from_str(TOKEN_OR, _line, _col, "|"));
                        }
 
                case STATE_XOR:
                        if(c == '=') {
-                               return(token_new2(_line, _col, "^="));
+                               return(token_new_from_str(TOKEN_ASSIGN_XOR, _line, _col, "^="));
                        }
 
                        _putnextchar(c);
-                       return(token_new2(_line, _col, "^"));
+                       return(token_new_from_str(TOKEN_ASSIGN_XOR, _line, _col, "^"));
 
                case STATE_ASSIGN:
                        if(c == '=') {
-                               return(token_new2(_line, _col, "=="));
+                               return(token_new_from_str(TOKEN_EQ, _line, _col, "=="));
                        }
 
                        _putnextchar(c);
-                       return(token_new2(_line, _col, "="));
+                       return(token_new_from_str(TOKEN_ASSIGN, _line, _col, "="));
 
                case STATE_STRING:
                        if(!tok) {
-                               tok = token_new2(_line, _col, "\"");
+                               tok = token_new_from_str(TOKEN_STRING, _line, _col, "\"");
                                assert(tok);
                        }
 
-                       str_appendc(tok->value, c);
+                       token_append_char(tok, c);
 
                        if(c == '\\') {
                                state = STATE_STRING_ESC;
@@ -394,17 +414,17 @@ struct token *lex_gettoken(void)
                        break;
 
                case STATE_STRING_ESC:
-                       str_appendc(tok->value, c);
+                       token_append_char(tok, c);
                        state = STATE_STRING;
                        break;
 
                case STATE_CHR:
                        if(!tok) {
-                               tok = token_new2(_line, _col, "'");
+                               tok = token_new_from_str(TOKEN_CHAR, _line, _col, "'");
                                assert(tok);
                        }
 
-                       str_appendc(tok->value, c);
+                       token_append_char(tok, c);
 
                        if(c == '\\') {
                                state = STATE_CHR_ESC;
@@ -415,13 +435,13 @@ struct token *lex_gettoken(void)
                        break;
 
                case STATE_CHR_ESC:
-                       str_appendc(tok->value, c);
+                       token_append_char(tok, c);
                        state = STATE_CHR;
                        break;
 
                case STATE_ID:
                        if(identifier_chr(c)) {
-                               str_appendc(tok->value, c);
+                               token_append_char(tok, c);
                        } else {
                                _putnextchar(c);
                                return(tok);
@@ -432,40 +452,40 @@ struct token *lex_gettoken(void)
                case STATE_MUL:
                        switch(c) {
                        case '=':
-                               return(token_new2(_line, _col, "*="));
+                               return(token_new_from_str(TOKEN_ASSIGN_MUL, _line, _col, "*="));
 
                        default:
                                _putnextchar(c);
-                               return(token_new2(_line, _col, "*"));
+                               return(token_new_from_str(TOKEN_MUL, _line, _col, "*"));
                        }
 
                case STATE_ADD:
                        switch(c) {
                        case '+':
-                               return(token_new2(_line, _col, "++"));
+                               return(token_new_from_str(TOKEN_INC, _line, _col, "++"));
 
                        case '=':
-                               return(token_new2(_line, _col, "+="));
+                               return(token_new_from_str(TOKEN_ASSIGN_ADD, _line, _col, "+="));
 
                        default:
                                _putnextchar(c);
-                               return(token_new2(_line, _col, "+"));
+                               return(token_new_from_str(TOKEN_ADD, _line, _col, "+"));
                        }
 
                case STATE_SUB:
                        switch(c) {
                        case '-':
-                               return(token_new2(_line, _col, "--"));
+                               return(token_new_from_str(TOKEN_DEC, _line, _col, "--"));
 
                        case '=':
-                               return(token_new2(_line, _col, "-="));
+                               return(token_new_from_str(TOKEN_ASSIGN_SUB, _line, _col, "-="));
 
                        case '>':
-                               return(token_new2(_line, _col, "->"));
+                               return(token_new_from_str(TOKEN_DMEMBER, _line, _col, "->"));
 
                        default:
                                _putnextchar(c);
-                               return(token_new2(_line, _col, "-"));
+                               return(token_new_from_str(TOKEN_SUB, _line, _col, "-"));
                        }
 
                case STATE_DOT:
@@ -474,14 +494,14 @@ struct token *lex_gettoken(void)
                        if(c >= '0' && c <= '9') {
                                state = STATE_NUM;
                        } else {
-                               return(token_new2(_line, _col, "."));
+                               return(token_new_from_str(TOKEN_MEMBER, _line, _col, "."));
                        }
 
                        break;
 
                case STATE_ZERO:
                        if(c == 'x' || c == 'X') {
-                               str_appendc(tok->value, 'x');
+                               token_append_char(tok, 'x');
                                state = STATE_NUM;
                        } else if(c >= '0' && c <= '9') {
                                _putnextchar(c);
@@ -498,7 +518,7 @@ struct token *lex_gettoken(void)
                case STATE_NUM:
                        /* FIXME: e, E may be in the middle, (u|U)(l|L|ll|LL) or vice versa may be at the end */
                        if(c >= '0' && c <= '9' || c == '.') {
-                               str_appendc(tok->value, c);
+                               token_append_char(tok, c);
                        } else {
                                _putnextchar(c);
                                return(tok);