lex: Clean up the token type

diff --git a/src/token.c b/src/token.c
index c1a515d2a99e5bcae176511d9c3eb73b747f5470..2c2ebb2a8f28cc6fac1d7943bb59ad9c97471249 100644 (file)
--- a/src/token.c
+++ b/src/token.c
@@ -5,7 +5,7 @@
 #include "token.h"
 #include "str.h"
 
-struct token *token_new(void)
+struct token *token_new(const token_type_t type, const int line, const int col)
 {
        struct token *tok;
 
@@ -13,30 +13,33 @@ struct token *token_new(void)
 
        if(tok) {
                memset(tok, 0, sizeof(*tok));
+
+               tok->type = type;
+               tok->line = line;
+               tok->column = col;
        }
 
        return(tok);
 }
 
-struct token *token_new2(const int line, const int col, const char *val)
+struct token *token_new_from_str(const token_type_t type, const int line, const int col, const char *val)
 {
        struct token *tok;
 
-       tok = token_new();
+       tok = token_new(type, line, col);
 
        if(tok) {
-               token_setpos(tok, line, col);
                assert(token_setvalue(tok, val) == 0);
        }
 
        return(tok);
 }
 
-struct token *token_new_from_char(const int line, const int col, const char chr)
+struct token *token_new_from_char(const token_type_t type, const int line, const int col, const char chr)
 {
        struct token *tok;
 
-       tok = token_new();
+       tok = token_new(type, line, col);
 
        if(tok) {
                char val[2];
@@ -44,7 +47,6 @@ struct token *token_new_from_char(const int line, const int col, const char chr)
                val[0] = chr;
                val[1] = 0;
 
-               token_setpos(tok, line, col);
                assert(token_setvalue(tok, val) == 0);
        }
 
@@ -57,14 +59,24 @@ void token_free(struct token *tok)
        return;
 }
 
-void token_setpos(struct token *tok, int line, int col)
+int token_setvalue(struct token *tok, const char *str)
 {
-       tok->line = line;
-       tok->column = col;
-       return;
+       int ret_val;
+
+       if(!tok->value) {
+               tok->value = str_new();
+       }
+
+       if(tok->value) {
+               ret_val = str_set(tok->value, str);
+       } else {
+               ret_val = -ENOMEM;
+       }
+
+       return(ret_val);
 }
 
-int token_setvalue(struct token *tok, const char *str)
+int token_append_str(struct token *tok, const char *str)
 {
        int ret_val;
 
@@ -73,7 +85,24 @@ int token_setvalue(struct token *tok, const char *str)
        }
 
        if(tok->value) {
-               ret_val = str_set(tok->value, str);
+               ret_val = str_appends(tok->value, str);
+       } else {
+               ret_val = -ENOMEM;
+       }
+
+       return(ret_val);
+}
+
+int token_append_char(struct token *tok, const char chr)
+{
+       int ret_val;
+
+       if(!tok->value) {
+               tok->value = str_new();
+       }
+
+       if(tok->value) {
+               ret_val = str_appendc(tok->value, chr);
        } else {
                ret_val = -ENOMEM;
        }

diff --git a/src/token.h b/src/token.h
index afd6a891d458821a104470762521d0e31ac9b586..c353dbb88d8b426c6ab0c6fc19d21350096ea60b 100644 (file)
--- a/src/token.h
+++ b/src/token.h
@@ -3,18 +3,73 @@
 
 #include "str.h"
 
+typedef enum {
+       TOKEN_INVALID = 0,
+       TOKEN_ADD,
+       TOKEN_AND,
+       TOKEN_ASSIGN,
+       TOKEN_ASSIGN_ADD,
+       TOKEN_ASSIGN_AND,
+       TOKEN_ASSIGN_MOD,
+       TOKEN_ASSIGN_MUL,
+       TOKEN_ASSIGN_OR,
+       TOKEN_ASSIGN_SHL,
+       TOKEN_ASSIGN_SHR,
+       TOKEN_ASSIGN_SUB,
+       TOKEN_ASSIGN_XOR,
+       TOKEN_CHAR,
+       TOKEN_COLON,
+       TOKEN_COMMA,
+       TOKEN_COMMENT,
+       TOKEN_COMPLEMENT,
+       TOKEN_DEC,
+       TOKEN_DIV,
+       TOKEN_DIV_ASSIGN,
+       TOKEN_DMEMBER,
+       TOKEN_EQ,
+       TOKEN_GE,
+       TOKEN_GT,
+       TOKEN_IDENTIFIER,
+       TOKEN_INC,
+       TOKEN_LAND,
+       TOKEN_LBRACE,
+       TOKEN_LBRACKET,
+       TOKEN_LE,
+       TOKEN_LOR,
+       TOKEN_LPAREN,
+       TOKEN_LT,
+       TOKEN_MEMBER,
+       TOKEN_MOD,
+       TOKEN_MUL,
+       TOKEN_NEQ,
+       TOKEN_NOT,
+       TOKEN_NUMBER,
+       TOKEN_OR,
+       TOKEN_QMARK,
+       TOKEN_RBRACE,
+       TOKEN_RBRACKET,
+       TOKEN_RPAREN,
+       TOKEN_SEMICOLON,
+       TOKEN_SHL,
+       TOKEN_SHR,
+       TOKEN_STRING,
+       TOKEN_SUB
+} token_type_t;
+
 struct token {
-       int line;
-       int column;
-       str_t *value;
-       size_t len;
+       int          line;
+       int          column;
+       token_type_t type;
+       str_t        *value;
+       size_t       len;
 };
 
-struct token *token_new2(const int, const int, const char*);
-struct token *token_new(void);
-struct token *token_new_from_char(const int, const int, const char);
+struct token *token_new_from_str(const token_type_t, const int, const int, const char*);
+struct token *token_new_from_char(const token_type_t, const int, const int, const char);
+
 void token_free(struct token*);
-void token_setpos(struct token*, const int, const int);
 int token_setvalue(struct token*, const char*);
+int token_append_str(struct token*, const char*);
+int token_append_char(struct token*, const char);
 
 #endif /* TOKEN_H */