From: Matthias Kruk <m@m10k.eu>
Date: Sun, 31 May 2020 05:53:29 +0000 (+0900)
Subject: lex: Assign tokens a type during the lexing step
X-Git-Url: https://git.corax.cc/?a=commitdiff_plain;h=7128f87b9629c31b10dac583490f65c0eda80fb3;p=ccc

lex: Assign tokens a type during the lexing step
---

diff --git a/src/lex.c b/src/lex.c
index 6175fd6..6555098 100644
--- a/src/lex.c
+++ b/src/lex.c
@@ -185,17 +185,37 @@ struct token *lex_gettoken(void)
 				break;
 
 			case '~':
+				return(token_new_from_char(TOKEN_COMPLEMENT, _line, _col, c));
+
 			case '(':
+				return(token_new_from_char(TOKEN_LPAREN, _line, _col, c));
+
 			case ')':
+				return(token_new_from_char(TOKEN_RPAREN, _line, _col, c));
+
 			case '{':
+				return(token_new_from_char(TOKEN_LBRACE, _line, _col, c));
+
 			case '}':
+				return(token_new_from_char(TOKEN_RBRACE, _line, _col, c));
+
 			case '[':
+				return(token_new_from_char(TOKEN_LBRACKET, _line, _col, c));
+
 			case ']':
+				return(token_new_from_char(TOKEN_RBRACKET, _line, _col, c));
+
 			case ':':
+				return(token_new_from_char(TOKEN_COLON, _line, _col, c));
+
 			case ';':
+				return(token_new_from_char(TOKEN_SEMICOLON, _line, _col, c));
+
 			case ',':
+				return(token_new_from_char(TOKEN_COMMA, _line, _col, c));
+
 			case '?':
-				return(token_new_from_char(_line, _col, c));
+				return(token_new_from_char(TOKEN_QMARK, _line, _col, c));
 
 			case '\r':
 				fprintf(stderr, "DOS user, eh?\n");
@@ -207,17 +227,17 @@ struct token *lex_gettoken(void)
 			default:
 				if(identifier_firstchr(c)) {
 					/* looks like an identifier */
-					tok = token_new_from_char(_line, _col, c);
+					tok = token_new_from_char(TOKEN_IDENTIFIER, _line, _col, c);
 					assert(tok);
 					state = STATE_ID;
 					break;
 				} else if(c == '0') {
-					tok = token_new_from_char(_line, _col, c);
+					tok = token_new_from_char(TOKEN_NUMBER, _line, _col, c);
 					assert(tok);
 					state = STATE_ZERO;
 					break;
 				} else if(c > '0' && c <= '9') {
-					tok = token_new_from_char(_line, _col, c);
+					tok = token_new_from_char(TOKEN_NUMBER, _line, _col, c);
 					assert(tok);
 					state = STATE_NUM;
 					break;
@@ -237,22 +257,22 @@ struct token *lex_gettoken(void)
 				break;
 
 			case '=':
-				return(token_new2(_line, _col, "/="));
+				return(token_new_from_str(TOKEN_DIV_ASSIGN, _line, _col, "/="));
 
 			default:
 				_putnextchar(c);
-				return(token_new2(_line, _col, "/"));
+				return(token_new_from_str(TOKEN_DIV, _line, _col, "/"));
 			}
 
 			break;
 
 		case STATE_COMMENT:
 			if(!tok) {
-				tok = token_new2(_line, _col, "/*");
+				tok = token_new_from_str(TOKEN_COMMENT, _line, _col, "/*");
 				assert(tok);
 			}
 
-			str_appendc(tok->value, c);
+			token_append_char(tok, c);
 
 			if(c == '*') {
 				state = STATE_COMMENT_END;
@@ -261,7 +281,7 @@ struct token *lex_gettoken(void)
 			break;
 
 		case STATE_COMMENT_END:
-			str_appendc(tok->value, c);
+			token_append_char(tok, c);
 
 			if(c == '/') {
 				return(tok);
@@ -277,23 +297,23 @@ struct token *lex_gettoken(void)
 				break;
 
 			case '=':
-				return(token_new2(_line, _col, "<="));
+				return(token_new_from_str(TOKEN_LE, _line, _col, "<="));
 				break;
 
 			default:
 				_putnextchar(c);
-				return(token_new2(_line, _col, "<"));
+				return(token_new_from_str(TOKEN_LT, _line, _col, "<"));
 			}
 
 			break;
 
 		case STATE_SHL:
 			if(c == '=') {
-				return(token_new2(_line, _col, "<<="));
+				return(token_new_from_str(TOKEN_ASSIGN_SHL, _line, _col, "<<="));
 			}
 
 			_putnextchar(c);
-			return(token_new2(_line, _col, "<<"));
+			return(token_new_from_str(TOKEN_SHL, _line, _col, "<<"));
 
 		case STATE_GT:
 			switch(c) {
@@ -302,88 +322,88 @@ struct token *lex_gettoken(void)
 				break;
 
 			case '=':
-				return(token_new2(_line, _col, ">="));
+				return(token_new_from_str(TOKEN_GE, _line, _col, ">="));
 
 			default:
 				_putnextchar(c);
-				return(token_new2(_line, _col, ">"));
+				return(token_new_from_str(TOKEN_GT, _line, _col, ">"));
 			}
 
 			break;
 
 		case STATE_SHR:
 			if(c == '=') {
-				return(token_new2(_line, _col, ">>="));
+				return(token_new_from_str(TOKEN_ASSIGN_SHR, _line, _col, ">>="));
 			}
 
 			_putnextchar(c);
-			return(token_new2(_line, _col, ">>"));
+			return(token_new_from_str(TOKEN_SHR, _line, _col, ">>"));
 
 		case STATE_NOT:
 			if(c == '=') {
-				return(token_new2(_line, _col, "!="));
+				return(token_new_from_str(TOKEN_NEQ, _line, _col, "!="));
 			}
 
 			_putnextchar(c);
-			return(token_new2(_line, _col, "!"));
+			return(token_new_from_str(TOKEN_NOT, _line, _col, "!"));
 
 		case STATE_MOD:
 			if(c == '=') {
-				return(token_new2(_line, _col, "%="));
+				return(token_new_from_str(TOKEN_ASSIGN_MOD, _line, _col, "%="));
 			}
 
 			_putnextchar(c);
-			return(token_new2(_line, _col, "%"));
+			return(token_new_from_str(TOKEN_MOD, _line, _col, "%"));
 
 		case STATE_AND:
 			switch(c) {
 			case '&':
-				return(token_new2(_line, _col, "&&"));
+				return(token_new_from_str(TOKEN_LAND, _line, _col, "&&"));
 
 			case '=':
-				return(token_new2(_line, _col, "&="));
+				return(token_new_from_str(TOKEN_ASSIGN_AND, _line, _col, "&="));
 
 			default:
 				_putnextchar(c);
-				return(token_new2(_line, _col, "&"));
+				return(token_new_from_str(TOKEN_AND, _line, _col, "&"));
 			}
 
 		case STATE_OR:
 			switch(c) {
 			case '|':
-				return(token_new2(_line, _col, "||"));
+				return(token_new_from_str(TOKEN_LOR, _line, _col, "||"));
 
 			case '=':
-				return(token_new2(_line, _col, "|="));
+				return(token_new_from_str(TOKEN_ASSIGN_OR, _line, _col, "|="));
 
 			default:
 				_putnextchar(c);
-				return(token_new2(_line, _col, "|"));
+				return(token_new_from_str(TOKEN_OR, _line, _col, "|"));
 			}
 
 		case STATE_XOR:
 			if(c == '=') {
-				return(token_new2(_line, _col, "^="));
+				return(token_new_from_str(TOKEN_ASSIGN_XOR, _line, _col, "^="));
 			}
 
 			_putnextchar(c);
-			return(token_new2(_line, _col, "^"));
+			return(token_new_from_str(TOKEN_ASSIGN_XOR, _line, _col, "^"));
 
 		case STATE_ASSIGN:
 			if(c == '=') {
-				return(token_new2(_line, _col, "=="));
+				return(token_new_from_str(TOKEN_EQ, _line, _col, "=="));
 			}
 
 			_putnextchar(c);
-			return(token_new2(_line, _col, "="));
+			return(token_new_from_str(TOKEN_ASSIGN, _line, _col, "="));
 
 		case STATE_STRING:
 			if(!tok) {
-				tok = token_new2(_line, _col, "\"");
+				tok = token_new_from_str(TOKEN_STRING, _line, _col, "\"");
 				assert(tok);
 			}
 
-			str_appendc(tok->value, c);
+			token_append_char(tok, c);
 
 			if(c == '\\') {
 				state = STATE_STRING_ESC;
@@ -394,17 +414,17 @@ struct token *lex_gettoken(void)
 			break;
 
 		case STATE_STRING_ESC:
-			str_appendc(tok->value, c);
+			token_append_char(tok, c);
 			state = STATE_STRING;
 			break;
 
 		case STATE_CHR:
 			if(!tok) {
-				tok = token_new2(_line, _col, "'");
+				tok = token_new_from_str(TOKEN_CHAR, _line, _col, "'");
 				assert(tok);
 			}
 
-			str_appendc(tok->value, c);
+			token_append_char(tok, c);
 
 			if(c == '\\') {
 				state = STATE_CHR_ESC;
@@ -415,13 +435,13 @@ struct token *lex_gettoken(void)
 			break;
 
 		case STATE_CHR_ESC:
-			str_appendc(tok->value, c);
+			token_append_char(tok, c);
 			state = STATE_CHR;
 			break;
 
 		case STATE_ID:
 			if(identifier_chr(c)) {
-				str_appendc(tok->value, c);
+				token_append_char(tok, c);
 			} else {
 				_putnextchar(c);
 				return(tok);
@@ -432,40 +452,40 @@ struct token *lex_gettoken(void)
 		case STATE_MUL:
 			switch(c) {
 			case '=':
-				return(token_new2(_line, _col, "*="));
+				return(token_new_from_str(TOKEN_ASSIGN_MUL, _line, _col, "*="));
 
 			default:
 				_putnextchar(c);
-				return(token_new2(_line, _col, "*"));
+				return(token_new_from_str(TOKEN_MUL, _line, _col, "*"));
 			}
 
 		case STATE_ADD:
 			switch(c) {
 			case '+':
-				return(token_new2(_line, _col, "++"));
+				return(token_new_from_str(TOKEN_INC, _line, _col, "++"));
 
 			case '=':
-				return(token_new2(_line, _col, "+="));
+				return(token_new_from_str(TOKEN_ASSIGN_ADD, _line, _col, "+="));
 
 			default:
 				_putnextchar(c);
-				return(token_new2(_line, _col, "+"));
+				return(token_new_from_str(TOKEN_ADD, _line, _col, "+"));
 			}
 
 		case STATE_SUB:
 			switch(c) {
 			case '-':
-				return(token_new2(_line, _col, "--"));
+				return(token_new_from_str(TOKEN_DEC, _line, _col, "--"));
 
 			case '=':
-				return(token_new2(_line, _col, "-="));
+				return(token_new_from_str(TOKEN_ASSIGN_SUB, _line, _col, "-="));
 
 			case '>':
-				return(token_new2(_line, _col, "->"));
+				return(token_new_from_str(TOKEN_DMEMBER, _line, _col, "->"));
 
 			default:
 				_putnextchar(c);
-				return(token_new2(_line, _col, "-"));
+				return(token_new_from_str(TOKEN_SUB, _line, _col, "-"));
 			}
 
 		case STATE_DOT:
@@ -474,14 +494,14 @@ struct token *lex_gettoken(void)
 			if(c >= '0' && c <= '9') {
 				state = STATE_NUM;
 			} else {
-				return(token_new2(_line, _col, "."));
+				return(token_new_from_str(TOKEN_MEMBER, _line, _col, "."));
 			}
 
 			break;
 
 		case STATE_ZERO:
 			if(c == 'x' || c == 'X') {
-				str_appendc(tok->value, 'x');
+				token_append_char(tok, 'x');
 				state = STATE_NUM;
 			} else if(c >= '0' && c <= '9') {
 				_putnextchar(c);
@@ -498,7 +518,7 @@ struct token *lex_gettoken(void)
 		case STATE_NUM:
 			/* FIXME: e, E may be in the middle, (u|U)(l|L|ll|LL) or vice versa may be at the end */
 			if(c >= '0' && c <= '9' || c == '.') {
-				str_appendc(tok->value, c);
+				token_append_char(tok, c);
 			} else {
 				_putnextchar(c);
 				return(tok);