doc: Add graph of the tokenizer automaton

diff --git a/doc/tokenizer.dot b/doc/tokenizer.dot
new file mode 100644 (file)
index 0000000..6b16619
--- /dev/null
+++ b/doc/tokenizer.dot
@@ -0,0 +1,152 @@
+digraph tokenizer {
+ none    [label = "none"];
+
+ div     [label = "/"];
+ comment [label = "/*"];
+ cesc    [label = "/* ... *"];
+ fdiv    [label = "/="];
+
+ lt      [label = "<"];
+ gt      [label = ">"];
+ le      [label = "<="];
+ ge      [label = ">="];
+ shl     [label = "<<"];
+ shr     [label = ">>"];
+ fshl    [label = "<<="];
+ fshr    [label = ">>="];
+ 
+ not     [label = "!"];
+ neq     [label = "!="];
+ mod     [label = "%"];
+ fmod    [label = "%="];
+ and     [label = "&"];
+ land    [label = "&&"];
+ or      [label = "|"];
+ lor     [label = "||"];
+ for     [label = "|="];
+ xor     [label = "^"];
+ fxor    [label = "^="];
+ assign  [label = "="];
+ eq      [label = "=="];
+ 
+ str     [label = "\"" ];
+ chr     [label = "'"];
+ chresc  [label = "'\\"];
+ stresc  [label = "\" ... \\"];
+
+ add     [label = "+"];
+ sub     [label = "-"];
+ mul     [label = "*"];
+ fadd    [label = "+="];
+ fsub    [label = "-="];
+ fmul    [label = "*="];
+ inc     [label = "++"];
+ dec     [label = "--"];
+
+ dot     [label = "."];
+ arr     [label = "->"];
+ 
+ compl   [label = "~"];
+ lparen  [label = "("];
+ rparen  [label = ")"];
+ lbrace  [label = "{"];
+ rbrace  [label = "}"];
+ lidx    [label = "["];
+ ridx    [label = "]"];
+ colon   [label = ":"];
+ scolon  [label = ";"];
+ comma   [label = ","];
+ qmark   [label = "?"];
+
+ zero    [label = "0"];
+ flt     [label = "float"];
+ oct     [label = "OCT"];
+ hex     [label = "HEX"];
+ decimal [label = "DEC"];
+ none -> div    [label = "/"];
+ div -> comment [label = "*"];
+ div -> fdiv    [label = "="];
+ comment -> cesc [label = "*"];
+ cesc -> comment [label = "[^/]"];
+
+ none -> lt       [label = "<"];
+ lt -> le         [label = "="];
+ lt -> shl        [label = "<"];
+ shl -> fshl      [label = "="];
+ none -> gt       [label = ">"];
+ gt -> ge         [label = "="];
+ gt -> shr        [label = ">"];
+ shr -> fshr      [label = "="];
+
+ not     [label = "!"];
+ neq     [label = "!="];
+ mod     [label = "%"];
+ fmod    [label = "%="];
+ and     [label = "&"];
+ land    [label = "&&"];
+ fand    [label = "&="];
+ or      [label = "|"];
+ lor     [label = "||"];
+ for     [label = "|="];
+ xor     [label = "^"];
+ fxor    [label = "^="];
+ assign  [label = "="];
+ eq      [label = "=="];
+
+ none -> not      [label = "!"];
+ not -> neq       [label = "="];
+ none -> mod      [label = "%"];
+ mod -> fmod      [label = "="];
+ none -> and      [label = "&"];
+ and -> land      [label = "&"];
+ and -> fand      [label = "="];
+ none -> or       [label = "|"];
+ or -> for        [label = "="];
+ or -> lor        [label = "|"];
+ none -> xor      [label = "^"];
+ xor -> fxor      [label = "="];
+ none -> assign   [label = "="];
+ assign -> eq     [label = "="];
+ 
+ none -> compl    [label = "~"];
+ none -> lparen   [label = "("];
+ none -> rparen   [label = ")"];
+ none -> lbrace   [label = "{"];
+ none -> rbrace   [label = "}"];
+ none -> lidx     [label = "["];
+ none -> ridx     [label = "]"];
+ none -> colon    [label = ":"];
+ none -> scolon   [label = ";"];
+ none -> comma    [label = ","];
+ none -> qmark    [label = "?"];
+
+ none -> add [label = "+"];
+ add -> fadd [label = "="];
+ add -> inc  [label = "+"];
+ none -> sub [label = "-"];
+ sub -> fsub [label = "="];
+ sub -> dec  [label = "-"];
+ sub -> arr  [label = ">"];
+ none -> mul [label = "*"];
+ mul -> fmul [label = "="];
+
+ none -> str [label = "\""];
+ none -> chr [label = "'"];
+ chr -> chresc [label = "\"];
+ chresc -> chr;
+ none -> dot [label = "."];
+
+ none -> zero [label = "0"];
+ zero -> hex [label = "x,X"];
+ hex -> hex [label = "[0-9A-Fa-f]"];
+ zero -> oct [label = "[0-7]"];
+ oct -> oct [label = "[0-7]"];
+ none -> decimal [label = "[1-9]"];
+ decimal -> decimal [label = "[0-9]"];
+
+ none -> flt [label = "."];
+ zero -> flt [label = "."];
+ decimal -> flt  [label = "."];
+ dot -> flt  [label = "[0-9]"]; 
+ flt -> flt [label = "[0-9]"]; 
+}