Compiler/lexical analyzer: Difference between revisions

| <tt>OP_MULTIPLY</tt> || multiply || <tt>*</tt>

| <tt>OP_DIVIDE</tt> || divide || <tt>/</tt>

| <tt>OP_MOD</tt> || mod || <tt>%</tt>

| <tt>OP_ADD</tt> || plus || <tt>+</tt>

| <tt>OP_SUBTRACT</tt> || minus || <tt>-</tt>

| <tt>OP_NEGATE</tt> || unary minus || <tt>-</tt>

| <tt>OP_LESS</tt> || less than || <tt><</tt>

| <tt>OP_LESSEQUAL</tt> || less than or equal || <tt><=</tt>

| <tt>OP_GREATER</tt> || greater than || <tt>&gt;</tt>

| <tt>OP_GREATEREQUAL</tt> || greater than or equal || <tt>&gt;=</tt>

| <tt>OP_EQUAL</tt> || equal || <tt>==</tt>

| <tt>OP_NOTEQUAL</tt> || not equal || <tt>&#33;=</tt>

| <tt>OP_NOT</tt> || unary not || <tt>&#33;</tt>

| <tt>OP_ASSIGN</tt> || assignment || <tt>=</tt>

| <tt>OP_AND</tt> || logical and || <tt>&amp;&amp;</tt>

| <tt>OP_OR</tt> || logical or || <tt>&brvbar;&brvbar;</tt>

* The <code>-</code> token should always be interpreted as <tt>OP_SUBTRACT</tt> by the lexer. Turning some <tt>OP_SUBTRACT</tt> into <tt>OP_NEGATE</tt> will be the job of the syntax analyzer, which is not part of this task.

| <tt>LEFTPAREN</tt> || left parenthesis || <tt>(</tt>

| <tt>RIGHTPAREN</tt> || right parenthesis || <tt>)</tt>

| <tt>LEFTBRACE</tt> || left brace || <tt>{</tt>

| <tt>RIGHTBRACE</tt> || right brace || <tt>}</tt>

| <tt>SEMICOLON</tt> || semi-colon || <tt>&#59;</tt>

| <tt>COMMA</tt> || comma || <tt>,</tt>

| <tt>KEYWORD_IF</tt> || <tt>if</tt>

| <tt>KEYWORD_ELSE</tt> || <tt>else</tt>

| <tt>KEYWORD_WHILE</tt> || <tt>while</tt>

| <tt>KEYWORD_PRINT</tt> || <tt>print</tt>

| <tt>KEYWORD_PUTC</tt> || <tt>putc</tt>

| <tt>IDENTIFIER</tt>

| <tt>INTEGER</tt>

| <tt>INTEGER</tt>

| <tt>STRING</tt>

| <tt>END_OF_INPUT</tt> || when the end of the input stream is reached

END_OF_INPUT OP_MULTIPLY OP_DIVIDE OP_MOD OP_ADD OP_SUBTRACT

OP_NEGATE OP_LESS OP_LESSEQUAL OP_GREATER OP_GREATEREQUAL OP_EQUAL

OP_NOTEQUAL OP_ASSIGN OP_AND OP_OR KEYWORD_IF KEYWORD_ELSE

KEYWORD_WHILE KEYWORD_PRINT KEYWORD_PUTC LEFTPAREN RIGHTPAREN LEFTBRACE

RIGHTBRACE SEMICOLON COMMA IDENTIFIER INTEGER STRING

# the token value (only for <tt>IDENTIFIER</tt>, <tt>INTEGER</tt>, and <tt>STRING</tt> tokens)

4 1 KEYWORD_PRINT

4 6 LEFTPAREN

4 7 STRING "Hello, World!\n"

4 24 RIGHTPAREN

4 25 SEMICOLON

5 1 END_OF_INPUT

4 1 IDENTIFIER phoenix_number

4 16 OP_ASSIGN

4 18 INTEGER 142857

4 24 SEMICOLON

5 1 KEYWORD_PRINT

5 6 LEFTPAREN

5 7 IDENTIFIER phoenix_number

5 21 COMMA

5 23 STRING "\n"

5 27 RIGHTPAREN

5 28 SEMICOLON

6 1 END_OF_INPUT

5 16 KEYWORD_PRINT

5 40 OP_SUBTRACT

6 16 KEYWORD_PUTC

6 40 OP_LESS

7 16 KEYWORD_IF

7 40 OP_GREATER

8 16 KEYWORD_ELSE

8 40 OP_LESSEQUAL

9 16 KEYWORD_WHILE

9 40 OP_GREATEREQUAL

10 16 LEFTBRACE

10 40 OP_EQUAL

11 16 RIGHTBRACE

11 40 OP_NOTEQUAL

12 16 LEFTPAREN

12 40 OP_AND

13 16 RIGHTPAREN

13 40 OP_OR

14 16 OP_SUBTRACT

14 40 SEMICOLON

15 16 OP_NOT

15 40 COMMA

16 16 OP_MULTIPLY

16 40 OP_ASSIGN

17 16 OP_DIVIDE

17 40 INTEGER 42

18 16 OP_MOD

18 40 STRING "String literal"

19 16 OP_ADD

19 40 IDENTIFIER variable_name

20 26 INTEGER 10

21 26 INTEGER 92

22 26 INTEGER 32

23 1 END_OF_INPUT

&"END_OF_INPUT OP_MULTIPLY OP_DIVIDE OP_MOD OP_ADD "

"OP_SUBTRACT OP_NEGATE OP_NOT OP_LESS OP_LESSEQUAL "

"OP_GREATER OP_GREATEREQUAL OP_EQUAL OP_NOTEQUAL OP_ASSIGN "

"OP_AND OP_OR KEYWORD_IF KEYWORD_ELSE KEYWORD_WHILE "

"KEYWORD_PRINT KEYWORD_PUTC LEFTPAREN RIGHTPAREN LEFTBRACE "

"RIGHTBRACE SEMICOLON COMMA IDENTIFIER INTEGER "

"STRING "

5 16 KEYWORD_PRINT

5 40 OP_SUBTRACT

6 16 KEYWORD_PUTC

6 40 OP_LESS

7 16 KEYWORD_IF

7 40 OP_GREATER

8 16 KEYWORD_ELSE

8 40 OP_LESSEQUAL

9 16 KEYWORD_WHILE

9 40 OP_GREATEREQUAL

10 16 LEFTBRACE

10 40 OP_EQUAL

11 16 RIGHTBRACE

11 40 OP_NOTEQUAL

12 16 LEFTPAREN

12 40 OP_AND

13 16 RIGHTPAREN

13 40 OP_OR

14 16 OP_SUBTRACT

14 40 SEMICOLON

15 16 OP_NOT

15 40 COMMA

16 16 OP_MULTIPLY

16 40 OP_ASSIGN

17 16 OP_DIVIDE

17 40 INTEGER 42

18 16 OP_MOD

18 40 STRING "String literal"

19 16 OP_ADD

19 40 IDENTIFIER variable_name

20 26 INTEGER 10

21 26 INTEGER 92

22 26 INTEGER 32

23 1 END_OF_INPUT

constant all_syms = {"END_OF_INPUT", "OP_MULTIPLY", "OP_DIVIDE", "OP_MOD", "OP_ADD",

"OP_SUBTRACT", "OP_NEGATE", "OP_NOT", "OP_LESS", "OP_LESSEQUAL", "OP_GREATER",

"OP_GREATEREQUAL", "OP_EQUAL", "OP_NOTEQUAL", "OP_ASSIGN", "OP_AND", "OP_OR",

"KEYWORD_IF", "KEYWORD_ELSE", "KEYWORD_WHILE", "KEYWORD_PRINT", "KEYWORD_PUTC",

"LEFTPAREN", "RIGHTPAREN", "LEFTBRACE", "RIGHTBRACE", "SEMICOLON", "COMMA",

"IDENTIFIER", "INTEGER", "STRING"}

5 16 KEYWORD_PRINT

5 40 OP_SUBTRACT

6 16 KEYWORD_PUTC

6 40 OP_LESS

7 16 KEYWORD_IF

7 40 OP_GREATER

8 16 KEYWORD_ELSE

8 40 OP_LESSEQUAL

9 16 KEYWORD_WHILE

9 40 OP_GREATEREQUAL

10 16 LEFTBRACE

10 40 OP_EQUAL

11 16 RIGHTBRACE

11 40 OP_NOTEQUAL

12 16 LEFTPAREN

12 40 OP_AND

13 16 RIGHTPAREN

13 40 OP_OR

14 16 OP_SUBTRACT

14 40 SEMICOLON

15 16 OP_NOT

15 40 COMMA

16 16 OP_MULTIPLY

16 40 OP_ASSIGN

17 16 OP_DIVIDE

17 40 INTEGER 42

18 16 OP_MOD

18 40 STRING "String literal"

19 16 OP_ADD

19 40 IDENTIFIER variable_name

20 26 INTEGER 10

21 26 INTEGER 92

22 26 INTEGER 32

23 1 END_OF_INPUT

&"END_OF_INPUT OP_MULTIPLY OP_DIVIDE OP_MOD OP_ADD "

"OP_SUBTRACT OP_NEGATE OP_NOT OP_LESS OP_LESSEQUAL "

"OP_GREATER OP_GREATEREQUAL OP_EQUAL OP_NOTEQUAL OP_ASSIGN "

"OP_AND OP_OR KEYWORD_IF KEYWORD_ELSE KEYWORD_WHILE "

"KEYWORD_PRINT KEYWORD_PUTC LEFTPAREN RIGHTPAREN LEFTBRACE "

"RIGHTBRACE SEMICOLON COMMA IDENTIFIER INTEGER "

"STRING "

5 16 KEYWORD_PRINT

5 40 OP_SUBTRACT

6 16 KEYWORD_PUTC

6 40 OP_LESS

7 16 KEYWORD_IF

7 40 OP_GREATER

8 16 KEYWORD_ELSE

8 40 OP_LESSEQUAL

9 16 KEYWORD_WHILE

9 40 OP_GREATEREQUAL

10 16 LEFTBRACE

10 40 OP_EQUAL

11 16 RIGHTBRACE

11 40 OP_NOTEQUAL

12 16 LEFTPAREN

12 40 OP_AND

13 16 RIGHTPAREN

13 40 OP_OR

14 16 OP_SUBTRACT

14 40 SEMICOLON

15 16 OP_NOT

15 40 COMMA

16 16 OP_MULTIPLY

16 40 OP_ASSIGN

17 16 OP_DIVIDE

17 40 INTEGER 42

18 16 OP_MOD

18 40 STRING "String literal"

19 16 OP_ADD

19 40 IDENTIFIER variable_name

20 26 INTEGER 10

21 26 INTEGER 92

22 26 INTEGER 32

22 29 END_OF_INPUT

tok_list(tk_EOI ) = "END_OF_INPUT"

tok_list(tk_Mul ) = "OP_MULTIPLY"

tok_list(tk_Div ) = "OP_DIVIDE"

tok_list(tk_Mod ) = "OP_MOD"

tok_list(tk_Add ) = "OP_ADD"

tok_list(tk_Sub ) = "OP_SUBTRACT"

tok_list(tk_Negate ) = "OP_NEGATE"

tok_list(tk_Not ) = "OP_NOT"

tok_list(tk_Lss ) = "OP_LESS"

tok_list(tk_Leq ) = "OP_LESSEQUAL"

tok_list(tk_Gtr ) = "OP_GREATER"

tok_list(tk_Geq ) = "OP_GREATEREQUAL"

tok_list(tk_Eq ) = "OP_EQUAL"

tok_list(tk_Neq ) = "OP_NOTEQUAL"

tok_list(tk_Assign ) = "OP_ASSIGN"

tok_list(tk_And ) = "OP_AND"

tok_list(tk_Or ) = "OP_OR"

tok_list(tk_If ) = "KEYWORD_IF"

tok_list(tk_Else ) = "KEYWORD_ELSE"

tok_list(tk_While ) = "KEYWORD_WHILE"

tok_list(tk_Print ) = "KEYWORD_PRINT"

tok_list(tk_Putc ) = "KEYWORD_PUTC"

tok_list(tk_Lparen ) = "LEFTPAREN"

tok_list(tk_Rparen ) = "RIGHTPAREN"

tok_list(tk_Lbrace ) = "LEFTBRACE"

tok_list(tk_Rbrace ) = "RIGHTBRACE"

tok_list(tk_Semi ) = "SEMICOLON"

tok_list(tk_Comma ) = "COMMA"

tok_list(tk_Ident ) = "IDENTIFIER"

tok_list(tk_Integer) = "INTEGER"

tok_list(tk_String ) = "STRING"

5 16 KEYWORD_PRINT

5 40 OP_SUBTRACT

6 16 KEYWORD_PUTC

6 40 OP_LESS

7 16 KEYWORD_IF

7 40 OP_GREATER

8 16 KEYWORD_ELSE

8 40 OP_LESSEQUAL

9 16 KEYWORD_WHILE

9 40 OP_GREATEREQUAL

10 16 LEFTBRACE

10 40 OP_EQUAL

11 16 RIGHTBRACE

11 40 OP_NOTEQUAL

12 16 LEFTPAREN

12 40 OP_AND

13 16 RIGHTPAREN

13 40 OP_OR

14 16 OP_SUBTRACT

14 40 SEMICOLON

15 16 OP_NOT

15 40 COMMA

16 16 OP_MULTIPLY

16 40 OP_ASSIGN

17 16 OP_DIVIDE

17 40 INTEGER 42

18 16 OP_MOD

18 40 STRING "String literal"

19 16 OP_ADD

19 40 IDENTIFIER variable_name

20 26 INTEGER 10

21 26 INTEGER 92

22 26 INTEGER 32

22 30 END_OF_INPUT

['OP_MULTIPLY' , '*' , ],

['OP_DIVIDE' , '/' , ],

['OP_MOD' , '%' , ],

['OP_ADD' , '+' , ],

['OP_SUBTRACT' , '-' , ],

['OP_LESSEQUAL' , '<=' , ],

['OP_LESS' , '<' , ],

['OP_GREATEREQUAL', '>=' , ],

['OP_GREATER' , '>' , ],

['OP_EQUAL' , '==' , ],

['OP_ASSIGN' , '=' , ],

['OP_NOT' , '!' , ],

['OP_NOTEQUAL' , '!=' , ],

['OP_AND' , '&&' , ],

['OP_OR' , '||' , ],

['KEYWORD_ELSE' , qr/else\b/ , ],

['KEYWORD_IF' , qr/if\b/ , ],

['KEYWORD_WHILE' , qr/while\b/ , ],

['KEYWORD_PRINT' , qr/print\b/ , ],

['KEYWORD_PUTC' , qr/putc\b/ , ],

['LEFTPAREN' , '(' , ],

['RIGHTPAREN' , ')' , ],

['LEFTBRACE' , '{' , ],

['RIGHTBRACE' , '}' , ],

['SEMICOLON' , ';' , ],

['COMMA' , ',' , ],

['IDENTIFIER' , qr/[_a-z][_a-z0-9]*/i, \&raw ],

['INTEGER' , qr/[0-9]+\b/ , \&raw ],

['INTEGER' , qr/'([^']*)(')?/ , \&char_val ],

['STRING' , qr/"([^"]*)(")?/ , \&string_raw],

['END_OF_INPUT' , qr/$/ , ],

5 16 KEYWORD_PRINT

5 40 OP_SUBTRACT

6 16 KEYWORD_PUTC

6 40 OP_LESS

7 16 KEYWORD_IF

7 40 OP_GREATER

8 16 KEYWORD_ELSE

8 40 OP_LESSEQUAL

9 16 KEYWORD_WHILE

9 40 OP_GREATEREQUAL

10 16 LEFTBRACE

10 40 OP_EQUAL

11 16 RIGHTBRACE

11 40 OP_NOT

11 41 OP_ASSIGN

12 16 LEFTPAREN

12 40 OP_AND

13 16 RIGHTPAREN

13 40 OP_OR

14 16 OP_SUBTRACT

14 40 SEMICOLON

15 16 OP_NOT

15 40 COMMA

16 16 OP_MULTIPLY

16 40 OP_ASSIGN

17 16 OP_DIVIDE

17 40 INTEGER 42

18 16 OP_MOD

18 40 STRING "String literal"

19 16 OP_ADD

19 40 IDENTIFIER variable_name

20 26 INTEGER 10

21 26 INTEGER 92

22 26 INTEGER 32

23 1 END_OF_INPUT

token operator:sym<*> { '*' { make 'OP_MULTIPLY' } }

token operator:sym</> { '/'<!before '*'> { make 'OP_DIVIDE' } }

token operator:sym<%> { '%' { make 'OP_MOD' } }

token operator:sym<+> { '+' { make 'OP_ADD' } }

token operator:sym<-> { '-' { make 'OP_SUBTRACT' } }

token operator:sym('<='){ '<=' { make 'OP_LESSEQUAL' } }

token operator:sym('<') { '<' { make 'OP_LESS' } }

token operator:sym('>='){ '>=' { make 'OP_GREATEREQUAL'} }

token operator:sym('>') { '>' { make 'OP_GREATER' } }

token operator:sym<==> { '==' { make 'OP_EQUAL' } }

token operator:sym<!=> { '!=' { make 'OP_NOTEQUAL' } }

token operator:sym<!> { '!' { make 'OP_NOT' } }

token operator:sym<=> { '=' { make 'OP_ASSIGN' } }

token operator:sym<&&> { '&&' { make 'OP_AND' } }

token operator:sym<||> { '||' { make 'OP_OR' } }

token keyword:sym<if> { 'if' { make 'KEYWORD_IF' } }

token keyword:sym<else> { 'else' { make 'KEYWORD_ELSE' } }

token keyword:sym<putc> { 'putc' { make 'KEYWORD_PUTC' } }

token keyword:sym<while> { 'while' { make 'KEYWORD_WHILE' } }

token keyword:sym<print> { 'print' { make 'KEYWORD_PRINT' } }

token symbol:sym<(> { '(' { make 'LEFT_PAREN' } }

token symbol:sym<)> { ')' { make 'RIGHT_PAREN' } }

token symbol:sym<{> { '{' { make 'LEFT_BRACE' } }

token symbol:sym<}> { '}' { make 'RIGHT_BRACE' } }

token symbol:sym<;> { ';' { make 'SEMICOLON' } }

token symbol:sym<,> { ',' { make 'COMMA' } }

token integer { <[0..9]>+ { make 'INTEGER ' ~ $/ } }

make 'STRING ' ~ $/;

token eoi { $ { make 'END_OF_INPUT' } }

my @pos = gather for $c_code.lines».chars.kv -> $line, $v {

5 16 KEYWORD_PRINT

5 40 OP_SUBTRACT

6 16 KEYWORD_PUTC

6 40 OP_LESS

7 16 KEYWORD_IF

7 40 OP_GREATER

8 16 KEYWORD_ELSE

8 40 OP_LESSEQUAL

9 16 KEYWORD_WHILE

9 40 OP_GREATEREQUAL

10 16 LEFT_BRACE

10 40 OP_EQUAL

11 16 RIGHT_BRACE

11 40 OP_NOTEQUAL

12 16 LEFT_PAREN

12 40 OP_AND

13 16 RIGHT_PAREN

13 40 OP_OR

14 16 OP_SUBTRACT

14 40 SEMICOLON

15 16 OP_NOT

15 40 COMMA

16 16 OP_MULTIPLY

16 40 OP_ASSIGN

17 16 OP_DIVIDE

17 40 INTEGER 42

18 16 OP_MOD

18 40 STRING "String literal"

19 16 OP_ADD

23 1 END_OF_INPUT

all_syms = ["END_OF_INPUT", "OP_MULTIPLY", "OP_DIVIDE", "OP_MOD", "OP_ADD", "OP_SUBTRACT",

"OP_NEGATE", "OP_NOT", "OP_LESS", "OP_LESSEQUAL", "OP_GREATER", "OP_GREATEREQUAL",

"OP_EQUAL", "OP_NOTEQUAL", "OP_ASSIGN", "OP_AND", "OP_OR", "KEYWORD_IF",

"KEYWORD_ELSE", "KEYWORD_WHILE", "KEYWORD_PRINT", "KEYWORD_PUTC", "LEFTPAREN",

"RIGHTPAREN", "LEFTBRACE", "RIGHTBRACE", "SEMICOLON", "COMMA", "IDENTIFIER",

"INTEGER", "STRING"]

5 16 KEYWORD_PRINT

5 40 OP_SUBTRACT

6 16 KEYWORD_PUTC

6 40 OP_LESS

7 16 KEYWORD_IF

7 40 OP_GREATER

8 16 KEYWORD_ELSE

8 40 OP_LESSEQUAL

9 16 KEYWORD_WHILE

9 40 OP_GREATEREQUAL

10 16 LEFTBRACE

10 40 OP_EQUAL

11 16 RIGHTBRACE

11 40 OP_NOTEQUAL

12 16 LEFTPAREN

12 40 OP_AND

13 16 RIGHTPAREN

13 40 OP_OR

14 16 OP_SUBTRACT

14 40 SEMICOLON

15 16 OP_NOT

15 40 COMMA

16 16 OP_MULTIPLY

16 40 OP_ASSIGN

17 16 OP_DIVIDE

17 40 INTEGER 42

18 16 OP_MOD

18 40 STRING "String literal"

19 16 OP_ADD

19 40 IDENTIFIER variable_name

20 26 INTEGER 10

21 26 INTEGER 92

22 26 INTEGER 32

23 1 END_OF_INPUT