Compiler/lexical analyzer: Difference between revisions

! Name !! Common name !! Character sequence

|-

|-

|-

|-

|-

|-

|-

|-

|-

|-

|-

|}

|-

| <tt>KEYWORD_IF</tt> || <tt>if</tt>

|-

| <tt>KEYWORD_WHILE</tt> || <tt>while</tt>

<pre>

</pre>

have to wait until syntax analysis

*/

/* character literal */ '\n'

/* character literal */ '\\'

| style="vertical-align:top" |

<b><pre>

</pre></b>

|}

{{task heading|Reference}}

<hr>

#define da_len(name) _qy_ ## name ## _p

} TokenType;

typedef struct {

int err_ln, err_col;

union {

TokenType sym;

} kwds[] = {

{"if", tk_If},

{"print", tk_Print},

case '-': next_ch(); return (tok_s){tk_Sub, err_line, err_col, {0}};

case '*': next_ch(); return (tok_s){tk_Mul, err_line, err_col, {0}};

case ';': next_ch(); return (tok_s){tk_Semi, err_line, err_col, {0}};

case ',': next_ch(); return (tok_s){tk_Comma,err_line, err_col, {0}};

case '/': next_ch(); return div_or_cmt(err_line, err_col);

case '\'': next_ch(); return char_lit(the_ch, err_line, err_col);

case '"' : return string_lit(the_ch, err_line, err_col);

default: return ident_or_int(err_line, err_col);

do {

tok = gettok();

tok.err_ln, tok.err_col,

if (tok.tok == tk_Integer) fprintf(dest_fp, " %4d", tok.n);

else if (tok.tok == tk_Ident) fprintf(dest_fp, " %s", tok.text);

<b>

<pre>

</pre>

</b>

constant true = 1, false = 0, EOF = -1

integer input_file, the_ch = ' ', the_col = 0, the_line = 1

case '/' then return div_or_cmt(err_line, err_col)

case '\'' then return char_lit(err_line, err_col)

case '"' then return string_lit(the_ch, err_line, err_col)

case else

procedure init()

put(key_words, "if", tk_If)

put(key_words, "print", tk_Print)

symbols['-'] = tk_Sub

symbols['*'] = tk_Mul

symbols[';'] = tk_Semi

symbols[','] = tk_Comma

end procedure

printf(STDOUT, "%5d %5d %-8s", {t[2], t[3], all_syms[t[1]]})

switch t[1] do

case else printf(STDOUT, "\n")

end switch

<b>

<pre>

</pre>

</b>

#define NELEMS(arr) (sizeof(arr) / sizeof(arr[0]))

} TokenType;

TokenType sym;

} kwds[] = {

{"if", tk_If},

{"print", tk_Print},

"*" {return tk_Mul;}

"/" {return tk_Div;}

"+" {return tk_Add;}

"-" {return tk_Sub;}

">" {return tk_Gtr;}

"<=" {return tk_Leq;}

"!=" {return tk_Neq;}

"&&" {return tk_And;}

";" {return tk_Semi;}

"," {return tk_Comma;}

"=" {return tk_Assign;}

{ident} {return get_ident_type(yytext);}

do {

tok = yylex();

if (tok == tk_Integer) printf(" %5d", yynval);

<b>

<pre>

</pre>

</b>

tk_Mul

tk_Div

tk_Add

tk_Sub

tk_Lss

tk_Leq

tk_Gtr

tk_Neq

tk_Assign

tk_And

tk_If

tk_While

tk_Print

case "-": tok = tk_sub: next_char(): exit sub

case "*": tok = tk_mul: next_char(): exit sub

case ";": tok = tk_semi: next_char(): exit sub

case ",": tok = tk_comma: next_char(): exit sub

case "/": ' div or comment

next_char()

exit sub

case "<": next_char(): tok = follow(err_line, err_col, "=", tk_Leq, tk_Lss): exit sub

case "&": next_char(): tok = follow(err_line, err_col, "&", tk_And, tk_EOI): exit sub

case DoubleQuote: ' string

v = cur_ch

sub init_lex(byval filein as string)

install("if", tk_if)

install("print", tk_print)

tok_list(tk_Mul ) = "OP_MULTIPLY"

tok_list(tk_Div ) = "OP_DIVIDE"

tok_list(tk_Add ) = "OP_ADD"

tok_list(tk_Sub ) = "OP_SUBTRACT"

tok_list(tk_Lss ) = "OP_LESS"

tok_list(tk_Leq ) = "OP_LESSEQUAL"

tok_list(tk_Gtr ) = "OP_GREATER"

tok_list(tk_Neq ) = "OP_NOTEQUAL"

tok_list(tk_Assign ) = "OP_ASSIGN"

tok_list(tk_And ) = "OP_AND"

tok_list(tk_If ) = "KEYWORD_IF"

tok_list(tk_While ) = "KEYWORD_WHILE"

tok_list(tk_Print ) = "KEYWORD_PRINT"

do

gettok(err_line, err_col, tok, v)

if tok = tk_integer orelse tok = tk_ident orelse tok = tk_string then print " " + v;

print

<b>

<pre>

</pre>

</b>

my @tokens = (

);

my ($line, $col) = $linecol->(substr $input, $pos, $-[0] - $pos);

$pos = $-[0];

# Get the token type that was identified by the scanner regex

my $type = $main::REGMARK;

die "Unrecognized token $1 at line $line, col $col\n" if $type eq '!';

my ($name, $evaluator) = @{$tokens[$type]}[0, 2];

# Get the token value

my $value;

if ($@) { chomp $@; die "$@ in $name at line $line, col $col\n" }

}

# Print the output line

print "$line\t$col\t$name".($value ? "\t$value" : '')."\n";

{{out|case=test case 3}}

<pre>

</pre>

proto token operator {*}

proto token keyword {*}

token keyword:sym<if> { 'if' { make 'KEYWORD_IF' } }

token keyword:sym<putc> { 'putc' { make 'KEYWORD_PUTC' } }

token keyword:sym<while> { 'while' { make 'KEYWORD_WHILE' } }

parse_it( $tokenizer );</lang>

</pre>

# following two must remain in the same order

"RIGHTPAREN", "LEFTBRACE", "RIGHTBRACE", "SEMICOLON", "COMMA", "IDENTIFIER",

"INTEGER", "STRING"]

# single character only symbols

symbols = { '{': tk_Lbrace, '}': tk_Rbrace, '(': tk_Lparen, ')': tk_Rparen, '+': tk_Add, '-': tk_Sub,

the_ch = " " # dummy first char - but it must be a space

elif the_ch == '/': return div_or_cmt(err_line, err_col)

elif the_ch == '\'': return char_lit(err_line, err_col)

elif the_ch == '"': return string_lit(the_ch, err_line, err_col)

elif the_ch in symbols:

<b>

<pre>

</pre>

</b>