TRIZEN / Perl-Tokenizer-0.11 / t / 03-code.t 

            

              
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186

              
#!perl -T
use 5.006;
use strict;
use warnings;
use Test::More;
plan tests => 311;
use Perl::Tokenizer qw(perl_tokens);
my @tokens = (
    [
        'tr{1{{{}}\}\}\{\{\{\{}3}{1{}/2\}\{{}\}}' => 'transliteration',
    ],
    [
     # The first comment is tricky to parse.
     'tr{}   # comment 1
           //   # comment 2' => ('transliteration', 'horizontal_space', 'comment',)
    ],
    ['1==2' => ('number', 'operator', 'number')],
    ['1!=2' => ('number', 'operator', 'number')],
    ['1<=2' => ('number', 'operator', 'number')],
    ['1>=2' => ('number', 'operator', 'number')],
    ['1>>2' => ('number', 'operator', 'number')],
    ['1<<2' => ('number', 'operator', 'number')],
    ['1%2'  => ('number', 'operator', 'number')],
    ['1**2' => ('number', 'operator', 'number')],
    ['1^2'  => ('number', 'operator', 'number')],
    ['1|2'  => ('number', 'operator', 'number')],
    ['1||2' => ('number', 'operator', 'number')],
    ['1&2'  => ('number', 'operator', 'number')],
    ['1&&2' => ('number', 'operator', 'number')],
    ['1//2' => ('number', 'operator', 'number')],
    ['1/2'  => ('number', 'operator', 'number')],
    ['~2'  => ('operator', 'number')],
    ['++2' => ('operator', 'number')],
    ['--2' => ('operator', 'number')],
    ['-2'  => ('operator', 'number')],
    ['+2'  => ('operator', 'number')],
    ['!2'  => ('operator', 'number')],
    ['!!2' => ('operator', 'operator', 'number')],
    ['+!2' => ('operator', 'operator', 'number')],
    ['-!2' => ('operator', 'operator', 'number')],
    ['!-2' => ('operator', 'operator', 'number')],
    ['!+2' => ('operator', 'operator', 'number')],
    ['"a"."b"'  => ('double_quoted_string', 'operator', 'double_quoted_string')],
    ['"a"&."b"' => ('double_quoted_string', 'operator', 'double_quoted_string')],
    ['"a"|."b"' => ('double_quoted_string', 'operator', 'double_quoted_string')],
    ['"a"^."b"' => ('double_quoted_string', 'operator', 'double_quoted_string')],
    ['~."a"' => ('operator', 'double_quoted_string')],
    ['$x<<=2' => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    ['$x>>=2' => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    ['$x||=2' => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    ['$x|=2'  => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    ['$x=2'   => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    ['$x-=2'  => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    ['$x//=2' => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    ['$x**=2' => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    ['$x%=2'  => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    ['$x.=2'  => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    ['$x/=2'  => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    ['$x^=2'  => ('scalar_sigil', 'var_name', 'assignment_operator', 'number')],
    [
     '$x / 3 / 4' => (
                      'scalar_sigil',     'var_name', 'horizontal_space', 'operator', 'horizontal_space', 'number',
                      'horizontal_space', 'operator', 'horizontal_space', 'number'
                     )
    ],
    [
     '$x // 3 // 4' => (
                        'scalar_sigil',     'var_name', 'horizontal_space', 'operator', 'horizontal_space', 'number',
                        'horizontal_space', 'operator', 'horizontal_space', 'number'
                       )
    ],
    [
     '$x % 3 / 4' => (
                      'scalar_sigil',     'var_name', 'horizontal_space', 'operator', 'horizontal_space', 'number',
                      'horizontal_space', 'operator', 'horizontal_space', 'number'
                     )
    ],
    [
     '$x / 3 - 4' => (
                      'scalar_sigil',     'var_name', 'horizontal_space', 'operator', 'horizontal_space', 'number',
                      'horizontal_space', 'operator', 'horizontal_space', 'number'
                     )
    ],
    [
     '$s[0]{tr}[-1]-- > ++$m{ y };' => (
                                        'scalar_sigil',        'var_name',
                                        'right_bracket_open',  'number',
                                        'right_bracket_close', 'curly_bracket_open',
                                        'bare_word',           'curly_bracket_close',
                                        'right_bracket_open',  'operator',
                                        'number',              'right_bracket_close',
                                        'operator',            'horizontal_space',
                                        'operator',            'horizontal_space',
                                        'operator',            'scalar_sigil',
                                        'var_name',            'curly_bracket_open',
                                        'horizontal_space',    'bare_word',
                                        'horizontal_space',    'curly_bracket_close',
                                        'semicolon',
                                       )
    ],
    [
     '$h{s}++ < 3.14159' => (
                             'scalar_sigil',        'var_name', 'curly_bracket_open', 'bare_word',
                             'curly_bracket_close', 'operator', 'horizontal_space',   'operator',
                             'horizontal_space',    'number'
                            )
    ],
    [
     'while(<$fh>){print}' => (
                               'keyword',            'parenthesis_open', 'glob_readline', 'parenthesis_close',
                               'curly_bracket_open', 'keyword',          'curly_bracket_close'
                              )
    ],
    [
     '-s $foo;' => ('file_test', 'horizontal_space', 'scalar_sigil', 'var_name', 'semicolon')
    ],
    [
     'print STDERR 42,"bar"' =>
       ('keyword', 'horizontal_space', 'special_fh', 'horizontal_space', 'number', 'comma', 'double_quoted_string')
    ],
    [
     <<'CODE'
my @a = (<<'EOT',<<"EOF",<<EOD);   # here-docs
foo
EOT
bar
baz
EOF
qux
EOD
CODE
       => (
           'keyword',          'horizontal_space',    'array_sigil',      'var_name',
           'horizontal_space', 'assignment_operator', 'horizontal_space', 'parenthesis_open',
           'heredoc_beg',      'comma',               'heredoc_beg',      'comma',
           'heredoc_beg',      'parenthesis_close',   'semicolon',        'horizontal_space',
           'comment',          'vertical_space',      'heredoc',          'vertical_space',
           'heredoc',          'vertical_space',      'heredoc',          'vertical_space'
          )
    ],
);
foreach my $group (@tokens) {
    my ($code, @names) = @{$group};
    perl_tokens {
        my ($token) = @_;
        is($token, shift(@names));
    }
    $code;
    ok(!@names);
}