dump out method flags
[swftools.git] / lib / as3 / tokenizer.lex
1 /* tokenizer.lex
2
3    Routines for compiling Flash2 AVM2 ABC Actionscript
4
5    Extension module for the rfxswf library.
6    Part of the swftools package.
7
8    Copyright (c) 2008 Matthias Kramm <kramm@quiss.org>
9  
10    This program is free software; you can redistribute it and/or modify
11    it under the terms of the GNU General Public License as published by
12    the Free Software Foundation; either version 2 of the License, or
13    (at your option) any later version.
14
15    This program is distributed in the hope that it will be useful,
16    but WITHOUT ANY WARRANTY; without even the implied warranty of
17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18    GNU General Public License for more details.
19
20    You should have received a copy of the GNU General Public License
21    along with this program; if not, write to the Free Software
22    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
23 %{
24
25
26 #include <string.h>
27 #include <stdlib.h>
28 #include <stdio.h>
29 #include <stdarg.h>
30 #include "../utf8.h"
31 #include "tokenizer.h"
32 #include "files.h"
33
34 static void countlines(char*text, int len) {
35     int t;
36     for(t=0;t<len;t++) {
37         if(text[t]=='\n') {
38             current_line++;
39             current_column=0;
40         } else {
41             current_column++;
42         }
43     }
44 }
45
46 static int verbose = 1;
47 static void dbg(const char*format, ...)
48 {
49     char buf[1024];
50     int l;
51     va_list arglist;
52     if(!verbose)
53         return;
54     va_start(arglist, format);
55     vsprintf(buf, format, arglist);
56     va_end(arglist);
57     l = strlen(buf);
58     while(l && buf[l-1]=='\n') {
59         buf[l-1] = 0;
60         l--;
61     }
62     printf("(tokenizer) ");
63     printf("%s\n", buf);
64     fflush(stdout);
65 }
66
67 void syntaxerror(const char*format, ...)
68 {
69     char buf[1024];
70     int l;
71     va_list arglist;
72     if(!verbose)
73         return;
74     va_start(arglist, format);
75     vsprintf(buf, format, arglist);
76     va_end(arglist);
77     fprintf(stderr, "%s:%d:%d: error: %s\n", current_filename_short, current_line, current_column, buf);
78     fflush(stderr);
79     exit(1);
80 }
81
82
83 #ifndef YY_CURRENT_BUFFER
84 #define YY_CURRENT_BUFFER yy_current_buffer
85 #endif
86
87 void handleInclude(char*text, int len, char quotes)
88 {
89     char*filename = 0;
90     if(quotes) {
91         char*p1 = strchr(text, '"');
92         char*p2 = strrchr(text, '"');
93         if(!p1 || !p2 || p1==p2) {
94             syntaxerror("Invalid include in line %d\n", current_line);
95         }
96         *p2 = 0;
97         filename = strdup(p1+1);
98     } else {
99         int i1=0,i2=len;
100         // find start
101         while(!strchr(" \n\r\t", text[i1])) i1++;
102         // strip
103         while(strchr(" \n\r\t", text[i1])) i1++;
104         while(strchr(" \n\r\t", text[i2-1])) i2--;
105         if(i2!=len) text[i2]=0;
106         filename = strdup(&text[i1]);
107     }
108     
109     char*fullfilename = enter_file(filename, YY_CURRENT_BUFFER);
110     yyin = fopen(fullfilename, "rb");
111     if (!yyin) {
112         syntaxerror("Couldn't open include file \"%s\"\n", fullfilename);
113     }
114
115     yy_switch_to_buffer(yy_create_buffer( yyin, YY_BUF_SIZE ) );
116     //BEGIN(INITIAL); keep context
117 }
118
119 static void handleString(char*s, int len)
120 {
121     if(s[0]=='"') {
122         if(s[len-1]!='"') syntaxerror("String doesn't end with '\"'");
123         s++;len-=2;
124     }
125     else if(s[0]=='\'') {
126         if(s[len-1]!='\'') syntaxerror("String doesn't end with '\"'");
127         s++;len-=2;
128     }
129     else syntaxerror("String incorrectly terminated");
130     s[len] = 0;
131     avm2_lval.string = s;
132 }
133
134
135 char start_of_expression;
136
137 static inline int m(int type)
138 {
139     char*s = malloc(yyleng+1);
140     memcpy(s, yytext, yyleng);
141     s[yyleng]=0;
142
143     NEW(token_t,t);
144     t->type = type;
145     t->text = s;
146     avm2_lval.token = t;
147     return type;
148 }
149
150 static char numberbuf[64];
151 static inline int handlenumber()
152 {
153     if(yyleng>sizeof(numberbuf)-1)
154         syntaxerror("decimal number overflow");
155
156     char*s = numberbuf;
157     memcpy(s, yytext, yyleng);
158     s[yyleng]=0;
159
160     int t;
161     char is_float=0;
162     for(t=0;t<yyleng;t++) {
163         if(yytext[t]=='.') {
164             if(is_float)
165                 syntaxerror("Invalid number");
166             is_float=1;
167         } else if(!strchr("-0123456789", yytext[t])) {
168             syntaxerror("Invalid number");
169         }
170     }
171     if(is_float) {
172         avm2_lval.number_float = atof(s);
173         return T_FLOAT;
174     } 
175     char l = (yytext[0]=='-');
176
177     char*max = l?"1073741824":"2147483647";
178     if(yyleng-l>10)
179         syntaxerror("integer overflow");
180     if(yyleng-l==10) {
181         int t;
182         for(t=0;t<yyleng-l;t++) {
183             if(yytext[l+t]>max[t])
184                 syntaxerror("integer overflow %s > %s", s+l,max);
185             else if(yytext[l+t]<max[t])
186                 break;
187         }
188     }
189     if(yytext[0]=='-') {
190         int v = atoi(s);
191         avm2_lval.number_int = v;
192         if(v>-128)
193             return T_BYTE;
194         else if(v>=-32768)
195             return T_SHORT;
196         else
197             return T_INT;
198     } else {
199         unsigned int v = 0;
200         for(t=0;t<yyleng;t++) {
201             v*=10;
202             v+=yytext[t]-'0';
203         }
204         avm2_lval.number_uint = v;
205         if(v<128)
206             return T_BYTE;
207         else if(v<32768)
208             return T_SHORT;
209         else
210             return T_UINT;
211     }
212 }
213
214 void initialize_scanner();
215 #define YY_USER_INIT initialize_scanner();
216
217 #define c() {countlines(yytext, yyleng);}
218
219 %}
220
221 %s REGEXPOK
222 %s BEGINNING
223
224 NAME     [a-zA-Z_][a-zA-Z0-9_\\]*
225
226 NUMBER   -?[0-9]+(\.[0-9]*)?
227
228 STRING   ["](\\[\x00-\xff]|[^\\"\n])*["]|['](\\[\x00-\xff]|[^\\'\n])*[']
229 S        [ \n\r\t]
230 MULTILINE_COMMENT [/][*]+([*][^/]|[^/*]|[\x00-\x1f])*[*]+[/]
231 SINGLELINE_COMMENT \/\/[^\n]*\n
232 REGEXP   [/]([^/\n]|\\[/])*[/][a-zA-Z]*
233 %%
234
235
236 {SINGLELINE_COMMENT}         {c(); /* single line comment */}
237 {MULTILINE_COMMENT}          {c(); /* multi line comment */}
238 [/][*]                       {syntaxerror("syntax error: unterminated comment", yytext);}
239
240 ^include{S}+{STRING}{S}*/\n    {c();handleInclude(yytext, yyleng, 1);}
241 ^include{S}+[^" \t\r\n][\x20-\xff]*{S}*/\n    {c();handleInclude(yytext, yyleng, 0);}
242 {STRING}                     {c(); BEGIN(INITIAL);handleString(yytext, yyleng);return T_STRING;}
243
244 <BEGINNING,REGEXPOK>{
245 {REGEXP}                     {c(); BEGIN(INITIAL);return m(T_REGEXP);} 
246 }
247
248 \xef\xbb\xbf                 {/* utf 8 bom */}
249 {S}                          {c();}
250
251 {NUMBER}                     {c(); BEGIN(INITIAL);return handlenumber();}
252
253 3rr0r                        {/* for debugging: generates a tokenizer-level error */
254                               syntaxerror("3rr0r");}
255
256 [!][=]                       {BEGIN(REGEXPOK);return m(T_NE);}
257 [=][=][=]                    {BEGIN(REGEXPOK);return m(T_EQEQEQ);}
258 [=][=]                       {BEGIN(REGEXPOK);return m(T_EQEQ);}
259 [>][=]                       {return m(T_GE);}
260 [<][=]                       {return m(T_LE);}
261 [+][=]                       {return m(T_PLUSBY);}
262 [-][=]                       {return m(T_MINUSBY);}
263 [-][-]                       {BEGIN(INITIAL);return m(T_MINUSMINUS);}
264 [+][+]                       {BEGIN(INITIAL);return m(T_PLUSPLUS);}
265 \.\.                         {return m(T_DOTDOT);}
266 \.                           {return m('.');}
267 ::                           {return m(T_COLONCOLON);}
268 :                            {return m(':');}
269 implements                   {return m(KW_IMPLEMENTS);}
270 interface                    {return m(KW_INTERFACE);}
271 namespace                    {return m(KW_NAMESPACE);}
272 protected                    {return m(KW_PROTECTED);}
273 override                     {return m(KW_OVERRIDE);}
274 internal                     {return m(KW_INTERNAL);}
275 function                     {return m(KW_FUNCTION);}
276 package                      {return m(KW_PACKAGE);}
277 private                      {return m(KW_PRIVATE);}
278 Boolean                      {return m(KW_BOOLEAN);}
279 dynamic                      {return m(KW_DYNAMIC);}
280 extends                      {return m(KW_EXTENDS);}
281 public                       {return m(KW_PUBLIC);}
282 native                       {return m(KW_NATIVE);}
283 static                       {return m(KW_STATIC);}
284 import                       {return m(KW_IMPORT);}
285 Number                       {return m(KW_NUMBER);}
286 while                        {return m(KW_WHILE);}
287 class                        {return m(KW_CLASS);}
288 const                        {return m(KW_CONST);}
289 final                        {return m(KW_FINAL);}
290 false                        {return m(KW_FALSE);}
291 break                        {return m(KW_BREAK);}
292 true                         {return m(KW_TRUE);}
293 uint                         {return m(KW_UINT);}
294 null                         {return m(KW_NULL);}
295 else                         {return m(KW_ELSE);}
296 use                          {return m(KW_USE);}
297 int                          {return m(KW_INT);}
298 new                          {return m(KW_NEW);}
299 get                          {return m(KW_GET);}
300 for                          {return m(KW_FOR);}
301 set                          {return m(KW_SET);}
302 var                          {return m(KW_VAR);}
303 is                           {return m(KW_IS) ;}
304 if                           {return m(KW_IF) ;}
305 as                           {return m(KW_AS);}
306 {NAME}                       {c();BEGIN(INITIAL);return m(T_IDENTIFIER);}
307
308 [+-\/*^~@$!%&\(=\[\]\{\}|?:;,.<>] {c();BEGIN(REGEXPOK);return m(yytext[0]);}
309 [\)\]]                            {c();BEGIN(INITIAL);return m(yytext[0]);}
310
311 .                            {char c1=yytext[0];
312                               char buf[128];
313                               buf[0] = yytext[0];
314                               int t;
315                               for(t=1;t<128;t++) {
316                                   char c = buf[t]=input();
317                                   if(c=='\n' || c==EOF)  {
318                                       buf[t] = 0;
319                                       break;
320                                   }
321                               }
322                               if(c1>='0' && c1<='9')
323                                   syntaxerror("syntax error: %s (identifiers must not start with a digit)");
324                               else
325                                   syntaxerror("syntax error: %s", buf);
326                               printf("\n");
327                               exit(1);
328                               yyterminate();
329                              }
330 <<EOF>>                      {c();
331                               void*b = leave_file();
332                               if (!b) {
333                                  yyterminate();
334                                  yy_delete_buffer(YY_CURRENT_BUFFER);
335                                  return m(T_EOF);
336                               } else {
337                                   yy_delete_buffer(YY_CURRENT_BUFFER);
338                                   yy_switch_to_buffer(b);
339                               }
340                              }
341
342 %%
343
344 int yywrap()
345 {
346     return 1;
347 }
348
349 static char mbuf[256];
350 char*token2string(token_t*t)
351 {
352     int nr=t->type;
353     if(nr==T_STRING)     return "<string>";
354     else if(nr==T_INT)     return "<int>";
355     else if(nr==T_UINT)     return "<uint>";
356     else if(nr==T_FLOAT)     return "<float>";
357     else if(nr==T_REGEXP)     return "REGEXP";
358     else if(nr==T_EOF)        return "***END***";
359     else if(nr==T_GE)         return ">=";
360     else if(nr==T_LE)         return "<=";
361     else if(nr==T_MINUSMINUS) return "--";
362     else if(nr==T_PLUSPLUS)   return "++";
363     else if(nr==KW_IMPLEMENTS) return "implements";
364     else if(nr==KW_INTERFACE)  return "interface";
365     else if(nr==KW_NAMESPACE)  return "namespace";
366     else if(nr==KW_PROTECTED)  return "protected";
367     else if(nr==KW_OVERRIDE)   return "override";
368     else if(nr==KW_INTERNAL)   return "internal";
369     else if(nr==KW_FUNCTION)   return "function";
370     else if(nr==KW_PACKAGE)    return "package";
371     else if(nr==KW_PRIVATE)    return "private";
372     else if(nr==KW_BOOLEAN)    return "Boolean";
373     else if(nr==KW_DYNAMIC)    return "dynamic";
374     else if(nr==KW_EXTENDS)    return "extends";
375     else if(nr==KW_PUBLIC)     return "public";
376     else if(nr==KW_NATIVE)     return "native";
377     else if(nr==KW_STATIC)     return "static";
378     else if(nr==KW_IMPORT)     return "import";
379     else if(nr==KW_NUMBER)     return "number";
380     else if(nr==KW_CLASS)      return "class";
381     else if(nr==KW_CONST)      return "const";
382     else if(nr==KW_FINAL)      return "final";
383     else if(nr==KW_FALSE)      return "False";
384     else if(nr==KW_TRUE)       return "True";
385     else if(nr==KW_UINT)       return "uint";
386     else if(nr==KW_NULL)       return "null";
387     else if(nr==KW_ELSE)       return "else";
388     else if(nr==KW_USE)        return "use";
389     else if(nr==KW_INT)        return "int";
390     else if(nr==KW_NEW)        return "new";
391     else if(nr==KW_GET)        return "get";
392     else if(nr==KW_FOR)        return "for";
393     else if(nr==KW_SET)        return "set";
394     else if(nr==KW_VAR)        return "var";
395     else if(nr==KW_IS)         return "is";
396     else if(nr==KW_AS)         return "as";
397     else if(nr==T_IDENTIFIER) {
398         if(strlen(t->text)>sizeof(mbuf)-1)
399             return "ID(...)";
400         sprintf(mbuf, "ID(%s)", t->text);
401         return mbuf;
402     } else {
403         sprintf(mbuf, "%d", nr);
404         return mbuf;
405     }
406 }
407
408 void initialize_scanner()
409 {
410     BEGIN(BEGINNING);
411 }
412