10 static void countlines(char*text, int len) {
22 static int verbose = 1;
23 static void dbg(const char*format, ...)
30 va_start(arglist, format);
31 vsprintf(buf, format, arglist);
34 while(l && buf[l-1]=='\n') {
38 printf("(tokenizer) ");
43 void syntaxerror(const char*format, ...)
50 va_start(arglist, format);
51 vsprintf(buf, format, arglist);
53 fprintf(stderr, "%s:%d:%d: error: %s\n", current_filename, current_line, current_column, buf);
59 #ifndef YY_CURRENT_BUFFER
60 #define YY_CURRENT_BUFFER yy_current_buffer
63 void handleInclude(char*text, int len, char quotes)
67 char*p1 = strchr(text, '"');
68 char*p2 = strrchr(text, '"');
69 if(!p1 || !p2 || p1==p2) {
70 syntaxerror("Invalid include in line %d\n", current_line);
73 filename = strdup(p1+1);
77 while(!strchr(" \n\r\t", text[i1])) i1++;
79 while(strchr(" \n\r\t", text[i1])) i1++;
80 while(strchr(" \n\r\t", text[i2-1])) i2--;
81 if(i2!=len) text[i2]=0;
82 filename = strdup(&text[i1]);
85 char*fullfilename = enter_file(filename, YY_CURRENT_BUFFER);
86 yyin = fopen(fullfilename, "rb");
88 syntaxerror("Couldn't open include file \"%s\"\n", fullfilename);
91 yy_switch_to_buffer(yy_create_buffer( yyin, YY_BUF_SIZE ) );
92 //BEGIN(INITIAL); keep context
95 char start_of_expression;
97 static inline int m(int type)
99 char*s = malloc(yyleng+1);
100 memcpy(s, yytext, yyleng);
110 static char numberbuf[64];
111 static inline int handlenumber()
113 if(yyleng>sizeof(numberbuf)-1)
114 syntaxerror("decimal number overflow");
117 memcpy(s, yytext, yyleng);
122 for(t=0;t<yyleng;t++) {
125 syntaxerror("Invalid number");
127 } else if(!strchr("-0123456789", yytext[t])) {
128 syntaxerror("Invalid number");
132 avm2_lval.number_float = atof(s);
135 char l = (yytext[0]=='-');
137 char*max = l?"1073741824":"2147483647";
139 syntaxerror("integer overflow");
142 for(t=0;t<yyleng-l;t++) {
143 if(yytext[l+t]>max[t])
144 syntaxerror("integer overflow %s > %s", s+l,max);
145 else if(yytext[l+t]<max[t])
150 avm2_lval.number_int = atoi(s);
154 for(t=0;t<yyleng;t++) {
158 avm2_lval.number_uint = v;
161 /* useless- numbers are usually smaller if stored in the constant pool
162 else if(v<0x80000000u)
169 void initialize_scanner();
170 #define YY_USER_INIT initialize_scanner();
172 #define c() {countlines(yytext, yyleng);}
179 NAME [a-zA-Z_][a-zA-Z0-9_\\]*
181 NUMBER -?[0-9]+(\.[0-9]*)?
183 STRING ["](\\[\x00-\xff]|[^\\"\n])*["]|['](\\[\x00-\xff]|[^\\'\n])*[']
185 MULTILINE_COMMENT [/][*]([*][^/]|[^*]|[\x00-\x31])*[*]+[/]
186 SINGLELINE_COMMENT \/\/[^\n]*\n
187 REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]*
191 {SINGLELINE_COMMENT} {c(); /* single line comment */}
192 {MULTILINE_COMMENT} {c(); /* multi line comment */}
193 [/][*] {syntaxerror("syntax error: unterminated comment", yytext);}
195 ^include{S}+{STRING}{S}*/\n {c();handleInclude(yytext, yyleng, 1);}
196 ^include{S}+[^" \t\r\n][\x20-\xff]*{S}*/\n {c();handleInclude(yytext, yyleng, 0);}
197 {STRING} {c(); BEGIN(INITIAL);return m(T_STRING);}
199 <BEGINNING,REGEXPOK>{
200 {REGEXP} {c(); BEGIN(INITIAL);return m(T_REGEXP);}
203 \xef\xbb\xbf {/* utf 8 bom */}
206 {NUMBER} {c(); BEGIN(INITIAL);return handlenumber();}
208 [>][=] {return m(T_GE);}
209 [<][=] {return m(T_LE);}
210 [-][-] {BEGIN(INITIAL);return m(T_MINUSMINUS);}
211 [+][+] {BEGIN(INITIAL);return m(T_PLUSPLUS);}
212 == {BEGIN(REGEXPOK);return m(T_EQEQ);}
213 \.\. {return m(T_DOTDOT);}
215 :: {return m(T_COLONCOLON);}
217 implements {return m(KW_IMPLEMENTS);}
218 interface {return m(KW_INTERFACE);}
219 namespace {return m(KW_NAMESPACE);}
220 protected {return m(KW_PROTECTED);}
221 override {return m(KW_OVERRIDE);}
222 internal {return m(KW_INTERNAL);}
223 function {return m(KW_FUNCTION);}
224 package {return m(KW_PACKAGE);}
225 private {return m(KW_PRIVATE);}
226 Boolean {return m(KW_BOOLEAN);}
227 dynamic {return m(KW_DYNAMIC);}
228 extends {return m(KW_EXTENDS);}
229 public {return m(KW_PUBLIC);}
230 native {return m(KW_NATIVE);}
231 static {return m(KW_STATIC);}
232 import {return m(KW_IMPORT);}
233 Number {return m(KW_NUMBER);}
234 class {return m(KW_CLASS);}
235 const {return m(KW_CONST);}
236 final {return m(KW_FINAL);}
237 False {return m(KW_FALSE);}
238 True {return m(KW_TRUE);}
239 uint {return m(KW_UINT);}
240 null {return m(KW_NULL);}
241 use {return m(KW_USE);}
242 int {return m(KW_INT);}
243 new {return m(KW_NEW);}
244 get {return m(KW_GET);}
245 for {return m(KW_FOR);}
246 set {return m(KW_SET);}
247 var {return m(KW_VAR);}
248 is {return m(KW_IS) ;}
249 as {return m(KW_AS);}
250 {NAME} {c();BEGIN(INITIAL);return m(T_IDENTIFIER);}
252 [+-\/*^~@$!%&\(=\[\]\{\}|?:;,.<>] {c();BEGIN(REGEXPOK);return m(yytext[0]);}
253 [\)\]] {c();BEGIN(INITIAL);return m(yytext[0]);}
255 . {char c1=yytext[0];
260 char c = buf[t]=input();
261 if(c=='\n' || c==EOF) {
266 if(c1>='0' && c1<='9')
267 syntaxerror("syntax error: %s (identifiers must not start with a digit)");
269 syntaxerror("syntax error: %s", buf);
275 void*b = leave_file();
278 yy_delete_buffer(YY_CURRENT_BUFFER);
281 yy_delete_buffer(YY_CURRENT_BUFFER);
282 yy_switch_to_buffer(b);
293 static char mbuf[256];
294 char*token2string(token_t*t)
297 if(nr==T_STRING) return "<string>";
298 else if(nr==T_INT) return "<int>";
299 else if(nr==T_UINT) return "<uint>";
300 else if(nr==T_FLOAT) return "<float>";
301 else if(nr==T_REGEXP) return "REGEXP";
302 else if(nr==T_EOF) return "***END***";
303 else if(nr==T_GE) return ">=";
304 else if(nr==T_LE) return "<=";
305 else if(nr==T_MINUSMINUS) return "--";
306 else if(nr==T_PLUSPLUS) return "++";
307 else if(nr==KW_IMPLEMENTS) return "implements";
308 else if(nr==KW_INTERFACE) return "interface";
309 else if(nr==KW_NAMESPACE) return "namespace";
310 else if(nr==KW_PROTECTED) return "protected";
311 else if(nr==KW_OVERRIDE) return "override";
312 else if(nr==KW_INTERNAL) return "internal";
313 else if(nr==KW_FUNCTION) return "function";
314 else if(nr==KW_PACKAGE) return "package";
315 else if(nr==KW_PRIVATE) return "private";
316 else if(nr==KW_BOOLEAN) return "Boolean";
317 else if(nr==KW_DYNAMIC) return "dynamic";
318 else if(nr==KW_EXTENDS) return "extends";
319 else if(nr==KW_PUBLIC) return "public";
320 else if(nr==KW_NATIVE) return "native";
321 else if(nr==KW_STATIC) return "static";
322 else if(nr==KW_IMPORT) return "import";
323 else if(nr==KW_NUMBER) return "number";
324 else if(nr==KW_CLASS) return "class";
325 else if(nr==KW_CONST) return "const";
326 else if(nr==KW_FINAL) return "final";
327 else if(nr==KW_FALSE) return "False";
328 else if(nr==KW_TRUE) return "True";
329 else if(nr==KW_UINT) return "uint";
330 else if(nr==KW_NULL) return "null";
331 else if(nr==KW_USE) return "use";
332 else if(nr==KW_INT) return "int";
333 else if(nr==KW_NEW) return "new";
334 else if(nr==KW_GET) return "get";
335 else if(nr==KW_FOR) return "for";
336 else if(nr==KW_SET) return "set";
337 else if(nr==KW_VAR) return "var";
338 else if(nr==KW_IS) return "is";
339 else if(nr==KW_AS) return "as";
340 else if(nr==T_IDENTIFIER) {
341 if(strlen(t->text)>sizeof(mbuf)-1)
343 sprintf(mbuf, "ID(%s)", t->text);
346 sprintf(mbuf, "%d", nr);
351 void initialize_scanner()