10 static void countlines(char*text, int len) {
22 static int verbose = 1;
23 static void dbg(const char*format, ...)
30 va_start(arglist, format);
31 vsprintf(buf, format, arglist);
34 while(l && buf[l-1]=='\n') {
38 printf("(tokenizer) ");
43 void syntaxerror(const char*format, ...)
50 va_start(arglist, format);
51 vsprintf(buf, format, arglist);
53 fprintf(stderr, "%s:%d:%d: error: %s\n", current_filename, current_line, current_column, buf);
59 #ifndef YY_CURRENT_BUFFER
60 #define YY_CURRENT_BUFFER yy_current_buffer
63 void handleInclude(char*text, int len, char quotes)
67 char*p1 = strchr(text, '"');
68 char*p2 = strrchr(text, '"');
69 if(!p1 || !p2 || p1==p2) {
70 syntaxerror("Invalid include in line %d\n", current_line);
73 filename = strdup(p1+1);
77 while(!strchr(" \n\r\t", text[i1])) i1++;
79 while(strchr(" \n\r\t", text[i1])) i1++;
80 while(strchr(" \n\r\t", text[i2-1])) i2--;
81 if(i2!=len) text[i2]=0;
82 filename = strdup(&text[i1]);
85 char*fullfilename = enter_file(filename, YY_CURRENT_BUFFER);
86 yyin = fopen(fullfilename, "rb");
88 syntaxerror("Couldn't open include file \"%s\"\n", fullfilename);
91 yy_switch_to_buffer(yy_create_buffer( yyin, YY_BUF_SIZE ) );
92 //BEGIN(INITIAL); keep context
95 char start_of_expression;
97 static inline int m(int type)
99 char*s = malloc(yyleng+1);
100 memcpy(s, yytext, yyleng);
110 void initialize_scanner();
111 #define YY_USER_INIT initialize_scanner();
113 #define c() {countlines(yytext, yyleng);}
120 NAME [a-zA-Z_][a-zA-Z0-9_\\]*
121 NUMBER -?[0-9]+(\.[0-9]*)?
122 STRING ["](\\[\x00-\xff]|[^\\"\n])*["]|['](\\[\x00-\xff]|[^\\'\n])*[']
124 MULTILINE_COMMENT [/][*]([*][^/]|[^*]|[\x00-\x31])*[*]+[/]
125 SINGLELINE_COMMENT \/\/[^\n]*\n
126 REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]*
130 {SINGLELINE_COMMENT} {c(); /* single line comment */}
131 {MULTILINE_COMMENT} {c(); /* multi line comment */}
132 [/][*] {syntaxerror("syntax error: unterminated comment", yytext);}
134 ^include{S}+{STRING}{S}*/\n {c();handleInclude(yytext, yyleng, 1);}
135 ^include{S}+[^" \t\r\n][\x20-\xff]*{S}*/\n {c();handleInclude(yytext, yyleng, 0);}
136 {STRING} {c(); return m(T_STRING);BEGIN(INITIAL);}
138 <BEGINNING,REGEXPOK>{
139 {REGEXP} {c(); return m(T_REGEXP);BEGIN(INITIAL);}
142 \xef\xbb\xbf {/* utf 8 bom */}
145 {NUMBER} {c();return m(T_NUMBER);BEGIN(INITIAL);}
146 [>][=] {return m(T_GE);}
147 [<][=] {return m(T_LE);}
148 [-][-] {return m(T_MINUSMINUS);BEGIN(INITIAL);}
149 [+][+] {return m(T_PLUSPLUS);BEGIN(INITIAL);}
150 == {return m(T_EQEQ);BEGIN(REGEXPOK);}
151 \.\. {return m(T_DOTDOT);}
153 :: {return m(T_COLONCOLON);}
155 implements {return m(T_IMPLEMENTS);}
156 interface {return m(T_INTERFACE);}
157 namespace {return m(T_NAMESPACE);}
158 protected {return m(T_PROTECTED);}
159 override {return m(T_OVERRIDE);}
160 internal {return m(T_INTERNAL);}
161 function {return m(T_FUNCTION);}
162 package {return m(T_PACKAGE);}
163 private {return m(T_PRIVATE);}
164 Boolean {return m(T_BOOLEAN);}
165 dynamic {return m(T_DYNAMIC);}
166 extends {return m(T_EXTENDS);}
167 public {return m(T_PUBLIC);}
168 native {return m(T_NATIVE);}
169 static {return m(T_STATIC);}
170 import {return m(T_IMPORT);}
171 number {return m(T_NUMBER);}
172 class {return m(T_CLASS);}
173 const {return m(T_CONST);}
174 final {return m(T_FINAL);}
175 False {return m(T_FALSE);}
176 True {return m(T_TRUE);}
177 uint {return m(T_UINT);}
178 null {return m(T_NULL);}
179 use {return m(T_USE);}
180 int {return m(T_INT);}
181 new {return m(T_NEW);}
182 get {return m(T_GET);}
183 for {return m(T_FOR);}
184 set {return m(T_SET);}
185 var {return m(T_VAR);}
186 is {return m(T_IS) ;}
188 {NAME} {c();BEGIN(INITIAL);return m(T_IDENTIFIER);}
190 [+-\/*^~@$!%&\(=\[\]\{\}|?:;,.<>] {c();BEGIN(REGEXPOK);return m(yytext[0]);}
191 [\)\]] {c();BEGIN(INITIAL);return m(yytext[0]);}
193 . {char c1=yytext[0];
198 char c = buf[t]=input();
199 if(c=='\n' || c==EOF) {
204 if(c1>='0' && c1<='9')
205 syntaxerror("syntax error: %s (identifiers must not start with a digit)");
207 syntaxerror("syntax error: %s", buf);
213 void*b = leave_file();
216 yy_delete_buffer(YY_CURRENT_BUFFER);
219 yy_delete_buffer(YY_CURRENT_BUFFER);
220 yy_switch_to_buffer(b);
231 static char mbuf[256];
232 char*token2string(token_t*t)
235 if(nr==T_STRING) return "STRING";
236 else if(nr==T_NUMBER) return "NUMBER";
237 else if(nr==T_REGEXP) return "REGEXP";
238 else if(nr==T_EOF) return "***END***";
239 else if(nr==T_GE) return ">=";
240 else if(nr==T_LE) return "<=";
241 else if(nr==T_MINUSMINUS) return "--";
242 else if(nr==T_PLUSPLUS) return "++";
243 else if(nr==T_IMPLEMENTS) return "implements";
244 else if(nr==T_INTERFACE) return "interface";
245 else if(nr==T_NAMESPACE) return "namespace";
246 else if(nr==T_PROTECTED) return "protected";
247 else if(nr==T_OVERRIDE) return "override";
248 else if(nr==T_INTERNAL) return "internal";
249 else if(nr==T_FUNCTION) return "function";
250 else if(nr==T_PACKAGE) return "package";
251 else if(nr==T_PRIVATE) return "private";
252 else if(nr==T_BOOLEAN) return "Boolean";
253 else if(nr==T_DYNAMIC) return "dynamic";
254 else if(nr==T_EXTENDS) return "extends";
255 else if(nr==T_PUBLIC) return "public";
256 else if(nr==T_NATIVE) return "native";
257 else if(nr==T_STATIC) return "static";
258 else if(nr==T_IMPORT) return "import";
259 else if(nr==T_NUMBER) return "number";
260 else if(nr==T_CLASS) return "class";
261 else if(nr==T_CONST) return "const";
262 else if(nr==T_FINAL) return "final";
263 else if(nr==T_FALSE) return "False";
264 else if(nr==T_TRUE) return "True";
265 else if(nr==T_UINT) return "uint";
266 else if(nr==T_NULL) return "null";
267 else if(nr==T_USE) return "use";
268 else if(nr==T_INT) return "int";
269 else if(nr==T_NEW) return "new";
270 else if(nr==T_GET) return "get";
271 else if(nr==T_FOR) return "for";
272 else if(nr==T_SET) return "set";
273 else if(nr==T_VAR) return "var";
274 else if(nr==T_IS) return "is";
275 else if(nr==T_AS) return "as";
276 else if(nr==T_IDENTIFIER) {
277 if(strlen(t->text)>sizeof(mbuf)-1)
279 sprintf(mbuf, "ID(%s)", t->text);
282 sprintf(mbuf, "%d", nr);
287 void initialize_scanner()