10 static void countlines(char*text, int len) {
22 static int verbose = 1;
23 static void dbg(const char*format, ...)
30 va_start(arglist, format);
31 vsprintf(buf, format, arglist);
34 while(l && buf[l-1]=='\n') {
38 printf("(tokenizer) ");
43 void syntaxerror(const char*format, ...)
50 va_start(arglist, format);
51 vsprintf(buf, format, arglist);
53 fprintf(stderr, "%s:%d:%d: error: %s\n", current_filename, current_line, current_column, buf);
59 #ifndef YY_CURRENT_BUFFER
60 #define YY_CURRENT_BUFFER yy_current_buffer
63 void handleInclude(char*text, int len, char quotes)
67 char*p1 = strchr(text, '"');
68 char*p2 = strrchr(text, '"');
69 if(!p1 || !p2 || p1==p2) {
70 syntaxerror("Invalid include in line %d\n", current_line);
73 filename = strdup(p1+1);
77 while(!strchr(" \n\r\t", text[i1])) i1++;
79 while(strchr(" \n\r\t", text[i1])) i1++;
80 while(strchr(" \n\r\t", text[i2-1])) i2--;
81 if(i2!=len) text[i2]=0;
82 filename = strdup(&text[i1]);
85 char*fullfilename = enter_file(filename, YY_CURRENT_BUFFER);
86 yyin = fopen(fullfilename, "rb");
88 syntaxerror("Couldn't open include file \"%s\"\n", fullfilename);
91 yy_switch_to_buffer(yy_create_buffer( yyin, YY_BUF_SIZE ) );
92 //BEGIN(INITIAL); keep context
95 static inline int m(int type)
97 char*s = malloc(yyleng+1);
98 memcpy(s, yytext, yyleng);
108 void initialize_scanner();
109 #define YY_USER_INIT initialize_scanner();
111 #define c() {countlines(yytext, yyleng);}
118 NAME [a-zA-Z_][a-zA-Z0-9_\\]*
119 NUMBER -?[0-9]+(\.[0-9]*)?
120 STRING ["](\\[\x00-\xff]|[^\\"\n])*["]|['](\\[\x00-\xff]|[^\\'\n])*[']
122 MULTILINE_COMMENT [/][*]([*][^/]|[^*]|[\x00-\x31])*[*]+[/]
123 SINGLELINE_COMMENT \/\/[^\n]*\n
124 REGEXP [/]([^/\n]|\\[/])*[/][a-zA-Z]*
128 {SINGLELINE_COMMENT} {c(); /* single line comment */}
129 {MULTILINE_COMMENT} {c(); /* multi line comment */}
130 [/][*] {syntaxerror("syntax error: unterminated comment", yytext);}
132 ^include{S}+{STRING}{S}*/\n {c();handleInclude(yytext, yyleng, 1);}
133 ^include{S}+[^" \t\r\n][\x20-\xff]*{S}*/\n {c();handleInclude(yytext, yyleng, 0);}
134 {STRING} {c(); return m(T_STRING);BEGIN(INITIAL);}
136 <BEGINNING,REGEXPOK>{
137 {REGEXP} {c(); return m(T_REGEXP);BEGIN(INITIAL);}
140 \xef\xbb\xbf {/* utf 8 bom */}
143 {NUMBER} {c();return m(T_NUMBER);BEGIN(INITIAL);}
144 [>][=] {return m(T_GE);}
145 [<][=] {return m(T_LE);}
146 [-][-] {return m(T_MINUSMINUS);BEGIN(INITIAL);}
147 [+][+] {return m(T_PLUSPLUS);BEGIN(INITIAL);}
148 == {return m(T_EQEQ);BEGIN(REGEXPOK);}
149 \.\. {return m(T_DOTDOT);}
151 :: {return m(T_COLONCOLON);}
153 implements {return m(T_IMPLEMENTS);}
154 interface {return m(T_INTERFACE);}
155 namespace {return m(T_NAMESPACE);}
156 protected {return m(T_PROTECTED);}
157 override {return m(T_OVERRIDE);}
158 internal {return m(T_INTERNAL);}
159 function {return m(T_FUNCTION);}
160 package {return m(T_PACKAGE);}
161 private {return m(T_PRIVATE);}
162 Boolean {return m(T_BOOLEAN);}
163 dynamic {return m(T_DYNAMIC);}
164 extends {return m(T_EXTENDS);}
165 public {return m(T_PUBLIC);}
166 native {return m(T_NATIVE);}
167 static {return m(T_STATIC);}
168 import {return m(T_IMPORT);}
169 number {return m(T_NUMBER);}
170 class {return m(T_CLASS);}
171 const {return m(T_CONST);}
172 final {return m(T_FINAL);}
173 False {return m(T_FALSE);}
174 True {return m(T_TRUE);}
175 uint {return m(T_UINT);}
176 null {return m(T_NULL);}
177 use {return m(T_USE);}
178 int {return m(T_INT);}
179 new {return m(T_NEW);}
180 get {return m(T_GET);}
181 for {return m(T_FOR);}
182 set {return m(T_SET);}
183 var {return m(T_VAR);}
184 is {return m(T_IS) ;}
186 {NAME} {c();BEGIN(INITIAL);return m(T_IDENTIFIER);}
188 [+-\/*^~@$!%&\(=\[\]\{\}|?:;,.<>] {c();BEGIN(REGEXPOK);return m(yytext[0]);}
189 [\)\]] {c();BEGIN(INITIAL);return m(yytext[0]);}
191 . {char c1=yytext[0];
196 char c = buf[t]=input();
197 if(c=='\n' || c==EOF) {
202 if(c1>='0' && c1<='9')
203 syntaxerror("syntax error: %s (identifiers must not start with a digit)");
205 syntaxerror("syntax error: %s", buf);
211 void*b = leave_file();
214 yy_delete_buffer(YY_CURRENT_BUFFER);
217 yy_delete_buffer(YY_CURRENT_BUFFER);
218 yy_switch_to_buffer(b);
229 static char mbuf[256];
230 char*token2string(token_t*t)
233 if(nr==T_STRING) return "STRING";
234 else if(nr==T_NUMBER) return "NUMBER";
235 else if(nr==T_REGEXP) return "REGEXP";
236 else if(nr==T_EOF) return "***END***";
237 else if(nr==T_GE) return ">=";
238 else if(nr==T_LE) return "<=";
239 else if(nr==T_MINUSMINUS) return "--";
240 else if(nr==T_PLUSPLUS) return "++";
241 else if(nr==T_IMPLEMENTS) return "implements";
242 else if(nr==T_INTERFACE) return "interface";
243 else if(nr==T_NAMESPACE) return "namespace";
244 else if(nr==T_PROTECTED) return "protected";
245 else if(nr==T_OVERRIDE) return "override";
246 else if(nr==T_INTERNAL) return "internal";
247 else if(nr==T_FUNCTION) return "function";
248 else if(nr==T_PACKAGE) return "package";
249 else if(nr==T_PRIVATE) return "private";
250 else if(nr==T_BOOLEAN) return "Boolean";
251 else if(nr==T_DYNAMIC) return "dynamic";
252 else if(nr==T_EXTENDS) return "extends";
253 else if(nr==T_PUBLIC) return "public";
254 else if(nr==T_NATIVE) return "native";
255 else if(nr==T_STATIC) return "static";
256 else if(nr==T_IMPORT) return "import";
257 else if(nr==T_NUMBER) return "number";
258 else if(nr==T_CLASS) return "class";
259 else if(nr==T_CONST) return "const";
260 else if(nr==T_FINAL) return "final";
261 else if(nr==T_FALSE) return "False";
262 else if(nr==T_TRUE) return "True";
263 else if(nr==T_UINT) return "uint";
264 else if(nr==T_NULL) return "null";
265 else if(nr==T_USE) return "use";
266 else if(nr==T_INT) return "int";
267 else if(nr==T_NEW) return "new";
268 else if(nr==T_GET) return "get";
269 else if(nr==T_FOR) return "for";
270 else if(nr==T_SET) return "set";
271 else if(nr==T_VAR) return "var";
272 else if(nr==T_IS) return "is";
273 else if(nr==T_AS) return "as";
274 else if(nr==T_IDENTIFIER) {
275 if(strlen(t->text)>sizeof(mbuf)-1)
277 sprintf(mbuf, "ID(%s)", t->text);
280 sprintf(mbuf, "%d", nr);
285 void initialize_scanner()