9 //RVALUE {NUMBER}|{PERCENT}|{NAME}|\"{STRING}\"|{DIM}
10 //<a>. {printf("<a>%s\n", yytext);}
11 // %x: exclusive, %s: inclusive
12 char*type_names[] = {"twip","number","command","string","assignment","identifier","label","end"};
19 static void count(char*text, int len, int condition)
32 static char*prefix = 0;
34 static char utf8buf[16];
35 static char* getUTF8(unsigned int charnum)
37 memset(utf8buf, 0, sizeof(utf8buf));
42 } else if(charnum <0x800) {
43 /* 0000 0080-0000 07FF 110xxxxx 10xxxxxx */
44 utf8buf[0] = 0xc0 | (charnum >> 6);
45 utf8buf[1] = 0x80 | (charnum & 0x3f);
47 } else if(charnum < 0x10000) {
48 /* 0000 0800-0000 FFFF 1110xxxx 10xxxxxx 10xxxxxx */
49 utf8buf[0] = 0xe0 | (charnum >> 12);
50 utf8buf[1] = 0x80 |((charnum >> 6)&0x3f);
51 utf8buf[2] = 0x80 |((charnum )&0x3f);
53 } else if(charnum < 0x200000) {
54 /* 0001 0000-001F FFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
55 utf8buf[0] = 0xf0 | (charnum >> 18);
56 utf8buf[1] = 0x80 |((charnum >> 12)&0x3f);
57 utf8buf[2] = 0x80 |((charnum >> 6 )&0x3f);
58 utf8buf[3] = 0x80 |((charnum )&0x3f);
60 } else if(charnum < 0x4000000) {
61 /* 0020 0000-03FF FFFF 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */
62 utf8buf[0] = 0xf8 | (charnum >> 24);
63 utf8buf[1] = 0x80 |((charnum >> 18)&0x3f);
64 utf8buf[2] = 0x80 |((charnum >> 12)&0x3f);
65 utf8buf[3] = 0x80 |((charnum >> 6 )&0x3f);
66 utf8buf[4] = 0x80 |((charnum )&0x3f);
68 } else if(charnum < 0x80000000) {
69 /* 0400 0000-7FFF FFFF 1111110x 10xxxxxx ... 10xxxxxx */
70 utf8buf[0] = 0xfc | (charnum >> 30);
71 utf8buf[1] = 0x80 |((charnum >> 24)&0x3f);
72 utf8buf[2] = 0x80 |((charnum >> 18)&0x3f);
73 utf8buf[3] = 0x80 |((charnum >> 12)&0x3f);
74 utf8buf[4] = 0x80 |((charnum >> 6 )&0x3f);
75 utf8buf[5] = 0x80 |((charnum )&0x3f);
78 fprintf(stderr, "Illegal character: 0x%08x\n", charnum);
83 static void unescapeString(string_t * tmp)
86 /* fixme - this routine expects the string to be
89 for (p1=tmp->str; (p=strchr(p1, '\\')); p1 = p+1)
95 case '\\': p[0] = '\\'; break;
96 case '"': p[0] = '"'; break;
97 case 'b': p[0] = '\b'; break;
98 case 'f': p[0] = '\f'; break;
99 case 'n': p[0] = '\n'; break;
100 case 'r': p[0] = '\r'; break;
101 case 't': p[0] = '\t'; break;
102 case 'x': case 'u': {
108 while(strchr("0123456789abcdefABCDEF", p[nr]) && nr < max+2) {
110 if(p[nr]>='0' && p[nr]<='9') num |= p[nr] - '0';
111 if(p[nr]>='a' && p[nr]<='f') num |= p[nr] - 'a' + 10;
112 if(p[nr]>='A' && p[nr]<='F') num |= p[nr] - 'A' + 10;
117 memcpy(p, utf8, new); // do not copy the terminating zero
123 tmp->len -= (nr-new);
126 char*to=p+new,*from=p+nr;
136 static void store(enum type_t type, int line, int column, char*text, int length)
138 struct token_t token;
142 token.column = column;
143 //printf("->%d(%s) %s\n", type, type_names[type], text);fflush(stdout);
148 string_set2(&tmp, "", 0);
149 token.text = (char*)mem_putstring(&strings, tmp);
152 string_set2(&tmp, text+1, length-2);
153 unescapeString(&tmp);
154 token.text = (char*)mem_putstring(&strings, tmp);
159 string_set2(&tmp, text, length);
162 token.text = (char*)mem_put(&strings, prefix, strlen(prefix));
163 mem_putstring(&strings, tmp);
165 token.text = (char*)mem_putstring(&strings, tmp);
170 string_set2(&tmp, text+1/*:*/, length-5/*.end*/);
171 token.text = (char*)mem_putstring(&strings, tmp);
174 string_set2(&tmp, text+1, length-1);
175 token.text = (char*)mem_putstring(&strings, tmp);
178 char*x = &text[length-1];
179 if(x[-1] == '-' || x[-1] == '+')
181 do{x--;} while(*x==32 || *x==10 || *x==13 || *x=='\t');
183 string_set2(&tmp, text, x-text);
184 token.text = (char*)mem_putstring(&strings, tmp);
185 /*char*y,*x = strchr(text, '=');
188 do{y--;} while(*y==32 || *y==10 || *y==13 || *y=='\t');
189 do{x++;} while(*x==32 || *x==10 || *x==13 || *x=='\t');
190 token.text1 = (char*)put(&strings, text, y-text + 1, 1);
191 token.text2 = (char*)put(&strings, x, length-(x-text), 1);*/
195 mem_put(&tokens, &token, sizeof(struct token_t));
199 #define MAX_INCLUDE_DEPTH 16
200 YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH];
201 int line_stack[MAX_INCLUDE_DEPTH];
202 int column_stack[MAX_INCLUDE_DEPTH];
203 int include_stack_ptr = 0;
205 void handleInclude(char*text, int len)
208 while(len >=1 && (text[0] == ' ' || text[0] == '\t')) {
211 while(len >= 1 && (text[len-1] == ' ' || text[len-1] == '\n')) {
214 if(len >= 2 && text[0] == '"' && text[len-1] == '"') {
218 if(include_stack_ptr >= MAX_INCLUDE_DEPTH) {
219 fprintf( stderr, "Includes nested too deeply" );
222 include_stack[include_stack_ptr] = YY_CURRENT_BUFFER;
223 line_stack[include_stack_ptr] = line;
224 column_stack[include_stack_ptr] = column;
226 yyin = fopen(text, "rb");
228 fprintf(stderr, "Couldn't open %s\n", text);
232 yy_create_buffer( yyin, YY_BUF_SIZE ) );
236 #define c() {count(yytext, yyleng, YY_START);}
237 #define s(type) {store(type, line, column, yytext, yyleng);}
243 NAME [a-zA-Z_./](-*[a-zA-Z0-9_./])*
244 TWIP (-?[0-9]+(\.([0-9]([05])?)?)?)
245 NUMBER -?[0-9]+(\.[0-9]*)?
247 STRING (\\.|[^\\"\n])*
249 RVALUE \"{STRING}\"|([^ \n\r\t]+)
253 <BINARY>\] {c();BEGIN(0);}
256 {TWIP}/[ \n\r\t] {s(TWIP);c();BEGIN(0);}
257 {NUMBER}/[ \n\r\t] {s(NUMBER);c();BEGIN(0);}
259 [ \t\r]#[^\n]*\n {c();}
260 \"{STRING}\" {s(STRING);c();BEGIN(0);}
261 \"{STRING}$ {c();printf("unterminated string in line %d: %s\n", line, yytext);exit(1);yyterminate();}
262 {NAME}{S}*\+= {s(ASSIGNMENT);prefix="<plus>";c();BEGIN(R);}
263 {NAME}{S}*-= {s(ASSIGNMENT);prefix="<minus>";c();BEGIN(R);}
264 {NAME}{S}*= {s(ASSIGNMENT);c();BEGIN(R);}
265 <R>{ /* values which appear only on the right-hand side of assignments, like: x=50% */
266 [^ :\n\t\r]* {s(IDENTIFIER);c();BEGIN(0);}
268 \.include{S}.*\n {handleInclude(yytext, yyleng);}
269 \.{NAME} {s(COMMAND);c();}
270 :([^.]|\.[^e]|\.e[^n]|\.en[^d]|\.end[^ \n\r\t]|[ \n\r\t])*\.end {s(RAWDATA);c();}
271 {NAME} {s(IDENTIFIER);c();}
272 "[" {c();BEGIN(BINARY);}
274 . {char c,c1=yytext[0];
275 printf("Syntax error in line %d, %d: %s", line, column, yytext);
278 if(c=='\n' || c==EOF)
282 if(c1>='0' && c1<='9')
283 printf(" (identifiers must not start with a digit)");
289 if ( --include_stack_ptr < 0 ) {
293 yy_delete_buffer( YY_CURRENT_BUFFER );
295 include_stack[include_stack_ptr] );
296 column = column_stack[include_stack_ptr];
297 line = line_stack[include_stack_ptr];
308 void freeTokens(struct token_t*file)
314 struct token_t* generateTokens(char*filename)
316 FILE*fi = fopen(filename, "rb");
318 struct token_t*result;
321 printf("Couldn't find file %s\n", filename);
328 mem_put(&strings, &t, 1); //hack- make all valid strings start at position >0
334 yy_delete_buffer(yy_current_buffer);
336 result = (struct token_t*)tokens.buffer;
337 num = tokens.pos/sizeof(struct token_t);
339 for(t=0;t<tokens.pos/sizeof(struct token_t);t++) {
341 result[t].text += (int)strings.buffer;