noit_tokenizer.c

Revision 1d69a01311e58ce6ddb007d690b4e9aa4ac08c8c, 6.2 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 5 years ago)
And make it all work on OpenBSD (ld -E ? sigh...)
Property mode set to `100644`

Line
1	/* Generated by re2c 0.12.0 on Fri Feb 8 14:52:45 2008 */
2	#line 1 "noit_tokenizer.re"
3	#include <stdlib.h>
4	#include <string.h>
5
6	struct token {
7	char *token;
8	const char *start;
9	const char *end;
10	const char *next;
11	enum { NT_IDENT, NT_DQSTRING, NT_SPACE, NT_UNKNOWN, NT_EOF } type;
12	};
13	#define SET_TOKEN(t,a) (t)->next = (a)
14
15	static void c_unescape(char p, char only) {
16	char *bt = p;
17	#define ASSIGN(a) *(bt++) = (a)
18	while(p[0] != '\0') {
19	if(p[0] == '\\' && p[1] != '\0' && (!only \|\| p[1] == *only)) {
20	switch(p[1]) {
21	case ' ': ASSIGN(' '); p+=2; break;
22	case '"': ASSIGN('"'); p+=2; break;
23	case 'n': ASSIGN('\n'); p+=2; break;
24	case 'r': ASSIGN('\r'); p+=2; break;
25	case 't': ASSIGN('\t'); p+=2; break;
26	case 'a': ASSIGN('\a'); p+=2; break;
27	case 'b': ASSIGN('\b'); p+=2; break;
28	case 'v': ASSIGN('\v'); p+=2; break;
29	case 'f': ASSIGN('\f'); p+=2; break;
30	case '0': ASSIGN('\0'); p+=2; break;
31	case '\\': ASSIGN('\\'); p+=2; break;
32	default: ASSIGN(p); p++; ASSIGN(p); p++; break;
33	}
34	}
35	else {
36	ASSIGN(*p); p++;
37	}
38	}
39	*bt = '\0';
40	}
41
42	#define BAIL_UNKNOWN do { t->type = NT_UNKNOWN; return -1; } while(0)
43	static int token_scan(struct token *t)
44	{
45	t->start = t->end = t->next;
46
47	mainpattern:
48
49	#line 50 "noit_tokenizer.c"
50	{
51	unsigned char yych;
52
53	yych = (unsigned char)*t->next;
54	switch(yych) {
55	case 0x00: goto yy10;
56	case 0x09:
57	case 0x0A:
58	case 0x0D:
59	case ' ': goto yy2;
60	case '"': goto yy4;
61	case '\'': goto yy6;
62	default: goto yy8;
63	}
64	yy2:
65	++t->next;
66	yych = (unsigned char)*t->next;
67	goto yy17;
68	yy3:
69	#line 53 "noit_tokenizer.re"
70	{ t->token = NULL;
71	t->end = t->next;
72	t->type = NT_SPACE;
73	return 1; }
74	#line 75 "noit_tokenizer.c"
75	yy4:
76	++t->next;
77	#line 57 "noit_tokenizer.re"
78	{ t->type = NT_DQSTRING;
79	if(t->start != t->end) {
80	t->start++;
81	t->end = t->next - 1;
82	t->token = malloc(t->end-t->start + 1);
83	strlcpy(t->token, t->start, t->end-t->start + 1);
84	c_unescape(t->token, NULL);
85	return 1;
86	}
87	else
88	goto dqstring;
89	}
90	#line 91 "noit_tokenizer.c"
91	yy6:
92	++t->next;
93	#line 69 "noit_tokenizer.re"
94	{ t->type = NT_IDENT;
95	if(t->start != t->end) {
96	t->start++;
97	t->end = t->next - 1;
98	t->token = malloc(t->end-t->start + 1);
99	strlcpy(t->token, t->start, t->end-t->start + 1);
100	return 1;
101	}
102	else
103	goto sqstring;
104	}
105	#line 106 "noit_tokenizer.c"
106	yy8:
107	++t->next;
108	yych = (unsigned char)*t->next;
109	goto yy13;
110	yy9:
111	#line 81 "noit_tokenizer.re"
112	{ char only = ' ';
113	t->end = t->next;
114	t->type = NT_IDENT;
115	t->token = malloc(t->end-t->start + 1);
116	strlcpy(t->token, t->start, t->end-t->start + 1);
117	c_unescape(t->token, &only);
118	return 1;
119	}
120	#line 121 "noit_tokenizer.c"
121	yy10:
122	++t->next;
123	#line 89 "noit_tokenizer.re"
124	{ t->token = NULL;
125	t->type = NT_EOF;
126	return 0;
127	}
128	#line 129 "noit_tokenizer.c"
129	yy12:
130	++t->next;
131	yych = (unsigned char)*t->next;
132	yy13:
133	switch(yych) {
134	case 0x00:
135	case 0x09:
136	case 0x0A:
137	case 0x0D:
138	case ' ': goto yy9;
139	case '\\': goto yy14;
140	default: goto yy12;
141	}
142	yy14:
143	++t->next;
144	yych = (unsigned char)*t->next;
145	switch(yych) {
146	case 0x00:
147	case 0x09:
148	case 0x0A:
149	case 0x0D: goto yy9;
150	case '\\': goto yy14;
151	default: goto yy12;
152	}
153	yy16:
154	++t->next;
155	yych = (unsigned char)*t->next;
156	yy17:
157	switch(yych) {
158	case 0x09:
159	case 0x0A:
160	case 0x0D:
161	case ' ': goto yy16;
162	default: goto yy3;
163	}
164	}
165	#line 94 "noit_tokenizer.re"
166
167
168	sqstring:
169
170	#line 171 "noit_tokenizer.c"
171	{
172	unsigned char yych;
173	yych = (unsigned char)*t->next;
174	switch(yych) {
175	case 0x00: goto yy23;
176	case '\'': goto yy20;
177	default: goto yy21;
178	}
179	yy20:
180	#line 98 "noit_tokenizer.re"
181	{ t->end = t->next;
182	goto mainpattern; }
183	#line 184 "noit_tokenizer.c"
184	yy21:
185	++t->next;
186	yych = (unsigned char)*t->next;
187	switch(yych) {
188	case 0x00:
189	case '\'': goto yy20;
190	default: goto yy21;
191	}
192	yy23:
193	++t->next;
194	#line 100 "noit_tokenizer.re"
195	{ BAIL_UNKNOWN; }
196	#line 197 "noit_tokenizer.c"
197	}
198	#line 101 "noit_tokenizer.re"
199
200
201	dqstring:
202
203	#line 204 "noit_tokenizer.c"
204	{
205	unsigned char yych;
206	yych = (unsigned char)*t->next;
207	switch(yych) {
208	case 0x00: goto yy33;
209	case '"': goto yy29;
210	case '\\': goto yy27;
211	default: goto yy31;
212	}
213	yy27:
214	yych = (unsigned char)*++t->next;
215	switch(yych) {
216	case 0x00: goto yy28;
217	case '"':
218	case '0':
219	case '\\':
220	case 'a':
221	case 'b':
222	case 'f':
223	case 'n':
224	case 'r':
225	case 't':
226	case 'v': goto yy37;
227	default: goto yy35;
228	}
229	yy28:
230	yy29:
231	++t->next;
232	#line 109 "noit_tokenizer.re"
233	{ t->end = t->next--;
234	goto mainpattern;
235	}
236	#line 237 "noit_tokenizer.c"
237	yy31:
238	++t->next;
239	#line 112 "noit_tokenizer.re"
240	{ goto dqstring; }
241	#line 242 "noit_tokenizer.c"
242	yy33:
243	++t->next;
244	#line 113 "noit_tokenizer.re"
245	{ BAIL_UNKNOWN; }
246	#line 247 "noit_tokenizer.c"
247	yy35:
248	++t->next;
249	#line 108 "noit_tokenizer.re"
250	{ goto dqstring; }
251	#line 252 "noit_tokenizer.c"
252	yy37:
253	++t->next;
254	#line 106 "noit_tokenizer.re"
255	{ goto dqstring; }
256	#line 257 "noit_tokenizer.c"
257	}
258	#line 114 "noit_tokenizer.re"
259
260	}
261
262	int noit_tokenize(const char input, char vector, int cnt) {
263	struct token t;
264	int i = 0;
265
266	SET_TOKEN(&t, input);
267	while(token_scan(&t) != -1) {
268	switch(t.type) {
269	case NT_IDENT:
270	case NT_DQSTRING:
271	if(i<*cnt) vector[i] = t.token;
272	i++;
273	break;
274	case NT_SPACE:
275	break;
276	case NT_EOF:
277	if(i<cnt) cnt = i;
278	return i;
279	case NT_UNKNOWN:
280	/* UNREACHED */
281	goto failure;
282	}
283	}
284	failure:
285	if(i<cnt) cnt = i;
286	return input - t.next;
287	}

Note: See TracBrowser for help on using the browser.

root/src/noit_tokenizer.c

Download in other formats: