URL https://opencores.org/ocsvn/eco32/eco32/trunk

Subversion Repositories eco32

[/] [eco32/] [trunk/] [lcc/] [cpp/] [tokens.c] - Blame information for rev 47

Go to most recent revision | Details | Compare with Previous | View Log


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cpp.h"
 
static char wbuf[2*OBS];
static char *wbp = wbuf;
 
/*
 * 1 for tokens that don't need whitespace when they get inserted
 * by macro expansion
 */
static const char wstab[] = {
        0,       /* END */
        0,       /* UNCLASS */
        0,       /* NAME */
        0,       /* NUMBER */
        0,       /* STRING */
        0,       /* CCON */
        1,      /* NL */
        0,       /* WS */
        0,       /* DSHARP */
        0,       /* EQ */
        0,       /* NEQ */
        0,       /* LEQ */
        0,       /* GEQ */
        0,       /* LSH */
        0,       /* RSH */
        0,       /* LAND */
        0,       /* LOR */
        0,       /* PPLUS */
        0,       /* MMINUS */
        0,       /* ARROW */
        1,      /* SBRA */
        1,      /* SKET */
        1,      /* LP */
        1,      /* RP */
        0,       /* DOT */
        0,       /* AND */
        0,       /* STAR */
        0,       /* PLUS */
        0,       /* MINUS */
        0,       /* TILDE */
        0,       /* NOT */
        0,       /* SLASH */
        0,       /* PCT */
        0,       /* LT */
        0,       /* GT */
        0,       /* CIRC */
        0,       /* OR */
        0,       /* QUEST */
        0,       /* COLON */
        0,       /* ASGN */
        1,      /* COMMA */
        0,       /* SHARP */
        1,      /* SEMIC */
        1,      /* CBRA */
        1,      /* CKET */
        0,       /* ASPLUS */
        0,       /* ASMINUS */
        0,       /* ASSTAR */
        0,       /* ASSLASH */
        0,       /* ASPCT */
        0,       /* ASCIRC */
        0,       /* ASLSH */
        0,       /* ASRSH */
        0,       /* ASOR */
        0,       /* ASAND */
        0,       /* ELLIPS */
        0,       /* DSHARP1 */
        0,       /* NAME1 */
        0,       /* DEFINED */
        0,       /* UMINUS */
};
 
void
maketokenrow(int size, Tokenrow *trp)
{
        trp->max = size;
        if (size>0)
                trp->bp = (Token *)domalloc(size*sizeof(Token));
        else
                trp->bp = NULL;
        trp->tp = trp->bp;
        trp->lp = trp->bp;
}
 
Token *
growtokenrow(Tokenrow *trp)
{
        int ncur = trp->tp - trp->bp;
        int nlast = trp->lp - trp->bp;
 
        trp->max = 3*trp->max/2 + 1;
        trp->bp = (Token *)realloc(trp->bp, trp->max*sizeof(Token));
        if (trp->bp == NULL)
                error(FATAL, "Out of memory from realloc");
        trp->lp = &trp->bp[nlast];
        trp->tp = &trp->bp[ncur];
        return trp->lp;
}
 
/*
 * Compare a row of tokens, ignoring the content of WS; return !=0 if different
 */
int
comparetokens(Tokenrow *tr1, Tokenrow *tr2)
{
        Token *tp1, *tp2;
 
        tp1 = tr1->tp;
        tp2 = tr2->tp;
        if (tr1->lp-tp1 != tr2->lp-tp2)
                return 1;
        for (; tp1<tr1->lp ; tp1++, tp2++) {
                if (tp1->type != tp2->type
                 || (tp1->wslen==0) != (tp2->wslen==0)
                 || tp1->len != tp2->len
                 || strncmp((char*)tp1->t, (char*)tp2->t, tp1->len)!=0)
                        return 1;
        }
        return 0;
}
 
/*
 * replace ntok tokens starting at dtr->tp with the contents of str.
 * tp ends up pointing just beyond the replacement.
 * Canonical whitespace is assured on each side.
 */
void
insertrow(Tokenrow *dtr, int ntok, Tokenrow *str)
{
        int nrtok = rowlen(str);
 
        dtr->tp += ntok;
        adjustrow(dtr, nrtok-ntok);
        dtr->tp -= ntok;
        movetokenrow(dtr, str);
        makespace(dtr);
        dtr->tp += nrtok;
        makespace(dtr);
}
 
/*
 * make sure there is WS before trp->tp, if tokens might merge in the output
 */
void
makespace(Tokenrow *trp)
{
        uchar *tt;
        Token *tp = trp->tp;
 
        if (tp >= trp->lp)
                return;
        if (tp->wslen) {
                if (tp->flag&XPWS
                 && (wstab[tp->type] || trp->tp>trp->bp && wstab[(tp-1)->type])) {
                        tp->wslen = 0;
                        return;
                }
                tp->t[-1] = ' ';
                return;
        }
        if (wstab[tp->type] || trp->tp>trp->bp && wstab[(tp-1)->type])
                return;
        tt = newstring(tp->t, tp->len, 1);
        *tt++ = ' ';
        tp->t = tt;
        tp->wslen = 1;
        tp->flag |= XPWS;
}
 
/*
 * Copy an entire tokenrow into another, at tp.
 * It is assumed that there is enough space.
 *  Not strictly conforming.
 */
void
movetokenrow(Tokenrow *dtr, Tokenrow *str)
{
        int nby;
 
        /* nby = sizeof(Token) * (str->lp - str->bp); */
        nby = (char *)str->lp - (char *)str->bp;
        memmove(dtr->tp, str->bp, nby);
}
 
/*
 * Move the tokens in a row, starting at tr->tp, rightward by nt tokens;
 * nt may be negative (left move).
 * The row may need to be grown.
 * Non-strictly conforming because of the (char *), but easily fixed
 */
void
adjustrow(Tokenrow *trp, int nt)
{
        int nby, size;
 
        if (nt==0)
                return;
        size = (trp->lp - trp->bp) + nt;
        while (size > trp->max)
                growtokenrow(trp);
        /* nby = sizeof(Token) * (trp->lp - trp->tp); */
        nby = (char *)trp->lp - (char *)trp->tp;
        if (nby)
                memmove(trp->tp+nt, trp->tp, nby);
        trp->lp += nt;
}
 
/*
 * Copy a row of tokens into the destination holder, allocating
 * the space for the contents.  Return the destination.
 */
Tokenrow *
copytokenrow(Tokenrow *dtr, Tokenrow *str)
{
        int len = rowlen(str);
 
        maketokenrow(len, dtr);
        movetokenrow(dtr, str);
        dtr->lp += len;
        return dtr;
}
 
/*
 * Produce a copy of a row of tokens.  Start at trp->tp.
 * The value strings are copied as well.  The first token
 * has WS available.
 */
Tokenrow *
normtokenrow(Tokenrow *trp)
{
        Token *tp;
        Tokenrow *ntrp = new(Tokenrow);
        int len;
 
        len = trp->lp - trp->tp;
        if (len<=0)
                len = 1;
        maketokenrow(len, ntrp);
        for (tp=trp->tp; tp < trp->lp; tp++) {
                *ntrp->lp = *tp;
                if (tp->len) {
                        ntrp->lp->t = newstring(tp->t, tp->len, 1);
                        *ntrp->lp->t++ = ' ';
                        if (tp->wslen)
                                ntrp->lp->wslen = 1;
                }
                ntrp->lp++;
        }
        if (ntrp->lp > ntrp->bp)
                ntrp->bp->wslen = 0;
        return ntrp;
}
 
/*
 * Debugging
 */
void
peektokens(Tokenrow *trp, char *str)
{
        Token *tp;
 
        tp = trp->tp;
        flushout();
        if (str)
                fprintf(stderr, "%s ", str);
        if (tp<trp->bp || tp>trp->lp)
                fprintf(stderr, "(tp offset %d) ", tp-trp->bp);
        for (tp=trp->bp; tp<trp->lp && tp<trp->bp+32; tp++) {
                if (tp->type!=NL) {
                        int c = tp->t[tp->len];
                        tp->t[tp->len] = 0;
                        fprintf(stderr, "%s", tp->t);
                        tp->t[tp->len] = c;
                }
                if (tp->type==NAME) {
                        fprintf(stderr, tp==trp->tp?"{*":"{");
                        prhideset(tp->hideset);
                        fprintf(stderr, "} ");
                } else
                        fprintf(stderr, tp==trp->tp?"{%x*} ":"{%x} ", tp->type);
        }
        fprintf(stderr, "\n");
        fflush(stderr);
}
 
void
puttokens(Tokenrow *trp)
{
        Token *tp;
        int len;
        uchar *p;
 
        if (verbose)
                peektokens(trp, "");
        tp = trp->bp;
        for (; tp<trp->lp; tp++) {
                len = tp->len+tp->wslen;
                p = tp->t-tp->wslen;
                while (tp<trp->lp-1 && p+len == (tp+1)->t - (tp+1)->wslen) {
                        tp++;
                        len += tp->wslen+tp->len;
                }
                if (len>OBS/2) {                /* handle giant token */
                        if (wbp > wbuf)
                                fwrite(wbuf, 1, wbp-wbuf, stdout);
                        fwrite((char *)p, 1, len, stdout);
                        wbp = wbuf;
                } else {
                        memcpy(wbp, p, len);
                        wbp += len;
                }
                if (wbp >= &wbuf[OBS]) {
                        fwrite(wbuf, 1, OBS, stdout);
                        if (wbp > &wbuf[OBS])
                                memcpy(wbuf, wbuf+OBS, wbp - &wbuf[OBS]);
                        wbp -= OBS;
                }
        }
        trp->tp = tp;
        if (cursource->fd==stdin)
                flushout();
}
 
void
flushout(void)
{
        if (wbp>wbuf) {
                fwrite(wbuf, 1, wbp-wbuf, stdout);
                fflush(stdout);
                wbp = wbuf;
        }
}
 
/*
 * turn a row into just a newline
 */
void
setempty(Tokenrow *trp)
{
        trp->tp = trp->bp;
        trp->lp = trp->bp+1;
        *trp->bp = nltoken;
}
 
/*
 * generate a number
 */
char *
outnum(char *p, int n)
{
        if (n>=10)
                p = outnum(p, n/10);
        *p++ = n%10 + '0';
        return p;
}
 
/*
 * allocate and initialize a new string from s, of length l, at offset o
 * Null terminated.
 */
uchar *
newstring(uchar *s, int l, int o)
{
        uchar *ns = (uchar *)domalloc(l+o+1);
 
        ns[l+o] = '\0';
        return (uchar*)strncpy((char*)ns+o, (char*)s, l) - o;
}

Browse

Tools

Subversion Repositories eco32

[/] [eco32/] [trunk/] [lcc/] [cpp/] [tokens.c] - Blame information for rev 47

Line No.	Rev	Author	Line
1	4	hellwig	`#include <stdio.h>`
2			`#include <stdlib.h>`
3			`#include <string.h>`
4			`#include "cpp.h"`
5
6			`static char wbuf[2*OBS];`
7			`static char *wbp = wbuf;`
8
9			`/*`
10			`* 1 for tokens that don't need whitespace when they get inserted`
11			`* by macro expansion`
12			`*/`
13			`static const char wstab[] = {`
14			`0, /* END */`
15			`0, /* UNCLASS */`
16			`0, /* NAME */`
17			`0, /* NUMBER */`
18			`0, /* STRING */`
19			`0, /* CCON */`
20			`1, /* NL */`
21			`0, /* WS */`
22			`0, /* DSHARP */`
23			`0, /* EQ */`
24			`0, /* NEQ */`
25			`0, /* LEQ */`
26			`0, /* GEQ */`
27			`0, /* LSH */`
28			`0, /* RSH */`
29			`0, /* LAND */`
30			`0, /* LOR */`
31			`0, /* PPLUS */`
32			`0, /* MMINUS */`
33			`0, /* ARROW */`
34			`1, /* SBRA */`
35			`1, /* SKET */`
36			`1, /* LP */`
37			`1, /* RP */`
38			`0, /* DOT */`
39			`0, /* AND */`
40			`0, /* STAR */`
41			`0, /* PLUS */`
42			`0, /* MINUS */`
43			`0, /* TILDE */`
44			`0, /* NOT */`
45			`0, /* SLASH */`
46			`0, /* PCT */`
47			`0, /* LT */`
48			`0, /* GT */`
49			`0, /* CIRC */`
50			`0, /* OR */`
51			`0, /* QUEST */`
52			`0, /* COLON */`
53			`0, /* ASGN */`
54			`1, /* COMMA */`
55			`0, /* SHARP */`
56			`1, /* SEMIC */`
57			`1, /* CBRA */`
58			`1, /* CKET */`
59			`0, /* ASPLUS */`
60			`0, /* ASMINUS */`
61			`0, /* ASSTAR */`
62			`0, /* ASSLASH */`
63			`0, /* ASPCT */`
64			`0, /* ASCIRC */`
65			`0, /* ASLSH */`
66			`0, /* ASRSH */`
67			`0, /* ASOR */`
68			`0, /* ASAND */`
69			`0, /* ELLIPS */`
70			`0, /* DSHARP1 */`
71			`0, /* NAME1 */`
72			`0, /* DEFINED */`
73			`0, /* UMINUS */`
74			`};`
75
76			`void`
77			`maketokenrow(int size, Tokenrow *trp)`
78			`{`
79			`trp->max = size;`
80			`if (size>0)`
81			`trp->bp = (Token )domalloc(sizesizeof(Token));`
82			`else`
83			`trp->bp = NULL;`
84			`trp->tp = trp->bp;`
85			`trp->lp = trp->bp;`
86			`}`
87
88			`Token *`
89			`growtokenrow(Tokenrow *trp)`
90			`{`
91			`int ncur = trp->tp - trp->bp;`
92			`int nlast = trp->lp - trp->bp;`
93
94			`trp->max = 3*trp->max/2 + 1;`
95			`trp->bp = (Token )realloc(trp->bp, trp->maxsizeof(Token));`
96			`if (trp->bp == NULL)`
97			`error(FATAL, "Out of memory from realloc");`
98			`trp->lp = &trp->bp[nlast];`
99			`trp->tp = &trp->bp[ncur];`
100			`return trp->lp;`
101			`}`
102
103			`/*`
104			`* Compare a row of tokens, ignoring the content of WS; return !=0 if different`
105			`*/`
106			`int`
107			`comparetokens(Tokenrow tr1, Tokenrow tr2)`
108			`{`
109			`Token tp1, tp2;`
110
111			`tp1 = tr1->tp;`
112			`tp2 = tr2->tp;`
113			`if (tr1->lp-tp1 != tr2->lp-tp2)`
114			`return 1;`
115			`for (; tp1<tr1->lp ; tp1++, tp2++) {`
116			`if (tp1->type != tp2->type`
117			`\|\| (tp1->wslen==0) != (tp2->wslen==0)`
118			`\|\| tp1->len != tp2->len`
119			`\|\| strncmp((char)tp1->t, (char)tp2->t, tp1->len)!=0)`
120			`return 1;`
121			`}`
122			`return 0;`
123			`}`
124
125			`/*`
126			`* replace ntok tokens starting at dtr->tp with the contents of str.`
127			`* tp ends up pointing just beyond the replacement.`
128			`* Canonical whitespace is assured on each side.`
129			`*/`
130			`void`
131			`insertrow(Tokenrow dtr, int ntok, Tokenrow str)`
132			`{`
133			`int nrtok = rowlen(str);`
134
135			`dtr->tp += ntok;`
136			`adjustrow(dtr, nrtok-ntok);`
137			`dtr->tp -= ntok;`
138			`movetokenrow(dtr, str);`
139			`makespace(dtr);`
140			`dtr->tp += nrtok;`
141			`makespace(dtr);`
142			`}`
143
144			`/*`
145			`* make sure there is WS before trp->tp, if tokens might merge in the output`
146			`*/`
147			`void`
148			`makespace(Tokenrow *trp)`
149			`{`
150			`uchar *tt;`
151			`Token *tp = trp->tp;`
152
153			`if (tp >= trp->lp)`
154			`return;`
155			`if (tp->wslen) {`
156			`if (tp->flag&XPWS`
157			`&& (wstab[tp->type] \|\| trp->tp>trp->bp && wstab[(tp-1)->type])) {`
158			`tp->wslen = 0;`
159			`return;`
160			`}`
161			`tp->t[-1] = ' ';`
162			`return;`
163			`}`
164			`if (wstab[tp->type] \|\| trp->tp>trp->bp && wstab[(tp-1)->type])`
165			`return;`
166			`tt = newstring(tp->t, tp->len, 1);`
167			`*tt++ = ' ';`
168			`tp->t = tt;`
169			`tp->wslen = 1;`
170			`tp->flag \|= XPWS;`
171			`}`
172
173			`/*`
174			`* Copy an entire tokenrow into another, at tp.`
175			`* It is assumed that there is enough space.`
176			`* Not strictly conforming.`
177			`*/`
178			`void`
179			`movetokenrow(Tokenrow dtr, Tokenrow str)`
180			`{`
181			`int nby;`
182
183			`/* nby = sizeof(Token) * (str->lp - str->bp); */`
184			`nby = (char )str->lp - (char )str->bp;`
185			`memmove(dtr->tp, str->bp, nby);`
186			`}`
187
188			`/*`
189			`* Move the tokens in a row, starting at tr->tp, rightward by nt tokens;`
190			`* nt may be negative (left move).`
191			`* The row may need to be grown.`
192			`* Non-strictly conforming because of the (char *), but easily fixed`
193			`*/`
194			`void`
195			`adjustrow(Tokenrow *trp, int nt)`
196			`{`
197			`int nby, size;`
198
199			`if (nt==0)`
200			`return;`
201			`size = (trp->lp - trp->bp) + nt;`
202			`while (size > trp->max)`
203			`growtokenrow(trp);`
204			`/* nby = sizeof(Token) * (trp->lp - trp->tp); */`
205			`nby = (char )trp->lp - (char )trp->tp;`
206			`if (nby)`
207			`memmove(trp->tp+nt, trp->tp, nby);`
208			`trp->lp += nt;`
209			`}`
210
211			`/*`
212			`* Copy a row of tokens into the destination holder, allocating`
213			`* the space for the contents. Return the destination.`
214			`*/`
215			`Tokenrow *`
216			`copytokenrow(Tokenrow dtr, Tokenrow str)`
217			`{`
218			`int len = rowlen(str);`
219
220			`maketokenrow(len, dtr);`
221			`movetokenrow(dtr, str);`
222			`dtr->lp += len;`
223			`return dtr;`
224			`}`
225
226			`/*`
227			`* Produce a copy of a row of tokens. Start at trp->tp.`
228			`* The value strings are copied as well. The first token`
229			`* has WS available.`
230			`*/`
231			`Tokenrow *`
232			`normtokenrow(Tokenrow *trp)`
233			`{`
234			`Token *tp;`
235			`Tokenrow *ntrp = new(Tokenrow);`
236			`int len;`
237
238			`len = trp->lp - trp->tp;`
239			`if (len<=0)`
240			`len = 1;`
241			`maketokenrow(len, ntrp);`
242			`for (tp=trp->tp; tp < trp->lp; tp++) {`
243			`ntrp->lp = tp;`
244			`if (tp->len) {`
245			`ntrp->lp->t = newstring(tp->t, tp->len, 1);`
246			`*ntrp->lp->t++ = ' ';`
247			`if (tp->wslen)`
248			`ntrp->lp->wslen = 1;`
249			`}`
250			`ntrp->lp++;`
251			`}`
252			`if (ntrp->lp > ntrp->bp)`
253			`ntrp->bp->wslen = 0;`
254			`return ntrp;`
255			`}`
256
257			`/*`
258			`* Debugging`
259			`*/`
260			`void`
261			`peektokens(Tokenrow trp, char str)`
262			`{`
263			`Token *tp;`
264
265			`tp = trp->tp;`
266			`flushout();`
267			`if (str)`
268			`fprintf(stderr, "%s ", str);`
269			`if (tp<trp->bp \|\| tp>trp->lp)`
270			`fprintf(stderr, "(tp offset %d) ", tp-trp->bp);`
271			`for (tp=trp->bp; tp<trp->lp && tp<trp->bp+32; tp++) {`
272			`if (tp->type!=NL) {`
273			`int c = tp->t[tp->len];`
274			`tp->t[tp->len] = 0;`
275			`fprintf(stderr, "%s", tp->t);`
276			`tp->t[tp->len] = c;`
277			`}`
278			`if (tp->type==NAME) {`
279			`fprintf(stderr, tp==trp->tp?"{*":"{");`
280			`prhideset(tp->hideset);`
281			`fprintf(stderr, "} ");`
282			`} else`
283			`fprintf(stderr, tp==trp->tp?"{%x*} ":"{%x} ", tp->type);`
284			`}`
285			`fprintf(stderr, "\n");`
286			`fflush(stderr);`
287			`}`
288
289			`void`
290			`puttokens(Tokenrow *trp)`
291			`{`
292			`Token *tp;`
293			`int len;`
294			`uchar *p;`
295
296			`if (verbose)`
297			`peektokens(trp, "");`
298			`tp = trp->bp;`
299			`for (; tp<trp->lp; tp++) {`
300			`len = tp->len+tp->wslen;`
301			`p = tp->t-tp->wslen;`
302			`while (tp<trp->lp-1 && p+len == (tp+1)->t - (tp+1)->wslen) {`
303			`tp++;`
304			`len += tp->wslen+tp->len;`
305			`}`
306			`if (len>OBS/2) { /* handle giant token */`
307			`if (wbp > wbuf)`
308			`fwrite(wbuf, 1, wbp-wbuf, stdout);`
309			`fwrite((char *)p, 1, len, stdout);`
310			`wbp = wbuf;`
311			`} else {`
312			`memcpy(wbp, p, len);`
313			`wbp += len;`
314			`}`
315			`if (wbp >= &wbuf[OBS]) {`
316			`fwrite(wbuf, 1, OBS, stdout);`
317			`if (wbp > &wbuf[OBS])`
318			`memcpy(wbuf, wbuf+OBS, wbp - &wbuf[OBS]);`
319			`wbp -= OBS;`
320			`}`
321			`}`
322			`trp->tp = tp;`
323			`if (cursource->fd==stdin)`
324			`flushout();`
325			`}`
326
327			`void`
328			`flushout(void)`
329			`{`
330			`if (wbp>wbuf) {`
331			`fwrite(wbuf, 1, wbp-wbuf, stdout);`
332			`fflush(stdout);`
333			`wbp = wbuf;`
334			`}`
335			`}`
336
337			`/*`
338			`* turn a row into just a newline`
339			`*/`
340			`void`
341			`setempty(Tokenrow *trp)`
342			`{`
343			`trp->tp = trp->bp;`
344			`trp->lp = trp->bp+1;`
345			`*trp->bp = nltoken;`
346			`}`
347
348			`/*`
349			`* generate a number`
350			`*/`
351			`char *`
352			`outnum(char *p, int n)`
353			`{`
354			`if (n>=10)`
355			`p = outnum(p, n/10);`
356			`*p++ = n%10 + '0';`
357			`return p;`
358			`}`
359
360			`/*`
361			`* allocate and initialize a new string from s, of length l, at offset o`
362			`* Null terminated.`
363			`*/`
364			`uchar *`
365			`newstring(uchar *s, int l, int o)`
366			`{`
367			`uchar ns = (uchar )domalloc(l+o+1);`
368
369			`ns[l+o] = '\0';`
370			`return (uchar)strncpy((char)ns+o, (char*)s, l) - o;`
371			`}`