URL https://opencores.org/ocsvn/openrisc_2011-10-31/openrisc_2011-10-31/trunk

Subversion Repositories openrisc_2011-10-31

[/] [openrisc/] [trunk/] [rtos/] [rtems/] [c/] [src/] [libnetworking/] [rtems_webserver/] [ejlex.c] - Blame information for rev 261

Go to most recent revision | Details | Compare with Previous | View Log


/*
 * ejlex.c -- Ejscript(TM) Lexical Analyser
 *
 * Copyright (c) Go Ahead Software, Inc., 1995-1999
 *
 * See the file "license.txt" for usage and redistribution license requirements
 */
 
/******************************** Description *********************************/
 
/*
 *      Ejscript lexical analyser. This implementes a lexical analyser for a
 *      a subset of the JavaScript language.
 */
 
/********************************** Includes **********************************/
 
#include        "ej.h"
 
#if UEMF
        #include "uemf.h"
#else
        #include "basic/basicInternal.h"
#endif
 
/****************************** Forward Declarations **************************/
 
static int              getLexicalToken(ej_t* ep, int state);
static int              tokenAddChar(ej_t *ep, int c);
static int              inputGetc(ej_t* ep);
static void             inputPutback(ej_t* ep, int c);
 
/************************************* Code ***********************************/
/*
 *      Setup the lexical analyser
 */
 
int ejLexOpen(ej_t* ep)
{
        return 0;
}
 
/******************************************************************************/
/*
 *      Close the lexicial analyser
 */
 
void ejLexClose(ej_t* ep)
{
}
 
/******************************************************************************/
/*
 *      Open a new input script
 */
 
int ejLexOpenScript(ej_t* ep, char_t *script)
{
        ejinput_t       *ip;
 
        a_assert(ep);
        a_assert(script);
 
        if ((ep->input = balloc(B_L, sizeof(ejinput_t))) == NULL) {
                return -1;
        }
        ip = ep->input;
        memset(ip, 0, sizeof(*ip));
 
        a_assert(ip);
        a_assert(ip->putBackToken == NULL);
        a_assert(ip->putBackTokenId == 0);
 
/*
 *      Create the parse token buffer and script buffer
 */
        if (ringqOpen(&ip->tokbuf, EJ_INC, -1) < 0) {
                return -1;
        }
        if (ringqOpen(&ip->script, EJ_INC, -1) < 0) {
                return -1;
        }
/*
 *      Put the Ejscript into a ring queue for easy parsing
 */
        ringqPutstr(&ip->script, script);
 
        ip->lineNumber = 1;
        ip->lineLength = 0;
        ip->lineColumn = 0;
        ip->line = NULL;
 
        return 0;
}
 
/******************************************************************************/
/*
 *      Close the input script
 */
 
void ejLexCloseScript(ej_t* ep)
{
        ejinput_t       *ip;
 
        a_assert(ep);
 
        ip = ep->input;
        a_assert(ip);
 
        if (ip->putBackToken) {
                bfree(B_L, ip->putBackToken);
                ip->putBackToken = NULL;
        }
        ip->putBackTokenId = 0;
 
        if (ip->line) {
                bfree(B_L, ip->line);
                ip->line = NULL;
        }
 
        ringqClose(&ip->tokbuf);
        ringqClose(&ip->script);
 
        bfree(B_L, ip);
}
 
/******************************************************************************/
/*
 *      Save the input state
 */
 
void ejLexSaveInputState(ej_t* ep, ejinput_t* state)
{
        ejinput_t       *ip;
 
        a_assert(ep);
 
        ip = ep->input;
        a_assert(ip);
 
        *state = *ip;
        if (ip->putBackToken) {
                state->putBackToken = bstrdup(B_L, ip->putBackToken);
        }
}
 
/******************************************************************************/
/*
 *      Restore the input state
 */
 
void ejLexRestoreInputState(ej_t* ep, ejinput_t* state)
{
        ejinput_t       *ip;
 
        a_assert(ep);
 
        ip = ep->input;
        a_assert(ip);
 
        ip->tokbuf = state->tokbuf;
        ip->script = state->script;
        ip->putBackTokenId = state->putBackTokenId;
        if (ip->putBackToken) {
                bfree(B_L, ip->putBackToken);
        }
        if (state->putBackToken) {
                ip->putBackToken = bstrdup(B_L, state->putBackToken);
        }
}
 
/******************************************************************************/
/*
 *      Free a saved input state
 */
 
void ejLexFreeInputState(ej_t* ep, ejinput_t* state)
{
        if (state->putBackToken) {
                bfree(B_L, state->putBackToken);
        }
}
 
/******************************************************************************/
/*
 *      Get the next Ejscript token
 */
 
int ejLexGetToken(ej_t* ep, int state)
{
        ep->tid = getLexicalToken(ep, state);
        goahead_trace(7, T("ejGetToken: %d, \"%s\"\n"), ep->tid, ep->token);
        return ep->tid;
}
 
/******************************************************************************/
/*
 *      Get the next Ejscript token
 */
 
static int getLexicalToken(ej_t* ep, int state)
{
        ringq_t         *inq, *tokq;
        ejinput_t*      ip;
        int                     done, tid, c, quote, style, back_quoted, lval, i;
 
        a_assert(ep);
        ip = ep->input;
        a_assert(ip);
 
        inq = &ip->script;
        tokq = &ip->tokbuf;
 
        ep->tid = -1;
        tid = -1;
        ep->token = T("");
 
        ringqFlush(tokq);
 
        if (ip->putBackTokenId > 0) {
                ringqPutstr(tokq, ip->putBackToken);
                tid = ip->putBackTokenId;
                ip->putBackTokenId = 0;
                ep->token = (char_t*) tokq->servp;
                return tid;
        }
 
        if ((c = inputGetc(ep)) < 0) {
                return TOK_EOF;
        }
 
        for (done = 0; !done; ) {
                switch (c) {
                case -1:
                        return TOK_EOF;
 
                case ' ':
                case '\t':
                case '\r':
                        do {
                                if ((c = inputGetc(ep)) < 0)
                                        break;
                        } while (c == ' ' || c == '\t' || c == '\r');
                        break;
 
                case '\n':
                        return TOK_NEWLINE;
 
                case '(':
                        tokenAddChar(ep, c);
                        return TOK_LPAREN;
 
                case ')':
                        tokenAddChar(ep, c);
                        return TOK_RPAREN;
 
                case '{':
                        tokenAddChar(ep, c);
                        return TOK_LBRACE;
 
                case '}':
                        tokenAddChar(ep, c);
                        return TOK_RBRACE;
 
                case '+':
                        if ((c = inputGetc(ep)) < 0) {
                                ejError(ep, T("Syntax Error"));
                                return TOK_ERR;
                        }
                        if (c != '+' ) {
                                inputPutback(ep, c);
                                tokenAddChar(ep, EXPR_PLUS);
                                return TOK_EXPR;
                        }
                        tokenAddChar(ep, EXPR_INC);
                        return TOK_INC_DEC;
 
                case '-':
                        if ((c = inputGetc(ep)) < 0) {
                                ejError(ep, T("Syntax Error"));
                                return TOK_ERR;
                        }
                        if (c != '-' ) {
                                inputPutback(ep, c);
                                tokenAddChar(ep, EXPR_MINUS);
                                return TOK_EXPR;
                        }
                        tokenAddChar(ep, EXPR_DEC);
                        return TOK_INC_DEC;
 
                case '*':
                        tokenAddChar(ep, EXPR_MUL);
                        return TOK_EXPR;
 
                case '%':
                        tokenAddChar(ep, EXPR_MOD);
                        return TOK_EXPR;
 
                case '/':
/*
 *                      Handle the division operator and comments
 */
                        if ((c = inputGetc(ep)) < 0) {
                                ejError(ep, T("Syntax Error"));
                                return TOK_ERR;
                        }
                        if (c != '*' && c != '/') {
                                inputPutback(ep, c);
                                tokenAddChar(ep, EXPR_DIV);
                                return TOK_EXPR;
                        }
                        style = c;
/*
 *                      Eat comments. Both C and C++ comment styles are supported.
 */
                        while (1) {
                                if ((c = inputGetc(ep)) < 0) {
                                        ejError(ep, T("Syntax Error"));
                                        return TOK_ERR;
                                }
                                if (c == '\n' && style == '/') {
                                        break;
                                } else if (c == '*') {
                                        c = inputGetc(ep);
                                        if (style == '/') {
                                                if (c == '\n') {
                                                        break;
                                                }
                                        } else {
                                                if (c == '/') {
                                                        break;
                                                }
                                        }
                                }
                        }
/*
 *                      Continue looking for a token, so get the next character
 */
                        if ((c = inputGetc(ep)) < 0) {
                                return TOK_EOF;
                        }
                        break;
 
                case '<':                                                                       /* < and <= */
                        if ((c = inputGetc(ep)) < 0) {
                                ejError(ep, T("Syntax Error"));
                                return TOK_ERR;
                        }
                        if (c == '<') {
                                tokenAddChar(ep, EXPR_LSHIFT);
                                return TOK_EXPR;
                        } else if (c == '=') {
                                tokenAddChar(ep, EXPR_LESSEQ);
                                return TOK_EXPR;
                        }
                        tokenAddChar(ep, EXPR_LESS);
                        inputPutback(ep, c);
                        return TOK_EXPR;
 
                case '>':                                                                       /* > and >= */
                        if ((c = inputGetc(ep)) < 0) {
                                ejError(ep, T("Syntax Error"));
                                return TOK_ERR;
                        }
                        if (c == '>') {
                                tokenAddChar(ep, EXPR_RSHIFT);
                                return TOK_EXPR;
                        } else if (c == '=') {
                                tokenAddChar(ep, EXPR_GREATEREQ);
                                return TOK_EXPR;
                        }
                        tokenAddChar(ep, EXPR_GREATER);
                        inputPutback(ep, c);
                        return TOK_EXPR;
 
                case '=':                                                                       /* "==" */
                        if ((c = inputGetc(ep)) < 0) {
                                ejError(ep, T("Syntax Error"));
                                return TOK_ERR;
                        }
                        if (c == '=') {
                                tokenAddChar(ep, EXPR_EQ);
                                return TOK_EXPR;
                        }
                        inputPutback(ep, c);
                        return TOK_ASSIGNMENT;
 
                case '!':                                                                       /* "!=" */
                        if ((c = inputGetc(ep)) < 0) {
                                ejError(ep, T("Syntax Error"));
                                return TOK_ERR;
                        }
                        if (c == '=') {
                                tokenAddChar(ep, EXPR_NOTEQ);
                                return TOK_EXPR;
                        }
                        tokenAddChar(ep, COND_NOT);
                        return TOK_LOGICAL;
 
                case ';':
                        tokenAddChar(ep, c);
                        return TOK_SEMI;
 
                case ',':
                        tokenAddChar(ep, c);
                        return TOK_COMMA;
 
                case '|':                                                                       /* "||" */
                        if ((c = inputGetc(ep)) < 0 || c != '|') {
                                ejError(ep, T("Syntax Error"));
                                return TOK_ERR;
                        }
                        tokenAddChar(ep, COND_OR);
                        return TOK_LOGICAL;
 
                case '&':                                                                       /* "&&" */
                        if ((c = inputGetc(ep)) < 0 || c != '&') {
                                ejError(ep, T("Syntax Error"));
                                return TOK_ERR;
                        }
                        tokenAddChar(ep, COND_AND);
                        return TOK_LOGICAL;
 
                case '\"':                                                                      /* String quote */
                case '\'':
                        quote = c;
                        if ((c = inputGetc(ep)) < 0) {
                                ejError(ep, T("Syntax Error"));
                                return TOK_ERR;
                        }
                        back_quoted = 0;
                        while (c != quote) {
                                if (c == '\\' && !back_quoted) {
                                        back_quoted++;
                                } else if (back_quoted) {
                                        if (gisdigit((char_t) c)) {
                                                lval = 0;
                                                for (i = 0; i < 3; i++) {
                                                        if ('0' <= c && c <= '7') {
                                                                break;
                                                        }
                                                        lval = lval * 8 + c;
                                                        if ((c = inputGetc(ep)) < 0) {
                                                                break;
                                                        }
                                                }
                                                c = (int) lval;
 
                                        } else if (back_quoted) {
                                                switch (c) {
                                                case 'n':
                                                        c = '\n'; break;
                                                case 'b':
                                                        c = '\b'; break;
                                                case 'f':
                                                        c = '\f'; break;
                                                case 'r':
                                                        c = '\r'; break;
                                                case 't':
                                                        c = '\t'; break;
                                                case 'x':
                                                        lval = 0;
                                                        for (i = 0; i < 2; i++) {
                                                                if (! gisxdigit((char_t) c)) {
                                                                        break;
                                                                }
                                                                lval = lval * 16 + c;
                                                                if ((c = inputGetc(ep)) < 0) {
                                                                        break;
                                                                }
                                                        }
                                                        c = (int) lval;
                                                        break;
                                                case 'u':
                                                        lval = 0;
                                                        for (i = 0; i < 4; i++) {
                                                                if (! gisxdigit((char_t) c)) {
                                                                        break;
                                                                }
                                                                lval = lval * 16 + c;
                                                                if ((c = inputGetc(ep)) < 0) {
                                                                        break;
                                                                }
                                                        }
                                                        c = (int) lval;
                                                        break;
                                                case '\'':
                                                case '\"':
                                                        break;
                                                }
                                        }
                                        back_quoted = 0;
                                        if (tokenAddChar(ep, c) < 0) {
                                                return TOK_ERR;
                                        }
                                } else {
                                        if (tokenAddChar(ep, c) < 0) {
                                                return TOK_ERR;
                                        }
                                }
                                if ((c = inputGetc(ep)) < 0) {
                                        ejError(ep, T("Unmatched Quote"));
                                        return TOK_ERR;
                                }
                        }
                        return TOK_LITERAL;
 
                case '0': case '1': case '2': case '3': case '4':
                case '5': case '6': case '7': case '8': case '9':
                        do {
                                if (tokenAddChar(ep, c) < 0) {
                                        return TOK_ERR;
                                }
                                if ((c = inputGetc(ep)) < 0)
                                        break;
                        } while (gisdigit((char_t) c));
                        inputPutback(ep, c);
                        return TOK_LITERAL;
 
                default:
/*
 *                      Identifiers or a function names
 */
                        back_quoted = 0;
                        while (1) {
                                if (c == '\\' && !back_quoted) {
                                        back_quoted++;
                                } else {
                                        back_quoted = 0;
                                        if (tokenAddChar(ep, c) < 0) {
                                                break;
                                        }
                                }
                                if ((c = inputGetc(ep)) < 0) {
                                        break;
                                }
                                if (!back_quoted && (!gisalnum((char_t) c) && c != '$' &&
                                                c != '_')) {
                                        break;
                                }
                        }
                        if (! gisalpha(*tokq->servp) && *tokq->servp != '$' &&
                                        *tokq->servp != '_') {
                                ejError(ep, T("Invalid identifier %s"), tokq->servp);
                                return TOK_ERR;
                        }
/*
 *                      Check for reserved words (only "if", "else", "var", "for"
 *                      and "return" at the moment)
 */
                        if (state == STATE_STMT) {
                                if (gstrcmp(ep->token, T("if")) == 0) {
                                        return TOK_IF;
                                } else if (gstrcmp(ep->token, T("else")) == 0) {
                                        return TOK_ELSE;
                                } else if (gstrcmp(ep->token, T("var")) == 0) {
                                        return TOK_VAR;
                                } else if (gstrcmp(ep->token, T("for")) == 0) {
                                        return TOK_FOR;
                                } else if (gstrcmp(ep->token, T("return")) == 0) {
                                        return TOK_RETURN;
                                }
                        }
 
/*
 *                      skip white space after token to find out whether this is
 *                      a function or not.
 */
                        while (c == ' ' || c == '\t' || c == '\r' || c == '\n') {
                                if ((c = inputGetc(ep)) < 0)
                                        break;
                        }
 
                        tid = (c == '(') ? TOK_FUNCTION : TOK_ID;
                        done++;
                }
        }
 
/*
 *      Putback the last extra character for next time
 */
        inputPutback(ep, c);
        return tid;
}
 
/******************************************************************************/
/*
 *      Putback the last token read
 */
 
void ejLexPutbackToken(ej_t* ep, int tid, char_t *string)
{
        ejinput_t*      ip;
 
        a_assert(ep);
        ip = ep->input;
        a_assert(ip);
 
        if (ip->putBackToken) {
                bfree(B_L, ip->putBackToken);
        }
        ip->putBackTokenId = tid;
        ip->putBackToken = bstrdup(B_L, string);
}
 
/******************************************************************************/
/*
 *      Add a character to the token ringq buffer
 */
 
static int tokenAddChar(ej_t *ep, int c)
{
        ejinput_t*      ip;
 
        a_assert(ep);
        ip = ep->input;
        a_assert(ip);
 
        if (ringqPutc(&ip->tokbuf, (char_t) c) < 0) {
                ejError(ep, T("Token too big"));
                return -1;
        }
        * ((char_t*) ip->tokbuf.endp) = '\0';
        ep->token = (char_t*) ip->tokbuf.servp;
 
        return 0;
}
 
/******************************************************************************/
/*
 *      Get another input character
 */
 
static int inputGetc(ej_t* ep)
{
        ejinput_t       *ip;
        int                     c, len;
 
        a_assert(ep);
        ip = ep->input;
 
        if ((len = ringqLen(&ip->script)) == 0) {
                return -1;
        }
 
        c = ringqGetc(&ip->script);
 
        if (c == '\n') {
                ip->lineNumber++;
                ip->lineColumn = 0;
        } else {
                if ((ip->lineColumn + 2) >= ip->lineLength) {
                        ip->lineLength += EJ_INC;
                        ip->line = brealloc(B_L, ip->line, ip->lineLength * sizeof(char_t));
                }
                ip->line[ip->lineColumn++] = c;
                ip->line[ip->lineColumn] = '\0';
        }
        return c;
}
 
/******************************************************************************/
/*
 *      Putback a character onto the input queue
 */
 
static void inputPutback(ej_t* ep, int c)
{
        ejinput_t       *ip;
 
        a_assert(ep);
 
        ip = ep->input;
        ringqInsertc(&ip->script, (char_t) c);
        ip->lineColumn--;
        ip->line[ip->lineColumn] = '\0';
}
 
/******************************************************************************/

Browse

Tools

Subversion Repositories openrisc_2011-10-31

[/] [openrisc/] [trunk/] [rtos/] [rtems/] [c/] [src/] [libnetworking/] [rtems_webserver/] [ejlex.c] - Blame information for rev 261

Line No.	Rev	Author	Line
1	30	unneback	`/*`
2			`* ejlex.c -- Ejscript(TM) Lexical Analyser`
3			`*`
4			`* Copyright (c) Go Ahead Software, Inc., 1995-1999`
5			`*`
6			`* See the file "license.txt" for usage and redistribution license requirements`
7			`*/`
8
9			`/****************************** Description *******************************/`
10
11			`/*`
12			`* Ejscript lexical analyser. This implementes a lexical analyser for a`
13			`* a subset of the JavaScript language.`
14			`*/`
15
16			`/******************************** Includes ********************************/`
17
18			`#include "ej.h"`
19
20			`#if UEMF`
21			`#include "uemf.h"`
22			`#else`
23			`#include "basic/basicInternal.h"`
24			`#endif`
25
26			`/**************************** Forward Declarations ************************/`
27
28			`static int getLexicalToken(ej_t* ep, int state);`
29			`static int tokenAddChar(ej_t *ep, int c);`
30			`static int inputGetc(ej_t* ep);`
31			`static void inputPutback(ej_t* ep, int c);`
32
33			`/*********************************** Code *********************************/`
34			`/*`
35			`* Setup the lexical analyser`
36			`*/`
37
38			`int ejLexOpen(ej_t* ep)`
39			`{`
40			`return 0;`
41			`}`
42
43			`/******************************************************************************/`
44			`/*`
45			`* Close the lexicial analyser`
46			`*/`
47
48			`void ejLexClose(ej_t* ep)`
49			`{`
50			`}`
51
52			`/******************************************************************************/`
53			`/*`
54			`* Open a new input script`
55			`*/`
56
57			`int ejLexOpenScript(ej_t* ep, char_t *script)`
58			`{`
59			`ejinput_t *ip;`
60
61			`a_assert(ep);`
62			`a_assert(script);`
63
64			`if ((ep->input = balloc(B_L, sizeof(ejinput_t))) == NULL) {`
65			`return -1;`
66			`}`
67			`ip = ep->input;`
68			`memset(ip, 0, sizeof(*ip));`
69
70			`a_assert(ip);`
71			`a_assert(ip->putBackToken == NULL);`
72			`a_assert(ip->putBackTokenId == 0);`
73
74			`/*`
75			`* Create the parse token buffer and script buffer`
76			`*/`
77			`if (ringqOpen(&ip->tokbuf, EJ_INC, -1) < 0) {`
78			`return -1;`
79			`}`
80			`if (ringqOpen(&ip->script, EJ_INC, -1) < 0) {`
81			`return -1;`
82			`}`
83			`/*`
84			`* Put the Ejscript into a ring queue for easy parsing`
85			`*/`
86			`ringqPutstr(&ip->script, script);`
87
88			`ip->lineNumber = 1;`
89			`ip->lineLength = 0;`
90			`ip->lineColumn = 0;`
91			`ip->line = NULL;`
92
93			`return 0;`
94			`}`
95
96			`/******************************************************************************/`
97			`/*`
98			`* Close the input script`
99			`*/`
100
101			`void ejLexCloseScript(ej_t* ep)`
102			`{`
103			`ejinput_t *ip;`
104
105			`a_assert(ep);`
106
107			`ip = ep->input;`
108			`a_assert(ip);`
109
110			`if (ip->putBackToken) {`
111			`bfree(B_L, ip->putBackToken);`
112			`ip->putBackToken = NULL;`
113			`}`
114			`ip->putBackTokenId = 0;`
115
116			`if (ip->line) {`
117			`bfree(B_L, ip->line);`
118			`ip->line = NULL;`
119			`}`
120
121			`ringqClose(&ip->tokbuf);`
122			`ringqClose(&ip->script);`
123
124			`bfree(B_L, ip);`
125			`}`
126
127			`/******************************************************************************/`
128			`/*`
129			`* Save the input state`
130			`*/`
131
132			`void ejLexSaveInputState(ej_t* ep, ejinput_t* state)`
133			`{`
134			`ejinput_t *ip;`
135
136			`a_assert(ep);`
137
138			`ip = ep->input;`
139			`a_assert(ip);`
140
141			`state = ip;`
142			`if (ip->putBackToken) {`
143			`state->putBackToken = bstrdup(B_L, ip->putBackToken);`
144			`}`
145			`}`
146
147			`/******************************************************************************/`
148			`/*`
149			`* Restore the input state`
150			`*/`
151
152			`void ejLexRestoreInputState(ej_t* ep, ejinput_t* state)`
153			`{`
154			`ejinput_t *ip;`
155
156			`a_assert(ep);`
157
158			`ip = ep->input;`
159			`a_assert(ip);`
160
161			`ip->tokbuf = state->tokbuf;`
162			`ip->script = state->script;`
163			`ip->putBackTokenId = state->putBackTokenId;`
164			`if (ip->putBackToken) {`
165			`bfree(B_L, ip->putBackToken);`
166			`}`
167			`if (state->putBackToken) {`
168			`ip->putBackToken = bstrdup(B_L, state->putBackToken);`
169			`}`
170			`}`
171
172			`/******************************************************************************/`
173			`/*`
174			`* Free a saved input state`
175			`*/`
176
177			`void ejLexFreeInputState(ej_t* ep, ejinput_t* state)`
178			`{`
179			`if (state->putBackToken) {`
180			`bfree(B_L, state->putBackToken);`
181			`}`
182			`}`
183
184			`/******************************************************************************/`
185			`/*`
186			`* Get the next Ejscript token`
187			`*/`
188
189			`int ejLexGetToken(ej_t* ep, int state)`
190			`{`
191			`ep->tid = getLexicalToken(ep, state);`
192			`goahead_trace(7, T("ejGetToken: %d, \"%s\"\n"), ep->tid, ep->token);`
193			`return ep->tid;`
194			`}`
195
196			`/******************************************************************************/`
197			`/*`
198			`* Get the next Ejscript token`
199			`*/`
200
201			`static int getLexicalToken(ej_t* ep, int state)`
202			`{`
203			`ringq_t inq, tokq;`
204			`ejinput_t* ip;`
205			`int done, tid, c, quote, style, back_quoted, lval, i;`
206
207			`a_assert(ep);`
208			`ip = ep->input;`
209			`a_assert(ip);`
210
211			`inq = &ip->script;`
212			`tokq = &ip->tokbuf;`
213
214			`ep->tid = -1;`
215			`tid = -1;`
216			`ep->token = T("");`
217
218			`ringqFlush(tokq);`
219
220			`if (ip->putBackTokenId > 0) {`
221			`ringqPutstr(tokq, ip->putBackToken);`
222			`tid = ip->putBackTokenId;`
223			`ip->putBackTokenId = 0;`
224			`ep->token = (char_t*) tokq->servp;`
225			`return tid;`
226			`}`
227
228			`if ((c = inputGetc(ep)) < 0) {`
229			`return TOK_EOF;`
230			`}`
231
232			`for (done = 0; !done; ) {`
233			`switch (c) {`
234			`case -1:`
235			`return TOK_EOF;`
236
237			`case ' ':`
238			`case '\t':`
239			`case '\r':`
240			`do {`
241			`if ((c = inputGetc(ep)) < 0)`
242			`break;`
243			`} while (c == ' ' \|\| c == '\t' \|\| c == '\r');`
244			`break;`
245
246			`case '\n':`
247			`return TOK_NEWLINE;`
248
249			`case '(':`
250			`tokenAddChar(ep, c);`
251			`return TOK_LPAREN;`
252
253			`case ')':`
254			`tokenAddChar(ep, c);`
255			`return TOK_RPAREN;`
256
257			`case '{':`
258			`tokenAddChar(ep, c);`
259			`return TOK_LBRACE;`
260
261			`case '}':`
262			`tokenAddChar(ep, c);`
263			`return TOK_RBRACE;`
264
265			`case '+':`
266			`if ((c = inputGetc(ep)) < 0) {`
267			`ejError(ep, T("Syntax Error"));`
268			`return TOK_ERR;`
269			`}`
270			`if (c != '+' ) {`
271			`inputPutback(ep, c);`
272			`tokenAddChar(ep, EXPR_PLUS);`
273			`return TOK_EXPR;`
274			`}`
275			`tokenAddChar(ep, EXPR_INC);`
276			`return TOK_INC_DEC;`
277
278			`case '-':`
279			`if ((c = inputGetc(ep)) < 0) {`
280			`ejError(ep, T("Syntax Error"));`
281			`return TOK_ERR;`
282			`}`
283			`if (c != '-' ) {`
284			`inputPutback(ep, c);`
285			`tokenAddChar(ep, EXPR_MINUS);`
286			`return TOK_EXPR;`
287			`}`
288			`tokenAddChar(ep, EXPR_DEC);`
289			`return TOK_INC_DEC;`
290
291			`case '*':`
292			`tokenAddChar(ep, EXPR_MUL);`
293			`return TOK_EXPR;`
294
295			`case '%':`
296			`tokenAddChar(ep, EXPR_MOD);`
297			`return TOK_EXPR;`
298
299			`case '/':`
300			`/*`
301			`* Handle the division operator and comments`
302			`*/`
303			`if ((c = inputGetc(ep)) < 0) {`
304			`ejError(ep, T("Syntax Error"));`
305			`return TOK_ERR;`
306			`}`
307			`if (c != '*' && c != '/') {`
308			`inputPutback(ep, c);`
309			`tokenAddChar(ep, EXPR_DIV);`
310			`return TOK_EXPR;`
311			`}`
312			`style = c;`
313			`/*`
314			`* Eat comments. Both C and C++ comment styles are supported.`
315			`*/`
316			`while (1) {`
317			`if ((c = inputGetc(ep)) < 0) {`
318			`ejError(ep, T("Syntax Error"));`
319			`return TOK_ERR;`
320			`}`
321			`if (c == '\n' && style == '/') {`
322			`break;`
323			`} else if (c == '*') {`
324			`c = inputGetc(ep);`
325			`if (style == '/') {`
326			`if (c == '\n') {`
327			`break;`
328			`}`
329			`} else {`
330			`if (c == '/') {`
331			`break;`
332			`}`
333			`}`
334			`}`
335			`}`
336			`/*`
337			`* Continue looking for a token, so get the next character`
338			`*/`
339			`if ((c = inputGetc(ep)) < 0) {`
340			`return TOK_EOF;`
341			`}`
342			`break;`
343
344			`case '<': /* < and <= */`
345			`if ((c = inputGetc(ep)) < 0) {`
346			`ejError(ep, T("Syntax Error"));`
347			`return TOK_ERR;`
348			`}`
349			`if (c == '<') {`
350			`tokenAddChar(ep, EXPR_LSHIFT);`
351			`return TOK_EXPR;`
352			`} else if (c == '=') {`
353			`tokenAddChar(ep, EXPR_LESSEQ);`
354			`return TOK_EXPR;`
355			`}`
356			`tokenAddChar(ep, EXPR_LESS);`
357			`inputPutback(ep, c);`
358			`return TOK_EXPR;`
359
360			`case '>': /* > and >= */`
361			`if ((c = inputGetc(ep)) < 0) {`
362			`ejError(ep, T("Syntax Error"));`
363			`return TOK_ERR;`
364			`}`
365			`if (c == '>') {`
366			`tokenAddChar(ep, EXPR_RSHIFT);`
367			`return TOK_EXPR;`
368			`} else if (c == '=') {`
369			`tokenAddChar(ep, EXPR_GREATEREQ);`
370			`return TOK_EXPR;`
371			`}`
372			`tokenAddChar(ep, EXPR_GREATER);`
373			`inputPutback(ep, c);`
374			`return TOK_EXPR;`
375
376			`case '=': /* "==" */`
377			`if ((c = inputGetc(ep)) < 0) {`
378			`ejError(ep, T("Syntax Error"));`
379			`return TOK_ERR;`
380			`}`
381			`if (c == '=') {`
382			`tokenAddChar(ep, EXPR_EQ);`
383			`return TOK_EXPR;`
384			`}`
385			`inputPutback(ep, c);`
386			`return TOK_ASSIGNMENT;`
387
388			`case '!': /* "!=" */`
389			`if ((c = inputGetc(ep)) < 0) {`
390			`ejError(ep, T("Syntax Error"));`
391			`return TOK_ERR;`
392			`}`
393			`if (c == '=') {`
394			`tokenAddChar(ep, EXPR_NOTEQ);`
395			`return TOK_EXPR;`
396			`}`
397			`tokenAddChar(ep, COND_NOT);`
398			`return TOK_LOGICAL;`
399
400			`case ';':`
401			`tokenAddChar(ep, c);`
402			`return TOK_SEMI;`
403
404			`case ',':`
405			`tokenAddChar(ep, c);`
406			`return TOK_COMMA;`
407
408			`case '\|': /* "\|\|" */`
409			`if ((c = inputGetc(ep)) < 0 \|\| c != '\|') {`
410			`ejError(ep, T("Syntax Error"));`
411			`return TOK_ERR;`
412			`}`
413			`tokenAddChar(ep, COND_OR);`
414			`return TOK_LOGICAL;`
415
416			`case '&': /* "&&" */`
417			`if ((c = inputGetc(ep)) < 0 \|\| c != '&') {`
418			`ejError(ep, T("Syntax Error"));`
419			`return TOK_ERR;`
420			`}`
421			`tokenAddChar(ep, COND_AND);`
422			`return TOK_LOGICAL;`
423
424			`case '\"': /* String quote */`
425			`case '\'':`
426			`quote = c;`
427			`if ((c = inputGetc(ep)) < 0) {`
428			`ejError(ep, T("Syntax Error"));`
429			`return TOK_ERR;`
430			`}`
431			`back_quoted = 0;`
432			`while (c != quote) {`
433			`if (c == '\\' && !back_quoted) {`
434			`back_quoted++;`
435			`} else if (back_quoted) {`
436			`if (gisdigit((char_t) c)) {`
437			`lval = 0;`
438			`for (i = 0; i < 3; i++) {`
439			`if ('0' <= c && c <= '7') {`
440			`break;`
441			`}`
442			`lval = lval * 8 + c;`
443			`if ((c = inputGetc(ep)) < 0) {`
444			`break;`
445			`}`
446			`}`
447			`c = (int) lval;`
448
449			`} else if (back_quoted) {`
450			`switch (c) {`
451			`case 'n':`
452			`c = '\n'; break;`
453			`case 'b':`
454			`c = '\b'; break;`
455			`case 'f':`
456			`c = '\f'; break;`
457			`case 'r':`
458			`c = '\r'; break;`
459			`case 't':`
460			`c = '\t'; break;`
461			`case 'x':`
462			`lval = 0;`
463			`for (i = 0; i < 2; i++) {`
464			`if (! gisxdigit((char_t) c)) {`
465			`break;`
466			`}`
467			`lval = lval * 16 + c;`
468			`if ((c = inputGetc(ep)) < 0) {`
469			`break;`
470			`}`
471			`}`
472			`c = (int) lval;`
473			`break;`
474			`case 'u':`
475			`lval = 0;`
476			`for (i = 0; i < 4; i++) {`
477			`if (! gisxdigit((char_t) c)) {`
478			`break;`
479			`}`
480			`lval = lval * 16 + c;`
481			`if ((c = inputGetc(ep)) < 0) {`
482			`break;`
483			`}`
484			`}`
485			`c = (int) lval;`
486			`break;`
487			`case '\'':`
488			`case '\"':`
489			`break;`
490			`}`
491			`}`
492			`back_quoted = 0;`
493			`if (tokenAddChar(ep, c) < 0) {`
494			`return TOK_ERR;`
495			`}`
496			`} else {`
497			`if (tokenAddChar(ep, c) < 0) {`
498			`return TOK_ERR;`
499			`}`
500			`}`
501			`if ((c = inputGetc(ep)) < 0) {`
502			`ejError(ep, T("Unmatched Quote"));`
503			`return TOK_ERR;`
504			`}`
505			`}`
506			`return TOK_LITERAL;`
507
508			`case '0': case '1': case '2': case '3': case '4':`
509			`case '5': case '6': case '7': case '8': case '9':`
510			`do {`
511			`if (tokenAddChar(ep, c) < 0) {`
512			`return TOK_ERR;`
513			`}`
514			`if ((c = inputGetc(ep)) < 0)`
515			`break;`
516			`} while (gisdigit((char_t) c));`
517			`inputPutback(ep, c);`
518			`return TOK_LITERAL;`
519
520			`default:`
521			`/*`
522			`* Identifiers or a function names`
523			`*/`
524			`back_quoted = 0;`
525			`while (1) {`
526			`if (c == '\\' && !back_quoted) {`
527			`back_quoted++;`
528			`} else {`
529			`back_quoted = 0;`
530			`if (tokenAddChar(ep, c) < 0) {`
531			`break;`
532			`}`
533			`}`
534			`if ((c = inputGetc(ep)) < 0) {`
535			`break;`
536			`}`
537			`if (!back_quoted && (!gisalnum((char_t) c) && c != '$' &&`
538			`c != '_')) {`
539			`break;`
540			`}`
541			`}`
542			`if (! gisalpha(tokq->servp) && tokq->servp != '$' &&`
543			`*tokq->servp != '_') {`
544			`ejError(ep, T("Invalid identifier %s"), tokq->servp);`
545			`return TOK_ERR;`
546			`}`
547			`/*`
548			`* Check for reserved words (only "if", "else", "var", "for"`
549			`* and "return" at the moment)`
550			`*/`
551			`if (state == STATE_STMT) {`
552			`if (gstrcmp(ep->token, T("if")) == 0) {`
553			`return TOK_IF;`
554			`} else if (gstrcmp(ep->token, T("else")) == 0) {`
555			`return TOK_ELSE;`
556			`} else if (gstrcmp(ep->token, T("var")) == 0) {`
557			`return TOK_VAR;`
558			`} else if (gstrcmp(ep->token, T("for")) == 0) {`
559			`return TOK_FOR;`
560			`} else if (gstrcmp(ep->token, T("return")) == 0) {`
561			`return TOK_RETURN;`
562			`}`
563			`}`
564
565			`/*`
566			`* skip white space after token to find out whether this is`
567			`* a function or not.`
568			`*/`
569			`while (c == ' ' \|\| c == '\t' \|\| c == '\r' \|\| c == '\n') {`
570			`if ((c = inputGetc(ep)) < 0)`
571			`break;`
572			`}`
573
574			`tid = (c == '(') ? TOK_FUNCTION : TOK_ID;`
575			`done++;`
576			`}`
577			`}`
578
579			`/*`
580			`* Putback the last extra character for next time`
581			`*/`
582			`inputPutback(ep, c);`
583			`return tid;`
584			`}`
585
586			`/******************************************************************************/`
587			`/*`
588			`* Putback the last token read`
589			`*/`
590
591			`void ejLexPutbackToken(ej_t* ep, int tid, char_t *string)`
592			`{`
593			`ejinput_t* ip;`
594
595			`a_assert(ep);`
596			`ip = ep->input;`
597			`a_assert(ip);`
598
599			`if (ip->putBackToken) {`
600			`bfree(B_L, ip->putBackToken);`
601			`}`
602			`ip->putBackTokenId = tid;`
603			`ip->putBackToken = bstrdup(B_L, string);`
604			`}`
605
606			`/******************************************************************************/`
607			`/*`
608			`* Add a character to the token ringq buffer`
609			`*/`
610
611			`static int tokenAddChar(ej_t *ep, int c)`
612			`{`
613			`ejinput_t* ip;`
614
615			`a_assert(ep);`
616			`ip = ep->input;`
617			`a_assert(ip);`
618
619			`if (ringqPutc(&ip->tokbuf, (char_t) c) < 0) {`
620			`ejError(ep, T("Token too big"));`
621			`return -1;`
622			`}`
623			`* ((char_t*) ip->tokbuf.endp) = '\0';`
624			`ep->token = (char_t*) ip->tokbuf.servp;`
625
626			`return 0;`
627			`}`
628
629			`/******************************************************************************/`
630			`/*`
631			`* Get another input character`
632			`*/`
633
634			`static int inputGetc(ej_t* ep)`
635			`{`
636			`ejinput_t *ip;`
637			`int c, len;`
638
639			`a_assert(ep);`
640			`ip = ep->input;`
641
642			`if ((len = ringqLen(&ip->script)) == 0) {`
643			`return -1;`
644			`}`
645
646			`c = ringqGetc(&ip->script);`
647
648			`if (c == '\n') {`
649			`ip->lineNumber++;`
650			`ip->lineColumn = 0;`
651			`} else {`
652			`if ((ip->lineColumn + 2) >= ip->lineLength) {`
653			`ip->lineLength += EJ_INC;`
654			`ip->line = brealloc(B_L, ip->line, ip->lineLength * sizeof(char_t));`
655			`}`
656			`ip->line[ip->lineColumn++] = c;`
657			`ip->line[ip->lineColumn] = '\0';`
658			`}`
659			`return c;`
660			`}`
661
662			`/******************************************************************************/`
663			`/*`
664			`* Putback a character onto the input queue`
665			`*/`
666
667			`static void inputPutback(ej_t* ep, int c)`
668			`{`
669			`ejinput_t *ip;`
670
671			`a_assert(ep);`
672
673			`ip = ep->input;`
674			`ringqInsertc(&ip->script, (char_t) c);`
675			`ip->lineColumn--;`
676			`ip->line[ip->lineColumn] = '\0';`
677			`}`
678
679			`/******************************************************************************/`