Subversion Repositories openrisc

/* ReaderTokenizer.java -- splits the input char sequence int tokens.

   Copyright (C) 2005 Free Software Foundation, Inc.

This file is part of GNU Classpath.

GNU Classpath is free software; you can redistribute it and/or modify

it under the terms of the GNU General Public License as published by

the Free Software Foundation; either version 2, or (at your option)

any later version.

GNU Classpath is distributed in the hope that it will be useful, but

WITHOUT ANY WARRANTY; without even the implied warranty of

MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

General Public License for more details.

You should have received a copy of the GNU General Public License

along with GNU Classpath; see the file COPYING.  If not, write to the

Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

02110-1301 USA.

Linking this library statically or dynamically with other modules is

making a combined work based on this library.  Thus, the terms and

conditions of the GNU General Public License cover the whole

combination.

As a special exception, the copyright holders of this library give you

permission to link this library with independent modules to produce an

executable, regardless of the license terms of these independent

modules, and to copy and distribute the resulting executable under

terms of your choice, provided that you also meet, for each linked

independent module, the terms and conditions of the license of that

module.  An independent module is a module which is not derived from

or based on this library.  If you modify this library, you may extend

this exception to your version of the library, but you are not

obligated to do so.  If you do not wish to do so, delete this

exception statement from your version. */

package gnu.javax.swing.text.html.parser.support.low;

import java.io.IOException;

import java.io.Reader;

/**

 * Reader splits the input char sequence into tokens.

 * @author Audrius Meskauskas, Lithuania (AudriusA@Bioinformatics.org)

 */

public class ReaderTokenizer

  extends Constants

{

  /**

   * This is set to true each time the getNextToken is called.

   * Used in preventing loops when all patterns refuse to accept

   * the invalid input.

   */

  protected boolean advanced;

  /**

   * If true, the returned tokens are also placed in the backup

   * queue.

   */

  protected boolean backupMode;

  /**

   * The buffer to read document into.

   */

  Buffer buffer = new Buffer();

  /**

   * The queue for supporting mark().

   */

  Queue backup = new Queue();

  /**

   * The queue of found tokens.

   */

  Queue queue = new Queue();

  /**

   * The reader to read the document from.

   */

  Reader reader;

  /**

   * Array of char tokens

   */

  char[] charTokens;

  /**

   * Array of string tokens.

   */

  String[] stringTokens;

  /**

   * The current reader position.

   */

  int readerPosition = -1;

  /**

   * Creates a new ReaderTokenizer. The reset(...) method must be

   * subsequently called to set the reader.

   */

  public ReaderTokenizer()

  {

  }

  /**

   * Return the sequence, used to separate lines in the document.

   * @return one of \n, \r or \r\n.

   */

  public String getEndOfLineSequence()

  {

    return buffer.getEndOfLineSequence();

  }

  /**

   * Get the next token.

   * @return

   */

  public Token getNextToken()

  {

    Token rt;

    advanced = true;

    try

      {

        if (queue.isEmpty())

          read(1);

        if (!queue.isEmpty())

          rt = queue.next();

        else

          rt = new Token(EOF, new Location(readerPosition));

      }

    catch (IOException ex)

      {

        throw new ParseException("IO Exception", ex);

      }

    if (backupMode)

      backup.add(rt);

    return rt;

  }

  /**

   * Get a token, lying the given number of tokens

   * ahead. getToken(0) will return the same token,

   * what would be returned by getNextToken().

   * getToken(..) does change the current position

   * in the input stream. If the end of stream is

   * reached, the EOF token is always returned.

   */

  public Token getTokenAhead(int ahead)

  {

    try

      {

        read(ahead - queue.size() + 1);

        return queue.size() >= ahead ? queue.get(ahead) : eofToken();

      }

    catch (IOException ex)

      {

        throw new ParseException("IO Exception", ex);

      }

  }

  /**

   * Get a token, bein immediatley ahead.

   * If the end of stream is

   * reached, the EOF token is always returned.

   * The method is equivalent calling getTokenAhead(0).

   */

  public Token getTokenAhead()

  {

    try

      {

        if (queue.isEmpty())

          read(1);

        if (!queue.isEmpty())

          return queue.get(0);

        else

          return eofToken();

      }

    catch (IOException ex)

      {

        throw new ParseException("IO Exception", ex);

      }

  }

  /**

   * Invokes the error handler.

   */

  public void error(String msg, Token at)

  {

    System.out.println(msg);

  }

  /**

   * Turns the backup mode on or off.

   * It is possible to return where the mark(true) was last called

   * by calling reset().

   * @param mode True if it is required to save tokens, making

   * returning to the current point possible.

   */

  public void mark(boolean mode)

  {

    backup.clear();

    backupMode = mode;

  }

  /**

   * Prepare for new parsing from the given stream.

   * @param a_reader A reader to parse from.

   */

  public void reset(Reader a_reader)

  {

    reader = a_reader;

    readerPosition = -1;

    buffer.reset();

    queue.clear();

  }

  /**

   * Reset the internal cursor to the position where the mark()

   * was last time called. Switches the backup mode off.

   */

  public void reset()

  {

    if (!backupMode)

      throw new AssertionError("Call mark(true) before using reset()!");

    backupMode = false;

    // That is now in the queue, will be appended to the end of backup.

    while (!queue.isEmpty())

      backup.add(queue.next());

    Queue t = queue;

    queue = backup;

    backup = t;

    backup.clear();

  }

  /**

   * Read the given number of the tokens. Add the needed number of EOF

   * tokens if there are no more data in the stream.

   * @param numberOfTokens The number of additional tokens to read.

   */

  void read(int numberOfTokens)

     throws IOException

  {

    if (numberOfTokens <= 0)

      return;

    for (int i = 0; i < numberOfTokens; i++)

      readToken();

  }

  /**

   * Read next token from the reader, add it to the queue

   */

  void readToken()

          throws IOException

  {

    Token t;

    int ch;

    enlarging:

    while (true)

      {

        t = tokenMatches();

        if (t != null)

          break enlarging;

        else

          {

            ch = reader.read();

            readerPosition++;

            if (ch == ETX)

              ch = ' ';

            if (ch < 0)

              {

                if (buffer.length() == 0)

                  {

                    queue.add(eofToken());

                    return;

                  }

                else

                  {

                    if (buffer.charAt(buffer.length() - 1) != ETX)

                      buffer.append(ETX, readerPosition++);

                    else

                      {

                        // Discard terminating ETX

                        buffer.setLength(buffer.length() - 1);

                        if (buffer.length() > 0)

                          {

                            t = new Token(OTHER, buffer.toString(),

                                          buffer.getLocation(0, buffer.length())

                                         );

                            queue.add(t);

                            buffer.setLength(0);

                          }

                        return;

                      }

                  }

              }

            else

              buffer.append((char) ch, readerPosition);

          }

      }

  }

  /**

   * Check if the end of buffer matches one of the tokens. If it does,

   * return this token and remove the token sequence from the end of

   * buffer.

   * @return The matching token.

   */

  Token tokenMatches()

  {

    Token rt = endMatches(buffer);

    if (rt != null) // Remove the matched image

      {

        // Consume future character if it was an entity and the future

        // character is semicolon.

        if (rt.kind == ENTITY)

          {

            if (buffer.charAt(buffer.length() - 1) == ';')

              buffer.setLength(buffer.length() - rt.getImage().length() - 1);

            else

              {

                error("Missing closing semicolon for entity '" + rt.getImage() +

                      "'", rt

                     );

                consumeBuffer(rt);

              }

          }

        else

          {

            consumeBuffer(rt);

          }

      }

    // If the buffer is not empty, some sequence does not match any tokens.

    // Add it to the queue as "OTHER".

    if (rt != null)

      {

        if (buffer.length() > 1)

          {

            String rest = buffer.toString();

            rest = rest.substring(0, rest.length() - 1);

            Token other =

              new Token(OTHER, rest, buffer.getLocation(0, buffer.length));

            queue.add(other);

            consumeBuffer(other);

          }

        queue.add(rt);

      }

    return rt;

  }

  private void consumeBuffer(Token rt)

  {

    buffer.delete(buffer.length() - rt.getImage().length() - 1,

                  buffer.length() - 1

                 );

  }

  /**

   * Create EOF token.

   */

  private Token eofToken()

  {

    return new Token(EOF, "#", new Location(readerPosition));

  }

}