gplexx.frame

##
## DO NOT EDIT THIS FILE CARELESSLY!
## THE CONTENTS OF THIS FILE ARE STRONGLY LINKED TO THE
## CODE OF Dfsa.cs AND TaskState.cs, AND ALL MUST VARY 
## TOGETHER
##
//
// gplexx.frame
// Version 1.0.0 of 01-November-2008
## Derived from gplex.frame version of 2-September-2006.
## Codepage support for files without a BOM. 
## Left and Right Anchored state support.
## Start condition stack. Two generic params.
## Using fixed length context handling for right anchors
//
##-->defines

using System;
using System.IO;
using System.Text;
using System.Collections.Generic;
#if !STANDALONE
using gppg;
#endif
##-->version240
##-->usingDcl
{   
    /// <summary>
    /// Summary Canonical example of GPLEX automaton
    /// </summary>
    
#if STANDALONE
    //
    // These are the dummy declarations for stand-alone GPLEX applications
    // normally these declarations would come from the parser.
    // If you declare /noparser, or %option noparser then you get this.
    //

    public enum Tokens
    { 
      EOF = 0, maxParseToken = int.MaxValue 
      // must have at least these two, values are almost arbitrary
    }

    public abstract class ScanBase
    {
        public abstract int yylex();
        protected virtual bool yywrap() { return true; }
#if BABEL
        protected abstract int CurrentSc { get; set; }
        // EolState is the 32-bit of state data persisted at 
        // the end of each line for Visual Studio colorization.  
        // The default is to return CurrentSc.  You must override
        // this if you want more complicated behavior.
        public virtual int EolState { 
            get { return CurrentSc; }
            set { CurrentSc = value; } 
        }
    }
    
    public interface IColorScan
    {
        void SetSource(string source, int offset);
        int GetNext(ref int state, out int start, out int end);
#endif // BABEL
    }

#endif // STANDALONE

    public abstract class ScanBuff
    {
        public const int EOF = -1;
        public abstract int Pos { get; set; }
        public abstract int Read();
        public abstract int Peek();
        public abstract int ReadPos { get; }
        public abstract string GetString(int b, int e);
        
        public const int UnicodeReplacementChar = 0xFFFD;
    }
    
    // If the compiler can't find ScanBase maybe you need to run
    // GPPG with the /gplex option, or GPLEX with /noparser
#if BABEL
    public sealed partial class Scanner : ScanBase, IColorScan
    {
        public ScanBuff buffer;
        int currentScOrd = 0;  // start condition ordinal
        
        protected override int CurrentSc 
        {
             // The current start state is a property
             // to try to avoid the user error of setting
             // scState but forgetting to update the FSA
             // start state "currentStart"
             //
             get { return currentScOrd; }  // i.e. return YY_START;
             set { currentScOrd = value;   // i.e. BEGIN(value);
                   currentStart = startState[value]; }
        }
#else  // BABEL
    public sealed partial class Scanner : ScanBase
    {
        public ScanBuff buffer;
        int currentScOrd = 0;  // start condition ordinal
#endif // BABEL
        
        private static int GetMaxParseToken() {
            System.Reflection.FieldInfo f = typeof(Tokens).GetField("maxParseToken");
            return (f == null ? int.MaxValue : (int)f.GetValue(null));
        }
        
        static int parserMax = GetMaxParseToken();
        
        enum Result {accept, noMatch, contextFound};

##-->consts

#region user code
##-->codeIncl
#endregion user code

        int state;
        int currentStart = startState[0];
        int chr;           // last character read
        int cNum = 0;      // ordinal number of chr
        int lNum = 0;      // current line number
        int lineStartNum;  // ordinal number at start of line
        //
        // The following instance variables are used, among other
        // things, for constructing the yylloc location objects.
        //
        int tokPos;        // buffer position at start of token
        int tokNum;        // ordinal number of first character
        int tokLen;        // number of characters in token
        int tokCol;        // zero-based column number at start of token
        int tokLin;        // line number at start of token
        int tokEPos;       // buffer position at end of token
        int tokECol;       // column number at end of token
        int tokELin;       // line number at end of token
        string tokTxt;     // lazily constructed text of token
#if STACK          
        private Stack<int> scStack = new Stack<int>();
#endif // STACK

##-->tableDef


#if BACKUP
        // ==============================================================
        // == Nested class used for backup in automata which do backup ==
        // ==============================================================

        internal class Context // class used for automaton backup.
        {
            public int bPos;
            public int cNum;
            public int lNum; // Need this in case of backup over EOL.
            public int state;
            public int cChr;
        }
#endif // BACKUP

        // ==============================================================
        // ==== Nested struct to support input switching in scanners ====
        // ==============================================================

		struct BufferContext {
			internal ScanBuff buffSv;
			internal int chrSv;
			internal int cNumSv;
			internal int lNumSv;
			internal int startSv;
		}
		
#region Buffer classes

        // ==============================================================
        // ===== Private methods to save and restore buffer contexts ====
        // ==============================================================

        /// <summary>
        /// This method creates a buffer context record from
        /// the current buffer object, together with some
        /// scanner state values. 
        /// </summary>
		BufferContext MkBuffCtx()
		{
			BufferContext rslt;
			rslt.buffSv = this.buffer;
			rslt.chrSv = this.chr;
			rslt.cNumSv = this.cNum;
			rslt.lNumSv = this.lNum;
			rslt.startSv = this.lineStartNum;
			return rslt;
		}

        /// <summary>
        /// This method restores the buffer value and allied
        /// scanner state from the given context record value.
        /// </summary>
		void RestoreBuffCtx(BufferContext value)
		{
			this.buffer = value.buffSv;
			this.chr = value.chrSv;
			this.cNum = value.cNumSv;
			this.lNum = value.lNumSv;
			this.lineStartNum = value.startSv;
        } 
        
        // ==============================================================
        // ===== Nested classes for various ScanBuff derived classes ====
        // ==============================================================
        // =============== Nested class for string input ================
        // ==============================================================

        /// <summary>
        /// This class read characters from a single string as
        /// required, for example, by Visual Studio language services
        /// </summary>
        public sealed class StringBuff : ScanBuff
        {
            string str;        // input buffer
            int bPos;          // current position in buffer
            int sLen;

            public StringBuff(string str)
            {
                this.str = str;
                this.sLen = str.Length;
            }

            private int Read16()
            {
                if (bPos < sLen) return str[bPos++];
#if BABEL
                else if (bPos == sLen) { bPos++; return '\n'; }   // one strike, see newline
#endif // BABEL
                else { bPos++; return EOF; }                      // two strikes and you're out!
            }
            
            /// <summary>
            /// Read returns UTF32 in this application.
            /// </summary>
            /// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
            public override int Read()
            {
                int utf16 = Read16();
                if (utf16 < 0xD800 || utf16 > 0xDBFF)
                    return utf16;
                else
                {
                    int low16 = Read16();
                    if (low16 < 0xDC00 || low16 > 0xDFFF)
                        return UnicodeReplacementChar;
                    else
                        return (0x10000 + (utf16 & 0x3FF << 10) + (low16 & 0x3FF));
                }
            }
            
            public override int ReadPos { get { return bPos - 1; } }

            public override int Peek()
            {
                if (bPos < sLen) return str[bPos];
                else return '\n';
            }

            public override string GetString(int beg, int end)
            {
                //  "end" can be greater than sLen with the BABEL
                //  option set.  Read returns a "virtual" EOL if
                //  an attempt is made to read past the end of the
                //  string buffer.  Without the guard any attempt 
                //  to fetch yytext for a token that includes the 
                //  EOL will throw an index exception.
                if (end > sLen) end = sLen;
                if (end <= beg) return ""; 
                else return str.Substring(beg, end - beg);
            }

            public override int Pos
            {
                get { return bPos; }
                set { bPos = value; }
            }
        }

#if !NOFILES
       // ====================== Nested class ==========================
       //  The LineBuff class contributed by Nigel Horspool, 
       //  nigelh@cs.uvic.cs
       // ==============================================================

        public sealed class LineBuff : ScanBuff
        {
            IList<string> line;    // list of source lines from a file
            int numLines;          // number of strings in line list
            string curLine;        // current line in that list
            int cLine;             // index of current line in the list
            int curLen;            // length of current line
            int curLineStart;      // position of line start in whole file
            int curLineEnd;        // position of line end in whole file
            int maxPos;            // max position ever visited in whole file
            int cPos;              // ordinal number of chr in source

            // Constructed from a list of strings, one per source line.
            // The lines have had trailing '\n' characters removed.
            public LineBuff(IList<string> lineList)
            {
                line = lineList;
                numLines = line.Count;
                cPos = curLineStart = 0;
                curLine = (numLines>0 ? line[0] : "");
                maxPos = curLineEnd = curLen = curLine.Length;
                cLine = 1;
            }

            public override int Read()
            {
                if (cPos < curLineEnd)
                    return curLine[cPos++ - curLineStart];
                if (cPos++ == curLineEnd)
                    return '\n';
                if (cLine >= numLines)
                    return EOF;
                curLine = line[cLine];
                curLen = curLine.Length;
                curLineStart = curLineEnd + 1;
                curLineEnd = curLineStart + curLen;
                if (curLineEnd>maxPos)
                    maxPos = curLineEnd;
                cLine++;
                return curLen>0? curLine[0] : '\n';
            }

            public override int Peek()
            {
                return (cPos < curLineEnd)? curLine[cPos - curLineStart] : '\n';
            }

            // To speed up searches for the line containing a position
            private int cachedPos = 0;
            private int cachedIx = 0;
            private int cachedLstart = 0;

            // Given a position pos within the entire source, the results are
            //   ix     -- the index of the containing line
            //   lstart -- the position of the first character on that line
            private void findIndex( int pos, out int ix, out int lstart )
            {
                if (pos >= cachedPos) {
                    ix = cachedIx;  lstart = cachedLstart;
                } else {
                    ix = lstart = 0;
                }
                for( ; ; ) {
                    int len = line[ix].Length + 1;
                    if (pos < lstart+len) break;
                    lstart += len;
                    ix++;
                }
                cachedPos = pos;
                cachedIx = ix;
                cachedLstart = lstart;
            }
        
            public override string GetString(int beg, int end)
            {
                if (beg >= maxPos || end <= beg) return "";
                int endIx, begIx, endLineStart, begLineStart;
                findIndex(beg, out begIx, out begLineStart);
                int begCol = beg - begLineStart;
                findIndex(end, out endIx, out endLineStart);
                int endCol = end - endLineStart;
                string s = line[begIx];
                if (begIx == endIx) {
                    // the usual case, substring all on one line
                    return (endCol <= s.Length)?
                        s.Substring(begCol, endCol-begCol)
                        : s.Substring(begCol) + "\n";
                }
                // the string spans multiple lines, yuk!
                StringBuilder sb = new StringBuilder();
                if (begCol < s.Length)
                    sb.Append(s.Substring(begCol));
                for( ; ; ) {
                    sb.Append("\n");
                    s = line[++begIx];
                    if (begIx >= endIx) break;
                    sb.Append(s);
                }
                if (endCol <= s.Length) {
                    sb.Append(s.Substring(0, endCol));
                } else {
                    sb.Append(s);
                    sb.Append("\n");
                }
                return sb.ToString();
            }

            public override int Pos
            {
                get { return cPos; }
                set {
                    cPos = value;
                    findIndex(cPos, out cLine, out curLineStart);
                    curLine = line[cLine];
                    curLineEnd = curLineStart+curLine.Length;
                }
            }
            
            public override int ReadPos { get { return cPos - 1; } }
        }
        
        // ==============================================================
        // =========== Nested class StreamBuff : byte files =============
        // ==============================================================

        /// <summary>
        /// This is the Buffer for byte files. It returns 8-bit characters.
        /// </summary>
        public class StreamBuff : ScanBuff
        {
            protected BufferedStream bStrm;   // input buffer
            protected int delta = 1;          // number of bytes in chr, could be 0 for EOF.

            public StreamBuff(Stream str) { this.bStrm = new BufferedStream(str); }

            /// <summary>
            /// Read returns subset [0 .. 0xFF] in this application.
            /// </summary>
            /// <returns>Unicode point as an int [0 .. 0xFF]; -1 for EOF</returns>
            public override int Read() {
                int ch0 = bStrm.ReadByte();
                delta = (ch0 == EOF ? 0 : 1);
                return ch0; 
            }
            
            public override int ReadPos {
                get { return (int)bStrm.Position - delta; }
            }

            public override int Peek()
            {
                int rslt = bStrm.ReadByte();
                bStrm.Seek(-delta, SeekOrigin.Current);
                return rslt;
            }

            public override string GetString(int beg, int end)
            {
                if (end - beg <= 0) return "";
                long savePos = bStrm.Position;
                char[] arr = new char[end - beg];
                bStrm.Position = (long)beg;
                for (int i = 0; i < (end - beg); i++)
                    arr[i] = (char)bStrm.ReadByte();
                bStrm.Position = savePos;
                return new String(arr);
            }

            // Pos is the position *after* reading chr!
            public override int Pos
            {
                get { return (int)bStrm.Position; }
                set { bStrm.Position = value; }
            }
            
            public static int GetCodePage(string command)
            {
                command = command.ToLower();
                if (command.StartsWith("codepage:"))
                    command = command.Substring(9);
                try
                {
                    if (command.Equals("raw"))
                        return -1;
                    else if (command.Equals("guess"))
                        return -2;
                    else if (command.Equals("default"))
                        return 0;
                    else if (char.IsDigit(command[0]))
                        return int.Parse(command);
                    else
                    {
                        Encoding enc = Encoding.GetEncoding(command);
                        return enc.CodePage;
                    }
                }
                catch
                {
                    Console.Error.WriteLine(
                        "Unknown codepage \"{0}\", using machine default", command);
                    return 0;
                }
            }
        }

#if !BYTEMODE       
        // ==============================================================
        // ===== Nested class CodePageBuff : for unicode text files =====
        // ========= Byte files mapped by a specified CodePage ==========
        // ==============================================================

        public sealed class CodePageBuff : StreamBuff
        {
        
            char[] map = new char[256];
            
            public CodePageBuff(Stream stream) : this(stream, Encoding.Default) { }

            
            public CodePageBuff(Stream stream, Encoding enc) : base(stream)
            {
                bool done;
                int bNum, cNum;
                byte[] bArray = new byte[256];
                if (!enc.IsSingleByte)
                    throw new NotImplementedException(
                        "Only UTF-8, UTF-16 and single-byte code pages allowed");
                Decoder decoder = enc.GetDecoder();
                for (int i = 0; i < 256; i++)
                    bArray[i] = (byte)i;
                decoder.Convert(bArray, 0, 256, this.map, 0, 256, true, out bNum, out cNum, out done);
            }
            
            /// <summary>
            /// Read returns subset unicode char corresponding to the
            /// given character in the chosen codepage.
            /// </summary>
            /// <returns>Unicode point as an int; -1 for EOF</returns>
            public override int Read() {
                int ch0 = bStrm.ReadByte();
                if (ch0 == EOF)
                {
                    delta = 0; return -1;
                }
                else
                {
                    delta = 1; return this.map[ch0];
                } 
            }
            
            public override string GetString(int beg, int end)
            {
                if (end - beg <= 0) return "";
                long savePos = bStrm.Position;
                int saveDelta = delta;
                char[] arr = new char[end - beg];
                bStrm.Position = (long)beg;
                for (int i = 0; i < (end - beg); i++)
                    arr[i] = (char)Read();
                bStrm.Position = savePos;
                delta = saveDelta;
                return new String(arr);
            }
        }
        

        // ==============================================================
        // ====== Nested class TextBuff : for unicode text files ========
        // ==== This is the UTF8 class, UTF16 handled by subclasses =====
        // ==============================================================

        /// <summary>
        /// This is the Buffer for UTF8 files.
        /// It attempts to read the encoding preamble, which for 
        /// this encoding should be unicode point \uFEFF which is 
        /// encoded as EF BB BF
        /// </summary>
        public class TextBuff : ScanBuff
        {
            protected BufferedStream bStrm;   // input buffer
            protected int delta = 1;          // length of chr, zero for EOF!
            

            /// <summary>
            /// TextBuff factory.  Reads the file preamble
            /// and returns a TextBuff, LittleEndTextBuff or
            /// BigEndTextBuff according to the result.
            /// </summary>
            /// <param name="strm">The underlying stream</param>
            /// <returns></returns>
            public static ScanBuff NewTextBuff(Stream strm, int fallbackCodepage)
            {
                // First check if this is a UTF16 file
                //
                int b0 = strm.ReadByte();
                int b1 = strm.ReadByte();

                if (b0 == 0xfe && b1 == 0xff)
                    return new BigEndTextBuff(strm);
                if (b0 == 0xff && b1 == 0xfe)
                    return new LittleEndTextBuff(strm);
                
                int b2 = strm.ReadByte();
                if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
                    return new TextBuff(strm);
                //
                // There is no unicode preamble, so
                // we go back to the bytefile default.
                //
                strm.Seek(0, SeekOrigin.Begin);
                
                switch (fallbackCodepage)
                {
                    case -2:
                        {
                            int guess = new Guesser.Scanner(strm).GuessCodepage();
                            strm.Seek(0, SeekOrigin.Begin);
                            if (guess == 0)
                                return new CodePageBuff(strm);
                            else if (guess == 65001)
                                return new TextBuff(strm);
                            else
                                return new StreamBuff(strm);
                        }                        
                    case -1:
                        return new StreamBuff(strm);
                    case 0:
                        return new CodePageBuff(strm); // Default codepage
                    case 65001:
                        return new TextBuff(strm);
                    case 1200:
                        return new LittleEndTextBuff(strm);
                    case 1201:
                        return new BigEndTextBuff(strm);
                    default:
                        return new CodePageBuff(strm, Encoding.GetEncoding(fallbackCodepage));
                }
            }

            protected TextBuff(Stream str) { 
                this.bStrm = new BufferedStream(str);
            }

            /// <summary>
            /// Read returns UTF32 in this application.
            /// </summary>
            /// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
            public override int Read()
            {
                int ch0 = bStrm.ReadByte();
                int ch1, ch2, ch3,ch4, ch5;
                int result = UnicodeReplacementChar;

                if (ch0 < 0x7f)
                {
                    delta = (ch0 == EOF ? 0 : 1);
                    result = ch0;
                }
                else if ((ch0 & 0xe0) == 0xc0)
                {
                    delta = 2;
                    ch1 = bStrm.ReadByte();
                    if ((ch1 & 0xc0) == 0x80)
                        result = ((ch0 & 0x1f) << 6) + (ch1 & 0x3f);
                }
                else if ((ch0 & 0xf0) == 0xe0)
                {
                    delta = 3;
                    ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte();
                    if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80)
                        result = ((ch0 & 0xf) << 12) + ((ch1 & 0x3f) << 6) + (ch2 & 0x3f);
                }
                else if ((ch0 & 0xf8) == 0xf0)
                {
                    delta = 4;
                    ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte(); ch3 = bStrm.ReadByte();
                    if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80 && (ch3 & 0xc0) == 0x80)
                    {
                        result = ((ch0 & 0x7) << 18) + ((ch1 & 0x3f) << 12) + 
                                 ((ch2 & 0x3f) << 6) + (ch3 & 0x3f);
                    }
                }
                else if ((ch0 & 0xfc) == 0xf8)
                {
                    delta = 5;
                    ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte();
                    ch3 = bStrm.ReadByte(); ch4 = bStrm.ReadByte();
                    if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80 && (ch3 & 0xc0) == 0x80 && (ch4 & 0xc0) == 0x80)
                    {
                        result = ((ch0 & 0x3) << 24) + ((ch1 & 0x3f) << 18) + 
                                     ((ch2 & 0x3f) << 12) + ((ch3 & 0x3f) << 6) + (ch4 & 0x3f);
                    }
                }
                else if ((ch0 & 0xfe) == 0xfc)
                {
                    delta = 6;
                    ch1 = bStrm.ReadByte(); ch2 = bStrm.ReadByte(); ch3 = bStrm.ReadByte();
                    ch4 = bStrm.ReadByte(); ch5 = bStrm.ReadByte();

                    if ((ch1 & 0xc0) == 0x80 && (ch2 & 0xc0) == 0x80 && 
                        (ch3 & 0xc0) == 0x80 && (ch4 & 0xc0) == 0x80 && (ch5 & 0xc0) == 0x80)
                    {
                        result = ((ch0 & 0x1) << 28) + ((ch1 & 0x3f) << 24) + ((ch2 & 0x3f) << 18) + 
                                     ((ch3 & 0x3f) << 12) + ((ch4 & 0x3f) << 6) + (ch5 & 0x3f);
                    }
                }
                return result;
            }

            public sealed override int ReadPos
            {
                get { return (int)bStrm.Position - delta; }
            }

            public sealed override int Peek()
            {
                int rslt = Read();
                bStrm.Seek(-delta, SeekOrigin.Current);
                return rslt;
            }

            /// <summary>
            /// Returns the string from the buffer between
            /// the given file positions.  This needs to be
            /// done carefully, as the number of characters
            /// is, in general, not equal to (end - beg).
            /// </summary>
            /// <param name="beg">Begin filepos</param>
            /// <param name="end">End filepos</param>
            /// <returns></returns>
            public override string GetString(int beg, int end)
            {
                int i;
                if (end - beg <= 0) return "";
                long savePos = bStrm.Position;
                int saveDelta = delta;
                //
                //  The length of the array cannot be larger than
                //  end - beg, since if the characters expand by 
                //  using surrogate pairs the UTF8 must have taken
                //  up at least four bytes for any such character.
                //
                char[] arr = new char[end - beg];
                
                bStrm.Position = (long)beg;
                // Normally each call of Read() returns a valid UTF16
                // value which will take up one place in the string.
                // However, if read returns a value > 0xFFFF then a
                // *pair* of surrogate characters must be added.
                for (i = 0; bStrm.Position < end; i++)
                {
                    int value = Read();
                    if (value < 0xFFFF)
                        arr[i] = (char)value;
                    else
                    {
                        int temp = value - 0x10000;
                        arr[i++] = (char)(0xD800 + temp / 1024);
                        arr[i]   = (char)(0xDC00 + temp % 1024);
                    }
                }
                bStrm.Position = savePos;
                delta = saveDelta;
                return new String(arr, 0, i);
            }

            // Pos is the position *after* reading chr!
            public sealed override int Pos
            {
                get { return (int)bStrm.Position; }
                set { bStrm.Position = value; }
            }
        }
        
        // ==============================================================
        // ====================== Nested class ==========================
        // ==============================================================
        
        /// <summary>
        /// This is the Buffer for Big-endian UTF16 files.
        /// </summary>
        public sealed class BigEndTextBuff : TextBuff
        {
            internal BigEndTextBuff(Stream str) : base(str) { } // 

            private int Read16()
            {
                int ch0 = bStrm.ReadByte();
                int ch1 = bStrm.ReadByte();
                if (ch1 == EOF)
                {
                    // An EOF in either byte counts as an EOF
                    delta = (ch0 == EOF ? 0 : 1);
                    return -1;
                }
                else
                {
                    delta = 2;
                    return (ch0 << 8) + ch1;
                }
            }
            
            /// <summary>
            /// Read returns UTF32 in this application.
            /// </summary>
            /// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
            public override int Read()
            {
                int utf16 = Read16();
                if (utf16 < 0xD800 || utf16 > 0xDBFF)
                    return utf16;
                else
                {
                    int low16 = Read16();
                    delta = 4;
                    if (low16 < 0xDC00 || low16 > 0xDFFF)
                        return UnicodeReplacementChar;
                    else
                        return (0x10000 + (utf16 & 0x3FF << 10) + (low16 & 0x3FF));
                }
            }
            
            /// <summary>
            /// Returns the string from the buffer between
            /// the given file positions.  This needs to be
            /// done carefully, as the number of characters
            /// is, in general, not equal to (end - beg).
            /// </summary>
            /// <param name="beg">Begin filepos</param>
            /// <param name="end">End filepos</param>
            /// <returns></returns>
            public sealed override string GetString(int beg, int end)
            {
                int i;
                if (end - beg <= 0) return "";
                long savePos = bStrm.Position;
                int saveDelta = delta;
                char[] arr = new char[end - beg];
                bStrm.Position = (long)beg;
                // This stream will have utf16 valid for strings
                for (i = 0; bStrm.Position < end; i++)
                    arr[i] = (char)Read();
                bStrm.Position = savePos;
                delta = saveDelta;
                return new String(arr, 0, i);
            }
        }

        // ==============================================================
        // ====================== Nested class ==========================
        // ==============================================================
        
        /// <summary>
        /// This is the Buffer for Little-endian UTF16 files.
        /// </summary>
        public sealed class LittleEndTextBuff : TextBuff
        {
            internal LittleEndTextBuff(Stream str) : base(str) { } // { this.bStrm = new BufferedStream(str); }

            private int Read16()
            {
                int ch0 = bStrm.ReadByte();
                int ch1 = bStrm.ReadByte();
                if (ch1 == EOF)
                {
                    // An EOF in either byte counts as an EOF
                    delta = (ch0 == EOF ? 0 : 1);
                    return -1;
                }
                else
                {
                    delta = 2;
                    return (ch1 << 8) + ch0;
                }
            }
            
            /// <summary>
            /// Read returns UTF32 in this application.
            /// </summary>
            /// <returns>Unicode point as an int [0 .. 0xFFFFF]; -1 for EOF</returns>
            public override int Read()
            {
                int utf16 = Read16();
                if (utf16 < 0xD800 || utf16 > 0xDBFF)
                    return utf16;
                else
                {
                    int low16 = Read16();
                    delta = 4;
                    if (low16 < 0xDC00 || low16 > 0xDFFF)
                        return UnicodeReplacementChar;
                    else
                        return (0x10000 + (utf16 & 0x3FF << 10) + (low16 & 0x3FF));
                }
            }
            
            /// <summary>
            /// Returns the string from the buffer between
            /// the given file positions.  This needs to be
            /// done carefully, as the number of characters
            /// is, in general, not equal to (end - beg).
            /// </summary>
            /// <param name="beg">Begin filepos</param>
            /// <param name="end">End filepos</param>
            /// <returns></returns>
            public sealed override string GetString(int beg, int end)
            {
                int i;
                if (end - beg <= 0) return "";
                long savePos = bStrm.Position;
                int saveDelta = delta;
                char[] arr = new char[end - beg];
                bStrm.Position = (long)beg;
                // This stream will have utf16 valid for strings
                for (i = 0; bStrm.Position < end; i++)
                    arr[i] = (char)Read16();
                bStrm.Position = savePos;
                delta = saveDelta;
                return new String(arr, 0, i);
            }
        }
#endif // !BYTEMODE
#endif // !NOFILES
#endregion Buffer classes

        // =================== End Nested classes =======================

#if !NOFILES
        public Scanner(Stream file) {
##-->bufferCtor
        }
        
#endif // !NOFILES

        public Scanner() { }

        void GetChr()
        {
            if (chr == '\n')  // This needs to be fixed for other conventions
            { 
                lineStartNum = cNum + 1; 
                lNum++; 
            }
            chr = buffer.Read();
            cNum++;
        }

        void MarkToken()
        {
            tokPos = buffer.ReadPos;
            tokNum = cNum;
            tokLin = lNum;
            tokCol = cNum - lineStartNum;
        }
        
        void MarkEnd()
        {
            tokTxt = null;
            tokLen = cNum - tokNum;
            tokEPos = buffer.ReadPos;
            tokELin = lNum;
            tokECol = cNum - lineStartNum;
        }

        // ==============================================================
        // =====    Initialization of string-based input buffers     ====
        // ==============================================================

        /// <summary>
        /// Create and initialize a StringBuff buffer object for this scanner
        /// </summary>
        /// <param name="source">the input string</param>
        /// <param name="offset">starting offset in the string</param>
        public void SetSource(string source, int offset)
        {
            this.buffer = new StringBuff(source);
            this.buffer.Pos = offset;
            this.lNum = 0;
            this.cNum = offset - 1;
            this.chr = '\n'; // to initialize yyline, yycol and lineStart
            GetChr();
        }

#if !NOFILES        
        // ================ LineBuffer Initialization ===================

        /// <summary>
        /// Create and initialize a LineBuff buffer object for this scanner
        /// </summary>
        /// <param name="source">the list of input strings</param>
        public void SetSource(IList<string> source)
        {
            this.buffer = new LineBuff(source);
            this.chr = '\n'; // to initialize yyline, yycol and lineStart
            this.lNum = 0;
            this.cNum = -1;
            GetChr();
        }

        // =============== StreamBuffer Initialization ==================

        /// <summary>
        /// Create and initialize a StreamBuff buffer object for this scanner.
        /// StreamBuff is buffer for 8-bit byte files.
        /// </summary>
        /// <param name="source">the input byte stream</param>
        public void SetSource(Stream source)
        {
            this.buffer = new StreamBuff(source);
            this.lNum = 0;
            this.cNum = -1;
            this.chr = '\n'; // to initialize yyline, yycol and lineStart
            GetChr();
        }
        
#if !BYTEMODE
        // ================ TextBuffer Initialization ===================

        /// <summary>
        /// Create and initialize a TextBuff buffer object for this scanner.
        /// TextBuff is a buffer for encoded unicode files.
        /// </summary>
        /// <param name="source">the input text file</param>
        /// <param name="fallbackCodepage">Codepage to use if file has
        /// no BOM. For 0, use machine default; for -1, 8-bit binary</param>
        public void SetSource(Stream source, int fallbackCodepage)
        {
            this.buffer = TextBuff.NewTextBuff(source, fallbackCodepage);
            this.lNum = 0;
            this.cNum = -1;
            this.chr = '\n'; // to initialize yyline, yycol and lineStart
            GetChr();
        }
#endif // !BYTEMODE
#endif // !NOFILES
        
        // ==============================================================

#if BABEL
        //
        //  Get the next token for Visual Studio
        //
        //  "state" is the inout mode variable that maintains scanner
        //  state between calls, using the EolState property. In principle,
        //  if the calls of EolState are costly set could be called once
        //  only per line, at the start; and get called only at the end
        //  of the line. This needs more infrastructure ...
        //
        public int GetNext(ref int state, out int start, out int end)
        {
            Tokens next;
            int s, e;
            s = state;        // state at start
            EolState = state;
            next = (Tokens)Scan();
            state = EolState;
            e = state;       // state at end;
            start = tokPos;
            end = tokEPos - 1; // end is the index of last char.
            return (int)next;
        }        
#endif // BABEL

        // ======== IScanner<> Implementation =========

        public override int yylex()
        {
            // parserMax is set by reflecting on the Tokens
            // enumeration.  If maxParseToken is defined
            // that is used, otherwise int.MaxValue is used.
            int next;
            do { next = Scan(); } while (next >= parserMax);
            return next;
        }
        
        int yyleng { get { return tokLen; } }
        int yypos { get { return tokPos; } }
        int yyline { get { return tokLin; } }
        int yycol { get { return tokCol; } }

        public string yytext
        {
            get 
            {
                if (tokTxt == null) 
                    tokTxt = buffer.GetString(tokPos, tokEPos);
                return tokTxt;
            }
        }

        void yyless(int n) { 
            buffer.Pos = tokPos;
            // Must read at least one char, so set before start.
            cNum = tokNum - 1; GetChr();
            // Now ensure that line counting is correct.
            lNum = tokLin;
            // And count the rest of the text.
            for (int i = 0; i < n; i++) GetChr();
            MarkEnd();
        }

        // ============ methods available in actions ==============

        internal int YY_START {
            get { return currentScOrd; }
            set { currentScOrd = value; 
                  currentStart = startState[value]; 
            } 
        }
        
        internal void BEGIN(int next) {
            currentScOrd = next;
            currentStart = startState[next];
        }

        // ============== The main tokenizer code =================

        int Scan()
        {
##-->prolog 
                for (; ; )
                {
                    int next;              // next state to enter                   
#if BACKUP
                    Result rslt = Result.noMatch;
#endif // BACKUP
#if LEFTANCHORS
                    for (;;)
                    {
                        // Discard characters that do not start any pattern.
                        // Must check the left anchor condition after *every* GetChr!
                        state = (lineStartNum == cNum ? anchorState[currentScOrd] : currentStart);
                        if ((next = NextState()) != goStart) 
                            break; // LOOP EXIT HERE...
                        GetChr();
                    }
                    
#else // !LEFTANCHORS
                    state = currentStart;
                    while ((next = NextState()) == goStart)
                        // At this point, the current character has no
                        // transition from the current state.  We discard 
                        // the "no-match" char.   In traditional LEX such 
                        // characters are echoed to the console.
                        GetChr();
#endif // LEFTANCHORS                    
                    // At last, a valid transition ...    
                    MarkToken();
                    state = next;
                    GetChr();
                    
                    while ((next = NextState()) > eofNum) // Exit for goStart AND for eofNum
#if BACKUP
                        if (state <= maxAccept && next > maxAccept) // need to prepare backup data
                        {
                            // ctx is an object. The fields may be 
                            // mutated by the call to Recurse2.
                            // On return the data in ctx is the
                            // *latest* accept state that was found.
                            Context ctx = new Context();
                            rslt = Recurse2(ctx, next);
                            if (rslt == Result.noMatch) 
                                RestoreStateAndPos(ctx);
                            break;
                        }
                        else
#endif // BACKUP
                        {
                            state = next;
                            GetChr();
                        }
                    if (state <= maxAccept) 
                    {
                        MarkEnd();
##-->actionCases
                    }
                }
##-->epilog
        }

#if BACKUP
        Result Recurse2(Context ctx, int next)
        {
            // Assert: at entry "state" is an accept state AND
            //         NextState(state, chr) != goStart AND
            //         NextState(state, chr) is not an accept state.
            //
            SaveStateAndPos(ctx);
            state = next;
            // if (state == eofNum) return Result.accept;
            GetChr();

            while ((next = NextState()) > eofNum)
            {
                if (state <= maxAccept && next > maxAccept) // need to update backup data
                    SaveStateAndPos(ctx);
                state = next;
                if (state == eofNum) return Result.accept;
                GetChr(); 
            }
            return (state <= maxAccept ? Result.accept : Result.noMatch);
        }

        void SaveStateAndPos(Context ctx)
        {
            ctx.bPos  = buffer.Pos;
            ctx.cNum  = cNum;
            ctx.lNum  = lNum;
            ctx.state = state;
            ctx.cChr  = chr;
        }

        void RestoreStateAndPos(Context ctx)
        {
            buffer.Pos = ctx.bPos;
            cNum  = ctx.cNum;
            lNum  = ctx.lNum;
            state = ctx.state;
            chr   = ctx.cChr;
        }

        void RestorePos(Context ctx) { buffer.Pos = ctx.bPos; cNum = ctx.cNum; }
#endif // BACKUP

        // ============= End of the tokenizer code ================

#if STACK        
        internal void yy_clear_stack() { scStack.Clear(); }
        internal int yy_top_state() { return scStack.Peek(); }
        
        internal void yy_push_state(int state)
        {
            scStack.Push(currentScOrd);
            BEGIN(state);
        }
        
        internal void yy_pop_state()
        {
            // Protect against input errors that pop too far ...
            if (scStack.Count > 0) {
				int newSc = scStack.Pop();
				BEGIN(newSc);
            } // Otherwise leave stack unchanged.
        }
 #endif // STACK

        internal void ECHO() { Console.Out.Write(yytext); }
        
##-->userCode
    } // end class Scanner
} // end namespace