BaseTokenizer abstract class BaseTokenizer : IDisposable Common methods and variables of all tokenizers. Documentation Code public ushort Column { get; } protected char Current { get; } public int InsertionPoint { get; protected set; } public ushort Line { get; } public int Position { get; } public TextSource Source { get; } protected StringBuilder StringBuffer { get; } public BaseTokenizer(TextSource source) protected void Advance() protected void Advance(int n) protected void Back() protected void Back(int n) protected bool ContinuesWithInsensitive(string s) protected bool ContinuesWithSensitive(string s) public void Dispose() public string FlushBuffer() public TextPosition GetCurrentPosition() protected char GetNext() protected char GetPrevious() protected string PeekString(int length) protected char SkipSpaces()