BaseTokenizer abstract class BaseTokenizer : IDisposable Common methods and variables of all tokenizers. Documentation Code protected readonly IEventAggregator _events protected readonly StringBuilder _stringBuffer public ushort Column { get; } protected char Current { get; } public int InsertionPoint { get; protected set; } public ushort Line { get; } public int Position { get; } public TextSource Source { get; } public BaseTokenizer(TextSource source, IEventAggregator events) protected void Advance() protected void Advance(int n) protected void Back() protected void Back(int n) protected bool ContinuesWithInsensitive(string s) protected bool ContinuesWithSensitive(string s) public void Dispose() public string FlushBuffer() public TextPosition GetCurrentPosition() protected char GetNext() protected char GetPrevious() protected string PeekString(int length) protected char SkipSpaces()