BaseTokenizer
Common methods and variables of all tokenizers.
using AngleSharp.Events;
using AngleSharp.Extensions;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;
namespace AngleSharp.Parser
{
[DebuggerStepThrough]
internal abstract class BaseTokenizer : IDisposable
{
protected readonly StringBuilder _stringBuffer;
protected readonly IEventAggregator _events;
private readonly Stack<ushort> _columns;
private readonly TextSource _source;
private ushort _column;
private ushort _row;
private char _current;
public int InsertionPoint {
get {
return _source.Index;
}
set {
int i;
for (i = _source.Index - value; i > 0; i--) {
BackUnsafe();
}
for (; i < 0; i++) {
AdvanceUnsafe();
}
}
}
public ushort Line => _row;
public ushort Column => _column;
public int Position => _source.Index;
protected char Current => _current;
public BaseTokenizer(TextSource source, IEventAggregator events)
{
_stringBuffer = Pool.NewStringBuilder();
_events = events;
_columns = new Stack<ushort>();
_source = source;
_current = ' ';
_column = 0;
_row = 1;
}
public void Dispose()
{
((IDisposable)_source)?.Dispose();
_stringBuffer.ToPool();
}
public TextPosition GetCurrentPosition()
{
return new TextPosition(_row, _column, Position);
}
protected bool ContinuesWith(string s, bool ignoreCase = true)
{
int index = _source.Index;
_source.Index--;
string text = _source.ReadCharacters(s.Length);
_source.Index = index;
if (text.Length == s.Length)
return text.Equals(s, ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
return false;
}
public void ResetInsertionPoint()
{
InsertionPoint = _source.Length;
}
protected char SkipSpaces()
{
char next = GetNext();
while (next.IsSpaceCharacter()) {
next = GetNext();
}
return next;
}
protected void SkipMostSpaces()
{
SkipSpaces();
Back();
}
protected char GetNext()
{
Advance();
return _current;
}
protected char GetPrevious()
{
Back();
return _current;
}
protected void Advance()
{
if (_current != '')
AdvanceUnsafe();
}
protected void Advance(int n)
{
while (n-- > 0 && _current != '') {
AdvanceUnsafe();
}
}
protected void Back()
{
if (InsertionPoint > 0)
BackUnsafe();
}
protected void Back(int n)
{
while (n-- > 0 && InsertionPoint > 0) {
BackUnsafe();
}
}
private void AdvanceUnsafe()
{
if (_current.IsLineBreak()) {
_columns.Push(_column);
_column = 0;
_row++;
}
_column++;
for (_current = _source.ReadCharacter(); _current == '\r'; _current = _source.ReadCharacter()) {
}
}
private void BackUnsafe()
{
_source.Index--;
if (_source.Index == 0) {
_column = 0;
_current = ' ';
} else {
_current = _source[_source.Index - 1];
if (_current == '\r')
BackUnsafe();
else if (_current.IsLineBreak()) {
_column = (ushort)((_columns.Count == 0) ? 1 : _columns.Pop());
_row--;
} else {
_column--;
}
}
}
}
}