Tokeniser.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;

namespace ExpressionParser
{
   public class Tokeniser
   {
      private readonly string text;
      private int pos;
      public Token CurrentToken { get; private set; }

      public Tokeniser(string code)
      {
         this.text = code;
         this.pos = 0;
         CurrentToken = new Token(TokenType.End,""); // Added to remove warning about non-initialised property

         NextToken();
        
      }

      // Return the current character or the null character if at the end.
      private char CurrentChar => pos < text.Length ? text[pos] : '\0';

      public void NextToken()
      {
         SkipWhitespace();

         if (pos >= text.Length)
         {
            CurrentToken = new Token(TokenType.End, "");
            return;
         }

         char ch = CurrentChar;

         if (ch == '+')
         {
            pos++;
            CurrentToken = new Token(TokenType.Plus, "+");
            return;
         }

         if (ch == '-')
         {
            pos++;
            CurrentToken = new Token(TokenType.Minus, "-");
            return;
         }

         if (ch == '*')
         {
            pos++;
            CurrentToken = new Token(TokenType.Multiply, "*");
            return;
         }

         if (ch == '/')
         {
            pos++;
            CurrentToken = new Token(TokenType.Divide, "/");
            return;
         }

         if (ch == '(')
         {
            pos++;
            CurrentToken = new Token(TokenType.LeftParen, "(");
            return;
         }

         if (ch == ')')
         {
            pos++;
            CurrentToken = new Token(TokenType.RightParen, ")");
            return;
         }

         if (ch == ',')
         {
            pos++;
            CurrentToken = new Token(TokenType.Comma, ",");
            return;
         }

         // Variable names are enclosed in { and }.
         if (ch == '{')
         {
            pos++; // consume '{'
            int start = pos;
            while (pos < text.Length && text[pos] != '}')
            {
               if (!char.IsLetterOrDigit(text[pos]) && ch != '.')
                  throw new Exception("Invalid character in variable name");
               pos++;
            }
            if (pos >= text.Length)
               throw new Exception("Unterminated variable token");

            string varName = text.Substring(start, pos - start).Trim();
            pos++; // consume '}'
            CurrentToken = new Token(TokenType.Variable, varName);
            return;
         }

         // String literal: accepts both '...' and "..."
         if (ch == '\'' || ch == '\"')
         {
            char quote = ch;
            pos++; // skip opening quote
            int start = pos;
            while (pos < text.Length && text[pos] != quote)
            {
               pos++;
            }
            if (pos >= text.Length)
               throw new Exception("Unterminated string literal");

            string strVal = text.Substring(start, pos - start);
            pos++; // skip closing quote
            CurrentToken = new Token(TokenType.String, strVal);
            return;
         }

         // Numeric literal.
         if (char.IsDigit(ch) || (ch == '.' && pos + 1 < text.Length && char.IsDigit(text[pos + 1])))
         {
            int start = pos;
            bool foundDot = false;
            while (pos < text.Length && (char.IsDigit(text[pos]) || text[pos] == '.'))
            {
               if (text[pos] == '.')
               {
                  if (foundDot) break;
                  foundDot = true;
               }
               pos++;
            }
            string numStr = text.Substring(start, pos - start);
            CurrentToken = new Token(TokenType.Number, numStr);
            return;
         }

         // Identifier: letter followed by letters/digits or underscore. (Used for function names)
         if (char.IsLetter(ch))
         {
            int start = pos;
            while (pos < text.Length && (char.IsLetterOrDigit(text[pos]) || text[pos] == '_'))
            {
               pos++;
            }
            string id = text.Substring(start, pos - start);
            CurrentToken = new Token(TokenType.Identifier, id);
            return;
         }

         throw new Exception($"Unexpected character: {ch}");

      }

      private void SkipWhitespace()
      {
         while (pos < text.Length && (char.IsWhiteSpace(text[pos]) || text[pos] == '\0'))
         {
            pos++;
         }
      }
   }
}