FSMLexer 0.0.1

Flexer

Finite-state-machine based lexer.

Flexer is a lexer library that acts like a finite state machine.

Lexer is supposed to split the given text document into a list of meaningful tokens, and in many cases you can process your text from left to right, making most choices based on the current state and current character in stream. This is the idea behind Flexer.

Quick start

Following example simply splits given string into words

namespace Flexer.QuickSample
{
    // Type alias for easier time using Character Classes.
    using CharCls = Flexer<Program.FlexerStates>.Rule.CharClass;

    internal class Program
    {
        // These are states, that Flexer can use. Some of them are tokens.
        public enum FlexerStates
        {
            Start,  // Initial state, used to skip spaces in
                    // the start of the document.
            Word,   // Word state, which acts also as a token.
            Space   // Space state.
        }

        static void Main(string[] args)
        {
            Flexer<FlexerStates> flexer = new();

            // Registering states takes approach of
            // writing these script-like instructions.
            flexer.Register(FlexerStates.Start)
                .When(CharCls.Whitespace).Skip()
                // Skip spaces at the start.
                .When(CharCls.Any).ChangeState(FlexerStates.Word)   
                // Any non-space symbol indicates start of a word
                .WhenEofDropState();    
                // If document is empty (from our perspective),
                // we don't need to get any tokens from it

            flexer.Register(FlexerStates.Word)
                .When(CharCls.Whitespace).NextState(FlexerStates.Space)
                // When encounter whitespace, yield the current word token
                // and change state to "Space"
                .When(CharCls.Any).Consume()
                // Any non-whitespace character is part of a word and must
                // be consumed in token
                .WhenEofYieldState();
                // Handle case when last word ends where ends the document

            flexer.Register(FlexerStates.Space)
                .When(CharCls.Whitespace).Skip()
                // There is no reason to keep whitespaces in result
                .When(CharCls.Any).ChangeState(FlexerStates.Word)
                // If we encounter anything other than whitespace, then should start word
                .WhenEofDropState();
                // No need to remember the last state

            foreach (var token in flexer.Flex("Lorem Ipsumn      is\n\t\rsimply" +
                " \t\n     dummy\r\n\ttext."))
            {
                Console.WriteLine($"Line {token.line} pos {token.pos}: " +
                    $"{token.type}({token.str})");
            }
        }
    }
}

If we launch this program, we would get following result in console:

Line 1 pos 6: Word(Lorem)
Line 1 pos 15: Word(Ipsumn)
Line 2 pos 0: Word(is)
Line 3 pos 10: Word(simply)
Line 4 pos 12: Word(dummy)
Line 5 pos 7: Word(text.)

Further reading

More examples:

• More sofisticated examples: Flexer.Samples