//! Contains the [`Token`] struct and its related types.

use std::{borrow::Cow, collections::HashMap, fmt::Display, str::FromStr, sync::OnceLock};

use crate::base::{
    self,
    log::SourceCodeDisplay,
    source_file::{SourceElement, SourceIterator, Span},
    Handler,
};
use derive_more::From;
use enum_as_inner::EnumAsInner;
use strum::IntoEnumIterator;
use strum_macros::EnumIter;

use super::error::{self, UnterminatedDelimitedComment};

/// Is an enumeration representing keywords in shulkerscript.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, EnumIter)]
#[allow(missing_docs)]
pub enum KeywordKind {
    Function,
    Pub,
    If,
    Else,
    Align,
    Anchored,
    As,
    AsAt,
    At,
    Facing,
    In,
    On,
    Positioned,
    Rotated,
    Store,
    Summon,
    Group,
    Run,
    Lua,
    Namespace,
    From,
    Import,
    Tag,
    Of,
    Replace,
}

impl Display for KeywordKind {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.as_str())
    }
}

/// Is an error that is returned when a string cannot be parsed into a [`Keyword`] in [`FromStr`]
/// trait implementation.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, thiserror::Error)]
#[error("invalid string representation of keyword.")]
pub struct KeywordParseError;

impl FromStr for KeywordKind {
    type Err = KeywordParseError;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        static STRING_KEYWORD_MAP: OnceLock<HashMap<&'static str, KeywordKind>> = OnceLock::new();
        let map = STRING_KEYWORD_MAP.get_or_init(|| {
            let mut map = HashMap::new();

            for keyword in Self::iter() {
                map.insert(keyword.as_str(), keyword);
            }

            map
        });

        map.get(s).copied().ok_or(KeywordParseError)
    }
}

impl KeywordKind {
    /// Gets the string representation of the keyword as a `&str`.
    #[must_use]
    pub fn as_str(self) -> &'static str {
        match self {
            Self::Function => "fn",
            Self::Pub => "pub",
            Self::If => "if",
            Self::Else => "else",
            Self::Align => "align",
            Self::Anchored => "anchored",
            Self::As => "as",
            Self::AsAt => "asat",
            Self::At => "at",
            Self::Facing => "facing",
            Self::In => "in",
            Self::On => "on",
            Self::Positioned => "positioned",
            Self::Rotated => "rotated",
            Self::Store => "store",
            Self::Summon => "summon",
            Self::Group => "group",
            Self::Run => "run",
            Self::Lua => "lua",
            Self::Namespace => "namespace",
            Self::From => "from",
            Self::Import => "import",
            Self::Tag => "tag",
            Self::Of => "of",
            Self::Replace => "replace",
        }
    }

    /// Whether the keyword starts an execute block.
    #[must_use]
    pub fn starts_execute_block(&self) -> bool {
        matches!(
            self,
            Self::If
                | Self::Align
                | Self::Anchored
                | Self::As
                | Self::AsAt
                | Self::At
                | Self::Facing
                | Self::In
                | Self::On
                | Self::Positioned
                | Self::Rotated
                | Self::Store
                | Self::Summon
        )
    }
}

/// Is an enumeration containing all kinds of tokens in the Shulkerscript programming language.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, From, EnumAsInner)]
#[allow(missing_docs)]
pub enum Token {
    WhiteSpaces(WhiteSpaces),
    Identifier(Identifier),
    Keyword(Keyword),
    Punctuation(Punctuation),
    Numeric(Numeric),
    Comment(Comment),
    DocComment(DocComment),
    CommandLiteral(CommandLiteral),
    StringLiteral(StringLiteral),
    MacroStringLiteral(MacroStringLiteral),
}

impl SourceElement for Token {
    fn span(&self) -> Span {
        match self {
            Self::WhiteSpaces(token) => token.span(),
            Self::Identifier(token) => token.span(),
            Self::Keyword(token) => token.span(),
            Self::Punctuation(token) => token.span(),
            Self::Numeric(token) => token.span(),
            Self::Comment(token) => token.span(),
            Self::DocComment(token) => token.span(),
            Self::CommandLiteral(token) => token.span(),
            Self::StringLiteral(token) => token.span(),
            Self::MacroStringLiteral(token) => token.span(),
        }
    }
}

/// Represents a contiguous sequence of whitespace characters.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct WhiteSpaces {
    /// Is the span that makes up the token.
    pub span: Span,
}

impl SourceElement for WhiteSpaces {
    fn span(&self) -> Span {
        self.span.clone()
    }
}
/// Represents a contiguous sequence of characters that are valid in an identifier.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Identifier {
    /// Is the span that makes up the token.
    pub span: Span,
}

impl SourceElement for Identifier {
    fn span(&self) -> Span {
        self.span.clone()
    }
}

/// Represents a contiguous sequence of characters that are reserved for a keyword.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Keyword {
    /// Is the span that makes up the token.
    pub span: Span,

    /// Is the [`KeywordKind`] that the token represents.
    pub keyword: KeywordKind,
}

impl SourceElement for Keyword {
    fn span(&self) -> Span {
        self.span.clone()
    }
}

/// Represents a single ASCII punctuation character.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Punctuation {
    /// Is the span that makes up the token.
    pub span: Span,

    /// Is the ASCII punctuation character that the token represents.
    pub punctuation: char,
}

impl SourceElement for Punctuation {
    fn span(&self) -> Span {
        self.span.clone()
    }
}

/// Represents a hardcoded numeric literal value in the source code.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Numeric {
    /// Is the span that makes up the token.
    pub span: Span,
}

impl SourceElement for Numeric {
    fn span(&self) -> Span {
        self.span.clone()
    }
}

/// Represents a hardcoded string literal value in the source code.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct StringLiteral {
    /// Is the span that makes up the token.
    pub span: Span,
}

impl StringLiteral {
    /// Returns the string content without escapement characters, leading and trailing double quotes.
    #[must_use]
    pub fn str_content(&self) -> Cow<str> {
        let string = self.span.str();
        let string = &string[1..string.len() - 1];
        if string.contains('\\') {
            let escaped = string
                .replace("\\n", "\n")
                .replace("\\r", "\r")
                .replace("\\t", "\t")
                .replace("\\\"", "\"")
                .replace("\\\\", "\\");
            Cow::Owned(escaped)
        } else {
            Cow::Borrowed(string)
        }
    }
}

impl SourceElement for StringLiteral {
    fn span(&self) -> Span {
        self.span.clone()
    }
}

/// Represents a hardcoded macro string literal value in the source code.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MacroStringLiteral {
    /// The backtick that starts the macro string literal.
    starting_backtick: Punctuation,
    /// The parts that make up the macro string literal.
    parts: Vec<MacroStringLiteralPart>,
    /// The backtick that ends the macro string literal.
    ending_backtick: Punctuation,
}

impl MacroStringLiteral {
    /// Returns the string content without escapement characters, leading and trailing double quotes.
    #[must_use]
    pub fn str_content(&self) -> String {
        use std::fmt::Write;

        let mut content = String::new();

        for part in &self.parts {
            match part {
                MacroStringLiteralPart::Text(span) => {
                    content += &crate::util::unescape_macro_string(span.str());
                }
                MacroStringLiteralPart::MacroUsage { identifier, .. } => {
                    write!(
                        content,
                        "$({})",
                        crate::transpile::util::identifier_to_macro(identifier.span.str())
                    )
                    .expect("can always write to string");
                }
            }
        }

        content
    }

    /// Returns the parts that make up the macro string literal.
    #[must_use]
    pub fn parts(&self) -> &[MacroStringLiteralPart] {
        &self.parts
    }
}

impl SourceElement for MacroStringLiteral {
    fn span(&self) -> Span {
        self.starting_backtick
            .span
            .join(&self.ending_backtick.span)
            .expect("Invalid macro string literal span")
    }
}

/// Represents a part of a macro string literal value in the source code.
#[allow(missing_docs)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum MacroStringLiteralPart {
    Text(Span),
    MacroUsage {
        dollar: Punctuation,
        open_brace: Punctuation,
        identifier: Identifier,
        close_brace: Punctuation,
    },
}

/// Is an enumeration representing the two kinds of comments in the Shulkerscript programming language.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum CommentKind {
    /// A comment that starts with `//` and ends at the end of the line.
    Line,

    /// A comment that starts with `/*` and ends with `*/`.
    Delimited,
}

/// Represents a portion of the source code that is ignored by the interpreter.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Comment {
    /// Is the span that makes up the token.
    pub span: Span,

    /// Is the kind of comment that the token represents.
    pub kind: CommentKind,
}

impl SourceElement for Comment {
    fn span(&self) -> Span {
        self.span.clone()
    }
}

/// Represents a documentation comment in the source code.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct DocComment {
    /// Is the span that makes up the token.
    pub span: Span,
}

impl SourceElement for DocComment {
    fn span(&self) -> Span {
        self.span.clone()
    }
}

impl DocComment {
    /// Returns the content of the doc comment without the leading `///`.
    #[must_use]
    pub fn content(&self) -> &str {
        &self.span.str().trim()[3..]
    }
}

/// Represents a hardcoded literal command in the source code.
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct CommandLiteral {
    /// Span that makes up the token.
    pub span: Span,
}

impl SourceElement for CommandLiteral {
    fn span(&self) -> Span {
        self.span.clone()
    }
}
impl CommandLiteral {
    /// Returns the command without the leading slash.
    #[must_use]
    pub fn clean_command(&self) -> &str {
        &self.span.str().trim()[1..]
    }
}

/// Is an error that can occur when invoking the [`Token::tokenize`] method.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, thiserror::Error)]
#[allow(missing_docs)]
pub enum TokenizeError {
    #[error("encountered a fatal lexical error that causes the process to stop.")]
    FatalLexicalError,

    #[error("the iterator argument is at the end of the source code.")]
    EndOfSourceCodeIteratorArgument,

    #[error(transparent)]
    InvalidMacroNameCharacter(#[from] InvalidMacroNameCharacter),

    #[error(transparent)]
    UnclosedMacroUsage(#[from] UnclosedMacroUsage),

    #[error(transparent)]
    EmptyMacroUsage(#[from] EmptyMacroUsage),
}

/// Is an error that can occur when the macro name contains invalid characters.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct InvalidMacroNameCharacter {
    /// The span of the invalid characters.
    pub span: Span,
}

impl Display for InvalidMacroNameCharacter {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "{}",
            base::log::Message::new(base::log::Severity::Error, format!("The macro name contains invalid characters: `{}`. Only alphanumeric characters and underscores are allowed.", self.span.str()))
        )?;
        write!(
            f,
            "\n{}",
            SourceCodeDisplay::new(&self.span, Option::<u8>::None)
        )
    }
}

impl std::error::Error for InvalidMacroNameCharacter {}

/// Is an error that can occur when the macro usage is not closed.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct UnclosedMacroUsage {
    /// The span of the unclosed macro usage.
    pub span: Span,
}

impl Display for UnclosedMacroUsage {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "{}",
            base::log::Message::new(
                base::log::Severity::Error,
                "A macro usage was opened with `$(` but never closed."
            )
        )?;
        write!(
            f,
            "\n{}",
            SourceCodeDisplay::new(&self.span, Option::<u8>::None)
        )
    }
}

impl std::error::Error for UnclosedMacroUsage {}

/// Is an error that can occur when the macro usage is not closed.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct EmptyMacroUsage {
    /// The span of the unclosed macro usage.
    pub span: Span,
}

impl Display for EmptyMacroUsage {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "{}",
            base::log::Message::new(
                base::log::Severity::Error,
                "A macro usage was opened with `$(` but closed immediately with `)`."
            )
        )?;
        write!(
            f,
            "\n{}",
            SourceCodeDisplay::new(&self.span, Option::<u8>::None)
        )
    }
}

impl std::error::Error for EmptyMacroUsage {}

impl Token {
    /// Increments the iterator while the predicate returns true.
    pub fn walk_iter(iter: &mut SourceIterator, predicate: impl Fn(char) -> bool) {
        while let Some((_, character)) = iter.peek() {
            if !predicate(character) {
                break;
            }

            iter.next();
        }
    }

    /// Creates a span from the given start location to the current location of the iterator.
    #[must_use]
    fn create_span(start: usize, iter: &mut SourceIterator) -> Span {
        iter.peek().map_or_else(
            || Span::to_end(iter.source_file().clone(), start).unwrap(),
            |(index, _)| Span::new(iter.source_file().clone(), start, index).unwrap(),
        )
    }

    /// Creates a span from the given start location to the current location of the iterator with the given offset.
    #[must_use]
    fn create_span_with_end_offset(
        start: usize,
        iter: &mut SourceIterator,
        end_offset: isize,
    ) -> Span {
        iter.peek().map_or_else(
            || Span::to_end_with_offset(iter.source_file().clone(), start, end_offset).unwrap(),
            |(index, _)| {
                Span::new(
                    iter.source_file().clone(),
                    start,
                    index.saturating_add_signed(end_offset),
                )
                .unwrap()
            },
        )
    }

    /// Checks if the given character is a valid first character of an identifier.
    fn is_first_identifier_character(character: char) -> bool {
        character == '_'
            || (!character.is_control()
                && !character.is_whitespace()
                && !character.is_ascii_punctuation()
                && !character.is_ascii_digit())
    }

    /// Checks if the given character is a valid character of an identifier.
    fn is_identifier_character(character: char) -> bool {
        character == '_'
            || (!character.is_control()
                && !character.is_whitespace()
                && !character.is_ascii_punctuation())
    }

    /// Handles a contiguous sequence of whitespace characters.
    fn handle_whitespace(iter: &mut SourceIterator, start: usize) -> Self {
        Self::walk_iter(iter, char::is_whitespace);

        WhiteSpaces {
            span: Self::create_span(start, iter),
        }
        .into()
    }

    /// Handles a contiguous sequence of characters that are valid in an identifier.
    fn handle_identifier_and_keyword(iter: &mut SourceIterator, start: usize) -> Self {
        Self::walk_iter(iter, Self::is_identifier_character);

        let span = Self::create_span(start, iter);
        let word = span.str();

        // Checks if the word is a keyword
        KeywordKind::from_str(word).ok().map_or_else(
            || Identifier { span: span.clone() }.into(),
            |kw| {
                Keyword {
                    span: span.clone(),
                    keyword: kw,
                }
                .into()
            },
        )
    }

    /// Handles a sequence starting with a slash
    fn handle_comment(
        iter: &mut SourceIterator,
        start: usize,
        character: char,
        prev_token: Option<&Self>,
        handler: &impl Handler<base::Error>,
    ) -> Result<Self, TokenizeError> {
        // Single line comment
        if let Some((_, '/')) = iter.peek() {
            iter.next();

            let is_doccomment = if let Some((_, '/')) = iter.peek() {
                iter.next();
                true
            } else {
                false
            };

            Self::walk_iter(iter, |character| !(character == '\n' || character == '\r'));

            let span = Self::create_span(start, iter);

            let comment = if is_doccomment {
                DocComment { span }.into()
            } else {
                Comment {
                    span,
                    kind: CommentKind::Line,
                }
                .into()
            };

            Ok(comment)
        }
        // Delimited comment
        else if let Some((_, '*')) = iter.peek() {
            iter.next();

            let mut is_terminated = false;

            while let Some((_, character)) = iter.next() {
                if character == '*' {
                    if let Some((_, '/')) = iter.peek() {
                        iter.next();

                        is_terminated = true;

                        break;
                    }
                }
            }

            // Checks if the comment is terminated
            if is_terminated {
                Ok(Comment {
                    span: Self::create_span(start, iter),
                    kind: CommentKind::Delimited,
                }
                .into())
            } else {
                handler.receive(error::Error::from(UnterminatedDelimitedComment {
                    span: Span::new(iter.source_file().clone(), start, start + 2).unwrap(),
                }));
                return Err(TokenizeError::FatalLexicalError);
            }
        }
        // When there is no second slash and at the start of a line
        else if prev_token.map_or(true, |token| token.span().str().contains('\n')) {
            Ok(Self::handle_command_literal(iter, start))
        }
        // Just a single slash punctuation
        else {
            Ok(Punctuation {
                span: Self::create_span(start, iter),
                punctuation: character,
            }
            .into())
        }
    }

    /// Handles a sequence of digits
    fn handle_numeric_literal(iter: &mut SourceIterator, start: usize) -> Self {
        // Tokenizes the whole number part
        Self::walk_iter(iter, |character| character.is_ascii_digit());

        Numeric {
            span: Self::create_span(start, iter),
        }
        .into()
    }

    /// Handles a sequence of characters that are enclosed in double quotes
    fn handle_string_literal(iter: &mut SourceIterator, start: usize) -> Self {
        let mut is_escaped = false;

        for (_, character) in iter.by_ref() {
            if character == '\\' {
                is_escaped = !is_escaped;
            } else if character == '"' && !is_escaped {
                break;
            } else {
                is_escaped = false;
            }
        }

        StringLiteral {
            span: Self::create_span(start, iter),
        }
        .into()
    }

    /// Handles a sequence of characters that are enclosed in backticks and contain macro usages
    fn handle_macro_string_literal(
        iter: &mut SourceIterator,
        mut start: usize,
    ) -> Result<Self, TokenizeError> {
        let mut is_escaped = false;
        let mut is_inside_macro = false;
        let mut encountered_open_parenthesis = false;
        let starting_backtick = Punctuation {
            span: Self::create_span(start, iter),
            punctuation: '`',
        };
        start += 1;
        let mut parts = Vec::new();

        while iter.peek().is_some() {
            let (index, character) = iter.next().unwrap();

            #[expect(clippy::collapsible_else_if)]
            if is_inside_macro {
                if character == ')' {
                    // Check if the macro usage is empty
                    if start + 2 == index {
                        return Err(EmptyMacroUsage {
                            span: Span::new(iter.source_file().clone(), start, index + 1).unwrap(),
                        }
                        .into());
                    }
                    parts.push(MacroStringLiteralPart::MacroUsage {
                        dollar: Punctuation {
                            span: Span::new(iter.source_file().clone(), start, start + 1).unwrap(),
                            punctuation: '$',
                        },
                        open_brace: Punctuation {
                            span: Span::new(iter.source_file().clone(), start + 1, start + 2)
                                .unwrap(),
                            punctuation: '(',
                        },
                        identifier: Identifier {
                            span: Self::create_span_with_end_offset(start + 2, iter, -1),
                        },
                        close_brace: Punctuation {
                            span: Span::new(iter.source_file().clone(), index, index + 1).unwrap(),
                            punctuation: ')',
                        },
                    });
                    start = index + 1;
                    is_inside_macro = false;
                } else if !encountered_open_parenthesis && character == '(' {
                    encountered_open_parenthesis = true;
                } else if encountered_open_parenthesis && !Self::is_identifier_character(character)
                {
                    if character == '`' {
                        return Err(UnclosedMacroUsage {
                            span: Span::new(iter.source_file().clone(), start, start + 2).unwrap(),
                        }
                        .into());
                    }

                    Self::walk_iter(iter, |c| c != ')' && !Self::is_identifier_character(c));
                    return Err(InvalidMacroNameCharacter {
                        span: Self::create_span(index, iter),
                    }
                    .into());
                }
            } else {
                if character == '$' && iter.peek().is_some_and(|(_, c)| c == '(') {
                    parts.push(MacroStringLiteralPart::Text(
                        Self::create_span_with_end_offset(start, iter, -1),
                    ));
                    start = index;
                    is_inside_macro = true;
                    encountered_open_parenthesis = false;
                } else if character == '\\' {
                    is_escaped = !is_escaped;
                } else if character == '`' && !is_escaped {
                    if start != index {
                        parts.push(MacroStringLiteralPart::Text(
                            Self::create_span_with_end_offset(start, iter, -1),
                        ));
                    }
                    start = index;
                    break;
                } else {
                    is_escaped = false;
                }
            }
        }

        if is_inside_macro {
            Err(UnclosedMacroUsage {
                span: Span::new(iter.source_file().clone(), start, start + 2).unwrap(),
            }
            .into())
        } else {
            Ok(MacroStringLiteral {
                starting_backtick,
                parts,
                ending_backtick: Punctuation {
                    span: Self::create_span(start, iter),
                    punctuation: '`',
                },
            }
            .into())
        }
    }

    /// Handles a command that is preceeded by a slash
    fn handle_command_literal(iter: &mut SourceIterator, start: usize) -> Self {
        Self::walk_iter(iter, |c| !(c.is_whitespace() && c.is_ascii_control()));

        CommandLiteral {
            span: Self::create_span(start, iter),
        }
        .into()
    }

    /// Lexes the source code from the given iterator.
    ///
    /// The tokenization starts at the current location of the iterator. The function moves the
    /// iterator at least once and forwards it until it makes a token. After the token is made, the
    /// iterator is left at the next character that is not part of the token.
    ///
    /// # Errors
    /// - [`TokenizeError::EndOfSourceCodeIteratorArgument`] - The iterator argument is at the end of the
    ///   source code.
    /// - [`TokenizeError::FatalLexicalError`] - A fatal lexical error occurred.
    pub fn tokenize(
        iter: &mut SourceIterator,
        handler: &impl Handler<base::Error>,
        prev_token: Option<&Self>,
    ) -> Result<Self, TokenizeError> {
        // Gets the first character
        let (start, character) = iter
            .next()
            .ok_or(TokenizeError::EndOfSourceCodeIteratorArgument)?;

        // Found white spaces
        if character.is_whitespace() {
            Ok(Self::handle_whitespace(iter, start))
        }
        // Found identifier/keyword
        else if Self::is_first_identifier_character(character) {
            Ok(Self::handle_identifier_and_keyword(iter, start))
        }
        // Found comment/single slash punctuation
        else if character == '/' {
            Self::handle_comment(iter, start, character, prev_token, handler)
        }
        // Found string literal
        else if character == '"' {
            Ok(Self::handle_string_literal(iter, start))
        }
        // Found macro string literal
        else if character == '`' {
            Self::handle_macro_string_literal(iter, start)
        }
        // Found numeric literal
        else if character.is_ascii_digit() {
            Ok(Self::handle_numeric_literal(iter, start))
        }
        // Found a punctuation
        else if character.is_ascii_punctuation() {
            Ok(Punctuation {
                span: Self::create_span(start, iter),
                punctuation: character,
            }
            .into())
        } else {
            unreachable!("all cases covered before")
        }
    }
}

#[cfg(test)]
mod tests {
    use crate::base::source_file::SourceFile;
    use shulkerbox::virtual_fs::{VFile, VFolder};
    use std::path::Path;

    use super::*;

    fn get_span(content: &str) -> Span {
        let source_file = VFile::Text(String::from(content));
        let mut vfolder = VFolder::new();
        vfolder.add_file("test.shu", source_file);
        let source_file =
            SourceFile::load(Path::new("test.shu"), "test".to_string(), &vfolder).unwrap();

        Span::new(source_file, 0, content.len()).unwrap()
    }

    #[test]
    fn test_string_literal() {
        let span = get_span(r#""Hello, World!""#);
        let string_literal = StringLiteral { span };

        let content = string_literal.str_content();
        assert_eq!(content, "Hello, World!");
        assert!(matches!(content, Cow::Borrowed(_)));

        let escaped_string_literal = StringLiteral {
            span: get_span(r#""Hello, \"World\"!""#),
        };
        let escaped_content = escaped_string_literal.str_content();
        assert_eq!(escaped_content, "Hello, \"World\"!");
        assert!(matches!(escaped_content, Cow::Owned(_)));
    }
}