allow escaping of $ in macro string

src/lexical/token.rs

@@ -10,7 +10,6 @@ use std::{
 
 use crate::base::{
     self,
-    log::SourceCodeDisplay,
     source_file::{SourceElement, SourceIterator, Span},
     Handler,
 };
@@ -445,101 +444,15 @@ impl CommandLiteral {
 /// Is an error that can occur when invoking the [`Token::tokenize`] method.
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, thiserror::Error)]
 #[allow(missing_docs)]
+#[expect(missing_copy_implementations)]
 pub enum TokenizeError {
     #[error("encountered a fatal lexical error that causes the process to stop.")]
     FatalLexicalError,
 
     #[error("the iterator argument is at the end of the source code.")]
     EndOfSourceCodeIteratorArgument,
-
-    #[error(transparent)]
-    InvalidMacroNameCharacter(#[from] InvalidMacroNameCharacter),
-
-    #[error(transparent)]
-    UnclosedExpressionInTemplateUsage(#[from] UnclosedExpressionInTemplateUsage),
-
-    #[error(transparent)]
-    EmptyExpressionInTemplateUsage(#[from] EmptyExpressionInTemplateUsage),
 }
 
-/// Is an error that can occur when the macro name contains invalid characters.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct InvalidMacroNameCharacter {
-    /// The span of the invalid characters.
-    pub span: Span,
-}
-
-impl Display for InvalidMacroNameCharacter {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "{}",
-            base::log::Message::new(base::log::Severity::Error, format!("The macro name contains invalid characters: `{}`. Only alphanumeric characters and underscores are allowed.", self.span.str()))
-        )?;
-        write!(
-            f,
-            "\n{}",
-            SourceCodeDisplay::new(&self.span, Option::<u8>::None)
-        )
-    }
-}
-
-impl std::error::Error for InvalidMacroNameCharacter {}
-
-/// Is an error that can occur when the expression is not closed.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct UnclosedExpressionInTemplateUsage {
-    /// The span of the unclosed expression.
-    pub span: Span,
-}
-
-impl Display for UnclosedExpressionInTemplateUsage {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "{}",
-            base::log::Message::new(
-                base::log::Severity::Error,
-                "An expression was opened with `$(` but never closed."
-            )
-        )?;
-        write!(
-            f,
-            "\n{}",
-            SourceCodeDisplay::new(&self.span, Option::<u8>::None)
-        )
-    }
-}
-
-impl std::error::Error for UnclosedExpressionInTemplateUsage {}
-
-/// Is an error that can occur when the expression is not closed.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct EmptyExpressionInTemplateUsage {
-    /// The span of the unclosed expression.
-    pub span: Span,
-}
-
-impl Display for EmptyExpressionInTemplateUsage {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "{}",
-            base::log::Message::new(
-                base::log::Severity::Error,
-                "An expression was opened with `$(` but closed immediately with `)`."
-            )
-        )?;
-        write!(
-            f,
-            "\n{}",
-            SourceCodeDisplay::new(&self.span, Option::<u8>::None)
-        )
-    }
-}
-
-impl std::error::Error for EmptyExpressionInTemplateUsage {}
-
 impl Token {
     /// Increments the iterator while the predicate returns true.
     pub fn walk_iter(iter: &mut SourceIterator, predicate: impl Fn(char) -> bool) {
@@ -743,7 +656,9 @@ impl Token {
         character: char,
         prev_token: Option<&Self>,
     ) -> Self {
-        if character == '$' && iter.peek().is_some_and(|(_, c)| c == '(') {
+        let prev_was_backslash = iter.prev().is_some_and(|(_, c)| c == '\\');
+
+        if !prev_was_backslash && character == '$' && iter.peek().is_some_and(|(_, c)| c == '(') {
             // starts immediately with expression, return punctuation
             return Punctuation {
                 span: Self::create_span(start, iter),
@@ -753,7 +668,9 @@ impl Token {
         }
 
         match (character, prev_token) {
-            ('(', Some(Self::Punctuation(punc))) if punc.punctuation == '$' => {
+            ('(', Some(Self::Punctuation(punc)))
+                if !prev_was_backslash && punc.punctuation == '$' =>
+            {
                 // Found expression opening parenthesis
                 iter.increase_template_string_expression_open_paren_count();
 
@@ -769,17 +686,19 @@ impl Token {
         loop {
             if character != '`' {
                 iter.reset_multipeek();
-                Self::walk_iter(iter, |c| c != '$' && c != '`');
+                Self::walk_iter(iter, |c| !matches!(c, '$' | '`' | '\\'));
             }
 
             iter.reset_multipeek();
-            let first_peek_none_or_backtick = iter.multipeek().map(|(_, c)| c);
+            let first_peek = iter.multipeek().map(|(_, c)| c);
             let second_peek_open_paren = iter.multipeek().is_some_and(|(_, c)| c == '(');
 
-            if character == '`'
+            if first_peek.is_some_and(|c| c == '\\') {
-                || first_peek_none_or_backtick.is_none_or(|c| c == '`')
+                iter.next();
-                || second_peek_open_paren
+                iter.next();
-            {
+            }
+
+            if character == '`' || first_peek.is_none_or(|c| c == '`') || second_peek_open_paren {
                 // Found expression start, end of text
 
                 break TemplateStringLiteralText {

src/lexical/token_stream.rs

@@ -5,13 +5,10 @@ use std::{fmt::Debug, sync::Arc};
 use derive_more::{Deref, From};
 use enum_as_inner::EnumAsInner;
 
-use crate::{
+use crate::base::{
-    base::{
     self,
     source_file::{SourceElement, SourceFile, Span},
     Handler,
-    },
-    lexical::Error,
 };
 
 use super::{
@@ -65,21 +62,6 @@ impl TokenStream {
                 Err(TokenizeError::FatalLexicalError) => {
                     tracing::error!("Fatal lexical error encountered while tokenizing source code");
                 }
-                Err(TokenizeError::InvalidMacroNameCharacter(err)) => {
-                    handler.receive(Error::TokenizeError(
-                        TokenizeError::InvalidMacroNameCharacter(err),
-                    ));
-                }
-                Err(TokenizeError::UnclosedExpressionInTemplateUsage(err)) => {
-                    handler.receive(Error::TokenizeError(
-                        TokenizeError::UnclosedExpressionInTemplateUsage(err),
-                    ));
-                }
-                Err(TokenizeError::EmptyExpressionInTemplateUsage(err)) => {
-                    handler.receive(Error::TokenizeError(
-                        TokenizeError::EmptyExpressionInTemplateUsage(err),
-                    ));
-                }
             }
         }
 

src/semantic/mod.rs

@@ -970,7 +970,7 @@ impl TemplateStringLiteral {
                     match expression.as_ref() {
                         Expression::Primary(Primary::Identifier(identifier)) => {
                             if let Some(variable_type) = scope.get_variable(identifier.span.str()) {
-                                // TODO: correct checks
+                                // TODO: template string correct checks
                                 // if variable_type != VariableType::MacroParameter {
                                 //     let err = error::Error::UnexpectedExpression(UnexpectedExpression(
                                 //         Box::new(Expression::Primary(Primary::Identifier(
@@ -988,8 +988,50 @@ impl TemplateStringLiteral {
                                 errs.push(err);
                             }
                         }
+                        Expression::Primary(Primary::Indexed(indexed)) => {
+                            if let Primary::Identifier(identifier) = indexed.object().as_ref() {
+                                if let Some(variable_type) =
+                                    scope.get_variable(identifier.span.str())
+                                {
+                                    match variable_type {
+                                        VariableType::BooleanStorageArray
+                                        | VariableType::ScoreboardArray
+                                        | VariableType::Tag
+                                        | VariableType::Scoreboard => {
+                                            // Valid types
+                                        }
                                         _ => {
-                            // TODO: handle other expressions in template string literals
+                                            let err = error::Error::UnexpectedExpression(
+                                                UnexpectedExpression(expression.clone()),
+                                            );
+                                            handler.receive(err.clone());
+                                            errs.push(err);
+                                        }
+                                    }
+                                } else {
+                                    let err = error::Error::UnknownIdentifier(UnknownIdentifier {
+                                        identifier: identifier.span.clone(),
+                                    });
+                                    handler.receive(err.clone());
+                                    errs.push(err);
+                                }
+                            } else {
+                                let err = error::Error::UnexpectedExpression(UnexpectedExpression(
+                                    expression.clone(),
+                                ));
+                                handler.receive(err.clone());
+                                errs.push(err);
+                            }
+                            if let Err(err) = indexed.index().analyze_semantics(scope, handler) {
+                                errs.push(err);
+                            }
+                        }
+                        _ => {
+                            let err = error::Error::UnexpectedExpression(UnexpectedExpression(
+                                expression.clone(),
+                            ));
+                            handler.receive(err.clone());
+                            errs.push(err);
                         }
                     }
                 }

src/syntax/syntax_tree/expression.rs

@@ -410,31 +410,6 @@ pub struct TemplateStringLiteral {
 }
 
 impl TemplateStringLiteral {
-    /// Returns the string content without escapement characters, leading and trailing double quotes.
-    #[must_use]
-    pub fn str_content(&self) -> String {
-        let mut content = String::new();
-
-        for part in &self.parts {
-            match part {
-                TemplateStringLiteralPart::Text(text) => {
-                    content += &crate::util::unescape_macro_string(text.span.str());
-                }
-                TemplateStringLiteralPart::Expression { expression, .. } => {
-                    // write!(
-                    //     content,
-                    //     "$({})",
-                    //     crate::util::identifier_to_macro(identifier.span.str())
-                    // )
-                    // .expect("can always write to string");
-                    todo!("handle expression in template string literal")
-                }
-            }
-        }
-
-        content
-    }
-
     /// Returns the parts that make up the template string literal.
     #[must_use]
     pub fn parts(&self) -> &[TemplateStringLiteralPart] {

src/transpile/conversions.rs

@@ -1,7 +1,17 @@
 //! Conversion functions for converting between tokens/ast-nodes and [`shulkerbox`] types
 
+use std::{borrow::Cow, sync::Arc};
+
 use shulkerbox::util::{MacroString as ExtMacroString, MacroStringPart as ExtMacroStringPart};
 
+use crate::{
+    base::{self, Handler},
+    semantic::error::UnexpectedExpression,
+    syntax::syntax_tree::expression::{TemplateStringLiteral, TemplateStringLiteralPart},
+    transpile::{Scope, TranspileError, TranspileResult},
+    util,
+};
+
 use super::util::{MacroString, MacroStringPart};
 
 impl From<MacroString> for ExtMacroString {
@@ -23,3 +33,39 @@ impl From<MacroStringPart> for ExtMacroStringPart {
         }
     }
 }
+
+impl TemplateStringLiteral {
+    pub fn as_str(
+        &self,
+        scope: &Arc<Scope>,
+        handler: &impl Handler<base::Error>,
+    ) -> TranspileResult<Cow<'_, str>> {
+        let mut res = Cow::Borrowed("");
+
+        for part in &self.parts {
+            match part {
+                TemplateStringLiteralPart::Text(s) => {
+                    let s = util::unescape_template_string(s.span.str());
+                    if res.is_empty() {
+                        res = s;
+                    } else {
+                        res.to_mut().push_str(&s);
+                    }
+                }
+                TemplateStringLiteralPart::Expression { expression, .. } => {
+                    let compiled = expression.comptime_eval(scope, handler)?;
+                    let s = compiled.to_string_no_macro().ok_or_else(|| {
+                        let err = TranspileError::UnexpectedExpression(UnexpectedExpression(
+                            expression.clone(),
+                        ));
+                        handler.receive(Box::new(err.clone()));
+                        err
+                    })?;
+                    res.to_mut().push_str(&s);
+                }
+            }
+        }
+
+        Ok(res)
+    }
+}

src/transpile/expression.rs

@@ -427,7 +427,14 @@ impl Primary {
                             expression: template_string_literal.span(),
                         })
                 } else {
-                    Ok(ComptimeValue::String(template_string_literal.str_content()))
+                    Ok(ComptimeValue::String(
+                        template_string_literal
+                            .as_str(scope, handler)
+                            .map_err(|_| NotComptime {
+                                expression: template_string_literal.span(),
+                            })?
+                            .into_owned(),
+                    ))
                 }
             }
         }

src/transpile/internal_functions.rs

@@ -18,7 +18,7 @@ use crate::{
         Expression, FunctionCall, Primary, TemplateStringLiteralPart,
     },
     transpile::{
-        error::{IllegalIndexing, IllegalIndexingReason, UnknownIdentifier},
+        error::{IllegalIndexing, IllegalIndexingReason, NotComptime, UnknownIdentifier},
         expression::{ComptimeValue, DataLocation, ExpectedType, StorageType},
         util::MacroString,
         TranspileError,
@@ -97,7 +97,6 @@ fn print_function(
 ) -> TranspileResult<Vec<Command>> {
     const PARAM_COLOR: &str = "gray";
 
-    #[expect(clippy::option_if_let_else)]
     fn get_identifier_part(
         ident: &Identifier,
         transpiler: &mut Transpiler,
@@ -133,6 +132,158 @@ fn print_function(
 
                     Ok((false, cmd, value))
                 }
+                VariableData::ComptimeValue { value, .. } => {
+                    let value = {
+                        let guard = value.read().map_err(|_| {
+                            TranspileError::NotComptime(NotComptime {
+                                expression: ident.span(),
+                            })
+                        })?;
+                        guard.as_ref().map_or_else(
+                            || "null".into(),
+                            super::expression::ComptimeValue::to_macro_string,
+                        )
+                    };
+
+                    Ok((
+                        value.contains_macros(),
+                        None,
+                        json!({"text": value.to_string(), "color": PARAM_COLOR}),
+                    ))
+                }
+                _ => Err(TranspileError::UnexpectedExpression(UnexpectedExpression(
+                    Box::new(Expression::Primary(Primary::Identifier(ident.to_owned()))),
+                ))),
+            }
+        } else {
+            Err(TranspileError::UnknownIdentifier(UnknownIdentifier {
+                identifier: ident.span(),
+            }))
+        }
+    }
+
+    fn get_indexed_part(
+        ident: &Identifier,
+        index: &Expression,
+        transpiler: &mut Transpiler,
+        scope: &Arc<Scope>,
+    ) -> TranspileResult<(bool, Option<Command>, JsonValue)> {
+        if let Some(var) = scope.get_variable(ident.span.str()).as_deref() {
+            match var {
+                VariableData::Scoreboard { objective } => {
+                    let Ok(ComptimeValue::String(target)) =
+                        index.comptime_eval(scope, &VoidHandler)
+                    else {
+                        return Err(TranspileError::IllegalIndexing(IllegalIndexing {
+                            reason: IllegalIndexingReason::InvalidComptimeType {
+                                expected: ExpectedType::String,
+                            },
+                            expression: index.span(),
+                        }));
+                    };
+
+                    let (cmd, value) = get_data_location(
+                        &DataLocation::ScoreboardValue {
+                            objective: objective.to_string(),
+                            target,
+                        },
+                        transpiler,
+                    );
+
+                    Ok((false, cmd, value))
+                }
+                VariableData::ScoreboardArray { objective, targets } => {
+                    let Ok(ComptimeValue::Integer(idx)) = index.comptime_eval(scope, &VoidHandler)
+                    else {
+                        return Err(TranspileError::IllegalIndexing(IllegalIndexing {
+                            reason: IllegalIndexingReason::InvalidComptimeType {
+                                expected: ExpectedType::Integer,
+                            },
+                            expression: index.span(),
+                        }));
+                    };
+
+                    #[expect(clippy::option_if_let_else)]
+                    if let Some(target) = usize::try_from(idx)
+                        .ok()
+                        .and_then(|index| targets.get(index))
+                    {
+                        let (cmd, value) = get_data_location(
+                            &DataLocation::ScoreboardValue {
+                                objective: objective.to_string(),
+                                target: target.to_string(),
+                            },
+                            transpiler,
+                        );
+                        Ok((false, cmd, value))
+                    } else {
+                        Err(TranspileError::IllegalIndexing(IllegalIndexing {
+                            reason: IllegalIndexingReason::IndexOutOfBounds {
+                                index: usize::try_from(idx).unwrap_or(usize::MAX),
+                                length: targets.len(),
+                            },
+                            expression: index.span(),
+                        }))
+                    }
+                }
+                VariableData::BooleanStorageArray {
+                    storage_name,
+                    paths,
+                } => {
+                    let Ok(ComptimeValue::Integer(idx)) = index.comptime_eval(scope, &VoidHandler)
+                    else {
+                        return Err(TranspileError::IllegalIndexing(IllegalIndexing {
+                            reason: IllegalIndexingReason::InvalidComptimeType {
+                                expected: ExpectedType::Integer,
+                            },
+                            expression: index.span(),
+                        }));
+                    };
+
+                    #[expect(clippy::option_if_let_else)]
+                    if let Some(path) = usize::try_from(idx).ok().and_then(|index| paths.get(index))
+                    {
+                        let (cmd, value) = get_data_location(
+                            &DataLocation::Storage {
+                                storage_name: storage_name.to_string(),
+                                path: path.to_string(),
+                                r#type: StorageType::Boolean,
+                            },
+                            transpiler,
+                        );
+                        Ok((false, cmd, value))
+                    } else {
+                        Err(TranspileError::IllegalIndexing(IllegalIndexing {
+                            reason: IllegalIndexingReason::IndexOutOfBounds {
+                                index: usize::try_from(idx).unwrap_or(usize::MAX),
+                                length: paths.len(),
+                            },
+                            expression: index.span(),
+                        }))
+                    }
+                }
+                VariableData::Tag { tag_name } => {
+                    let Ok(ComptimeValue::String(entity)) =
+                        index.comptime_eval(scope, &VoidHandler)
+                    else {
+                        return Err(TranspileError::IllegalIndexing(IllegalIndexing {
+                            reason: IllegalIndexingReason::InvalidComptimeType {
+                                expected: ExpectedType::String,
+                            },
+                            expression: index.span(),
+                        }));
+                    };
+
+                    let (cmd, value) = get_data_location(
+                        &DataLocation::Tag {
+                            tag_name: tag_name.clone(),
+                            entity,
+                        },
+                        transpiler,
+                    );
+
+                    Ok((false, cmd, value))
+                }
                 _ => Err(TranspileError::UnexpectedExpression(UnexpectedExpression(
                     Box::new(Expression::Primary(Primary::Identifier(ident.to_owned()))),
                 ))),
@@ -194,7 +345,7 @@ fn print_function(
         ("@a".into(), first)
     };
 
-    let mut contains_macro = matches!(target, MacroString::MacroString(_));
+    let mut contains_macro = target.contains_macros();
 
     let (mut cmds, parts) = match message_expression {
         Expression::Primary(primary) => match primary {
@@ -362,6 +513,24 @@ fn print_function(
                                     cmds.extend(cur_cmds);
                                     parts.push(part);
                                 }
+                                Expression::Primary(Primary::Indexed(indexed)) => {
+                                    match indexed.object().as_ref() {
+                                        Primary::Identifier(ident) => {
+                                            let (cur_contains_macro, cur_cmds, part) =
+                                                get_indexed_part(
+                                                    ident,
+                                                    indexed.index(),
+                                                    transpiler,
+                                                    scope,
+                                                )?;
+
+                                            contains_macro |= cur_contains_macro;
+                                            cmds.extend(cur_cmds);
+                                            parts.push(part);
+                                        }
+                                        _ => todo!("other expression in indexed"),
+                                    }
+                                }
                                 _ => todo!("other expression in template string literal"),
                             }
                         }

src/transpile/util.rs

@@ -14,6 +14,8 @@ use crate::{
     },
 };
 
+use super::expression::ComptimeValue;
+
 /// String that can contain macros
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -291,7 +293,7 @@ impl TemplateStringLiteral {
                     .iter()
                     .map(|part| match part {
                         TemplateStringLiteralPart::Text(text) => Ok(MacroStringPart::String(
-                            crate::util::unescape_macro_string(text.span.str()).to_string(),
+                            crate::util::unescape_template_string(text.span.str()).into_owned(),
                         )),
                         TemplateStringLiteralPart::Expression { expression, .. } => {
                             match expression.as_ref() {
@@ -305,6 +307,17 @@ impl TemplateStringLiteral {
                                             VariableData::MacroParameter { macro_name, .. } => Ok(
                                                 MacroStringPart::MacroUsage(macro_name.to_owned()),
                                             ),
+                                            VariableData::ComptimeValue { value, .. } => {
+                                                let value = value.read().unwrap().as_ref().map_or_else(
+                                                    || "null".into(),
+                                                    ComptimeValue::to_macro_string,
+                                                );
+                                                
+                                                match value.as_str() {
+                                                    Ok(s) => Ok(MacroStringPart::String(s.into_owned())),
+                                                    Err(_) => todo!("comptime value resulting in macro string with macros")
+                                                }
+                                            }
                                             _ => todo!("other identifiers in template strings"),
                                         }
                                     } else {
@@ -325,7 +338,7 @@ impl TemplateStringLiteral {
 
             Ok(macro_string)
         } else {
-            Ok(MacroString::String(self.str_content()))
+            Ok(MacroString::String(self.as_str(scope, handler)?.into_owned()))
         }
     }
 }

src/util.rs

@@ -20,15 +20,16 @@ pub fn escape_str(s: &str) -> Cow<'_, str> {
     }
 }
 
-/// Unescapes '\`', `\`, `\n`, `\r` and `\t` in a string.
+/// Unescapes '\`', `\`, `\n`, `\r` and `\t`, `\$` in a string.
 #[must_use]
-pub fn unescape_macro_string(s: &str) -> Cow<'_, str> {
+pub fn unescape_template_string(s: &str) -> Cow<'_, str> {
     if s.contains('\\') || s.contains('`') {
         Cow::Owned(
             s.replace("\\n", "\n")
                 .replace("\\r", "\r")
                 .replace("\\t", "\t")
                 .replace("\\`", "`")
+                .replace("\\$", "$")
                 .replace("\\\\", "\\"),
         )
     } else {
@@ -147,33 +148,33 @@ mod tests {
 
     #[test]
     fn test_unescape_macro_string() {
-        assert_eq!(unescape_macro_string("Hello, world!"), "Hello, world!");
+        assert_eq!(unescape_template_string("Hello, world!"), "Hello, world!");
         assert_eq!(
-            unescape_macro_string(r#"Hello, "world"!"#),
+            unescape_template_string(r#"Hello, "world"!"#),
             r#"Hello, "world"!"#
         );
         assert_eq!(
-            unescape_macro_string(r"Hello, \world\!"),
+            unescape_template_string(r"Hello, \world\!"),
             r"Hello, \world\!"
         );
         assert_eq!(
-            unescape_macro_string(r"Hello, \nworld\!"),
+            unescape_template_string(r"Hello, \nworld\!"),
             "Hello, \nworld\\!"
         );
         assert_eq!(
-            unescape_macro_string(r"Hello, \rworld\!"),
+            unescape_template_string(r"Hello, \rworld\!"),
             "Hello, \rworld\\!"
         );
         assert_eq!(
-            unescape_macro_string(r"Hello, \tworld\!"),
+            unescape_template_string(r"Hello, \tworld\!"),
             "Hello, \tworld\\!"
         );
         assert_eq!(
-            unescape_macro_string(r"Hello, \`world\!"),
+            unescape_template_string(r"Hello, \`world\!"),
             r"Hello, `world\!"
         );
         assert_eq!(
-            unescape_macro_string(r"Hello, \\world\!"),
+            unescape_template_string(r"Hello, \\world\!"),
             r"Hello, \world\!"
         );
     }