moritz
/
shulkerscript-lang
mirror of https://github.com/moritz-hoelting/shulkerscript-lang.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

allow escaping of $ in macro string

This commit is contained in:
Moritz Hölting 2025-08-26 21:15:41 +02:00
parent 183d3e85c6
commit b7d50f8222
9 changed files with 315 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
src/lexical/token.rs
View File

@ -10,7 +10,6 @@ use std::{
use crate::base::{
self,
log::SourceCodeDisplay,
source_file::{SourceElement, SourceIterator, Span},
Handler,
};
@ -445,101 +444,15 @@ impl CommandLiteral {
/// Is an error that can occur when invoking the [`Token::tokenize`] method.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, thiserror::Error)]
#[allow(missing_docs)]
#[expect(missing_copy_implementations)]
pub enum TokenizeError {
#[error("encountered a fatal lexical error that causes the process to stop.")]
FatalLexicalError,
#[error("the iterator argument is at the end of the source code.")]
EndOfSourceCodeIteratorArgument,
#[error(transparent)]
InvalidMacroNameCharacter(#[from] InvalidMacroNameCharacter),
#[error(transparent)]
UnclosedExpressionInTemplateUsage(#[from] UnclosedExpressionInTemplateUsage),
#[error(transparent)]
EmptyExpressionInTemplateUsage(#[from] EmptyExpressionInTemplateUsage),
}
/// Is an error that can occur when the macro name contains invalid characters.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct InvalidMacroNameCharacter {
/// The span of the invalid characters.
pub span: Span,
}
impl Display for InvalidMacroNameCharacter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}",
base::log::Message::new(base::log::Severity::Error, format!("The macro name contains invalid characters: `{}`. Only alphanumeric characters and underscores are allowed.", self.span.str()))
)?;
write!(
f,
"\n{}",
SourceCodeDisplay::new(&self.span, Option::<u8>::None)
)
}
}
impl std::error::Error for InvalidMacroNameCharacter {}
/// Is an error that can occur when the expression is not closed.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct UnclosedExpressionInTemplateUsage {
/// The span of the unclosed expression.
pub span: Span,
}
impl Display for UnclosedExpressionInTemplateUsage {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}",
base::log::Message::new(
base::log::Severity::Error,
"An expression was opened with `$(` but never closed."
)
)?;
write!(
f,
"\n{}",
SourceCodeDisplay::new(&self.span, Option::<u8>::None)
)
}
}
impl std::error::Error for UnclosedExpressionInTemplateUsage {}
/// Is an error that can occur when the expression is not closed.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct EmptyExpressionInTemplateUsage {
/// The span of the unclosed expression.
pub span: Span,
}
impl Display for EmptyExpressionInTemplateUsage {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}",
base::log::Message::new(
base::log::Severity::Error,
"An expression was opened with `$(` but closed immediately with `)`."
)
)?;
write!(
f,
"\n{}",
SourceCodeDisplay::new(&self.span, Option::<u8>::None)
)
}
}
impl std::error::Error for EmptyExpressionInTemplateUsage {}
impl Token {
/// Increments the iterator while the predicate returns true.
pub fn walk_iter(iter: &mut SourceIterator, predicate: impl Fn(char) -> bool) {
@ -743,7 +656,9 @@ impl Token {
character: char,
prev_token: Option<&Self>,
) -> Self {
if character == '$' && iter.peek().is_some_and(|(_, c)| c == '(') {
let prev_was_backslash = iter.prev().is_some_and(|(_, c)| c == '\\');
if !prev_was_backslash && character == '$' && iter.peek().is_some_and(|(_, c)| c == '(') {
// starts immediately with expression, return punctuation
return Punctuation {
span: Self::create_span(start, iter),
@ -753,7 +668,9 @@ impl Token {
}
match (character, prev_token) {
('(', Some(Self::Punctuation(punc))) if punc.punctuation == '$' => {
('(', Some(Self::Punctuation(punc)))
if !prev_was_backslash && punc.punctuation == '$' =>
{
// Found expression opening parenthesis
iter.increase_template_string_expression_open_paren_count();
@ -769,17 +686,19 @@ impl Token {
loop {
if character != '`' {
iter.reset_multipeek();
Self::walk_iter(iter, |c| c != '$' && c != '`');
Self::walk_iter(iter, |c| !matches!(c, '$' | '`' | '\\'));
}
iter.reset_multipeek();
let first_peek_none_or_backtick = iter.multipeek().map(|(_, c)| c);
let first_peek = iter.multipeek().map(|(_, c)| c);
let second_peek_open_paren = iter.multipeek().is_some_and(|(_, c)| c == '(');
if character == '`'
|| first_peek_none_or_backtick.is_none_or(|c| c == '`')
|| second_peek_open_paren
{
if first_peek.is_some_and(|c| c == '\\') {
iter.next();
iter.next();
}
if character == '`' || first_peek.is_none_or(|c| c == '`') || second_peek_open_paren {
// Found expression start, end of text
break TemplateStringLiteralText {

20
src/lexical/token_stream.rs
View File

@ -5,13 +5,10 @@ use std::{fmt::Debug, sync::Arc};
use derive_more::{Deref, From};
use enum_as_inner::EnumAsInner;
use crate::{
base::{
use crate::base::{
self,
source_file::{SourceElement, SourceFile, Span},
Handler,
},
lexical::Error,
};
use super::{
@ -65,21 +62,6 @@ impl TokenStream {
Err(TokenizeError::FatalLexicalError) => {
tracing::error!("Fatal lexical error encountered while tokenizing source code");
}
Err(TokenizeError::InvalidMacroNameCharacter(err)) => {
handler.receive(Error::TokenizeError(
TokenizeError::InvalidMacroNameCharacter(err),
));
}
Err(TokenizeError::UnclosedExpressionInTemplateUsage(err)) => {
handler.receive(Error::TokenizeError(
TokenizeError::UnclosedExpressionInTemplateUsage(err),
));
}
Err(TokenizeError::EmptyExpressionInTemplateUsage(err)) => {
handler.receive(Error::TokenizeError(
TokenizeError::EmptyExpressionInTemplateUsage(err),
));
}
}
}

46
src/semantic/mod.rs
View File

@ -970,7 +970,7 @@ impl TemplateStringLiteral {
match expression.as_ref() {
Expression::Primary(Primary::Identifier(identifier)) => {
if let Some(variable_type) = scope.get_variable(identifier.span.str()) {
// TODO: correct checks
// TODO: template string correct checks
// if variable_type != VariableType::MacroParameter {
// let err = error::Error::UnexpectedExpression(UnexpectedExpression(
// Box::new(Expression::Primary(Primary::Identifier(
@ -988,8 +988,50 @@ impl TemplateStringLiteral {
errs.push(err);
}
}
Expression::Primary(Primary::Indexed(indexed)) => {
if let Primary::Identifier(identifier) = indexed.object().as_ref() {
if let Some(variable_type) =
scope.get_variable(identifier.span.str())
{
match variable_type {
VariableType::BooleanStorageArray
| VariableType::ScoreboardArray
| VariableType::Tag
| VariableType::Scoreboard => {
// Valid types
}
_ => {
// TODO: handle other expressions in template string literals
let err = error::Error::UnexpectedExpression(
UnexpectedExpression(expression.clone()),
);
handler.receive(err.clone());
errs.push(err);
}
}
} else {
let err = error::Error::UnknownIdentifier(UnknownIdentifier {
identifier: identifier.span.clone(),
});
handler.receive(err.clone());
errs.push(err);
}
} else {
let err = error::Error::UnexpectedExpression(UnexpectedExpression(
expression.clone(),
));
handler.receive(err.clone());
errs.push(err);
}
if let Err(err) = indexed.index().analyze_semantics(scope, handler) {
errs.push(err);
}
}
_ => {
let err = error::Error::UnexpectedExpression(UnexpectedExpression(
expression.clone(),
));
handler.receive(err.clone());
errs.push(err);
}
}
}

25
src/syntax/syntax_tree/expression.rs
View File

@ -410,31 +410,6 @@ pub struct TemplateStringLiteral {
}
impl TemplateStringLiteral {
/// Returns the string content without escapement characters, leading and trailing double quotes.
#[must_use]
pub fn str_content(&self) -> String {
let mut content = String::new();
for part in &self.parts {
match part {
TemplateStringLiteralPart::Text(text) => {
content += &crate::util::unescape_macro_string(text.span.str());
}
TemplateStringLiteralPart::Expression { expression, .. } => {
// write!(
// content,
// "$({})",
// crate::util::identifier_to_macro(identifier.span.str())
// )
// .expect("can always write to string");
todo!("handle expression in template string literal")
}
}
}
content
}
/// Returns the parts that make up the template string literal.
#[must_use]
pub fn parts(&self) -> &[TemplateStringLiteralPart] {

46
src/transpile/conversions.rs
View File

@ -1,7 +1,17 @@
//! Conversion functions for converting between tokens/ast-nodes and [`shulkerbox`] types
use std::{borrow::Cow, sync::Arc};
use shulkerbox::util::{MacroString as ExtMacroString, MacroStringPart as ExtMacroStringPart};
use crate::{
base::{self, Handler},
semantic::error::UnexpectedExpression,
syntax::syntax_tree::expression::{TemplateStringLiteral, TemplateStringLiteralPart},
transpile::{Scope, TranspileError, TranspileResult},
util,
};
use super::util::{MacroString, MacroStringPart};
impl From<MacroString> for ExtMacroString {
@ -23,3 +33,39 @@ impl From<MacroStringPart> for ExtMacroStringPart {
}
}
}
impl TemplateStringLiteral {
pub fn as_str(
&self,
scope: &Arc<Scope>,
handler: &impl Handler<base::Error>,
) -> TranspileResult<Cow<'_, str>> {
let mut res = Cow::Borrowed("");
for part in &self.parts {
match part {
TemplateStringLiteralPart::Text(s) => {
let s = util::unescape_template_string(s.span.str());
if res.is_empty() {
res = s;
} else {
res.to_mut().push_str(&s);
}
}
TemplateStringLiteralPart::Expression { expression, .. } => {
let compiled = expression.comptime_eval(scope, handler)?;
let s = compiled.to_string_no_macro().ok_or_else(|| {
let err = TranspileError::UnexpectedExpression(UnexpectedExpression(
expression.clone(),
));
handler.receive(Box::new(err.clone()));
err
})?;
res.to_mut().push_str(&s);
}
}
}
Ok(res)
}
}

9
src/transpile/expression.rs
View File

@ -427,7 +427,14 @@ impl Primary {
expression: template_string_literal.span(),
})
} else {
Ok(ComptimeValue::String(template_string_literal.str_content()))
Ok(ComptimeValue::String(
template_string_literal
.as_str(scope, handler)
.map_err(|_| NotComptime {
expression: template_string_literal.span(),
})?
.into_owned(),
))
}
}
}

175
src/transpile/internal_functions.rs
View File

@ -18,7 +18,7 @@ use crate::{
Expression, FunctionCall, Primary, TemplateStringLiteralPart,
},
transpile::{
error::{IllegalIndexing, IllegalIndexingReason, UnknownIdentifier},
error::{IllegalIndexing, IllegalIndexingReason, NotComptime, UnknownIdentifier},
expression::{ComptimeValue, DataLocation, ExpectedType, StorageType},
util::MacroString,
TranspileError,
@ -97,7 +97,6 @@ fn print_function(
) -> TranspileResult<Vec<Command>> {
const PARAM_COLOR: &str = "gray";
#[expect(clippy::option_if_let_else)]
fn get_identifier_part(
ident: &Identifier,
transpiler: &mut Transpiler,
@ -133,6 +132,158 @@ fn print_function(
Ok((false, cmd, value))
}
VariableData::ComptimeValue { value, .. } => {
let value = {
let guard = value.read().map_err(|_| {
TranspileError::NotComptime(NotComptime {
expression: ident.span(),
})
})?;
guard.as_ref().map_or_else(
|| "null".into(),
super::expression::ComptimeValue::to_macro_string,
)
};
Ok((
value.contains_macros(),
None,
json!({"text": value.to_string(), "color": PARAM_COLOR}),
))
}
_ => Err(TranspileError::UnexpectedExpression(UnexpectedExpression(
Box::new(Expression::Primary(Primary::Identifier(ident.to_owned()))),
))),
}
} else {
Err(TranspileError::UnknownIdentifier(UnknownIdentifier {
identifier: ident.span(),
}))
}
}
fn get_indexed_part(
ident: &Identifier,
index: &Expression,
transpiler: &mut Transpiler,
scope: &Arc<Scope>,
) -> TranspileResult<(bool, Option<Command>, JsonValue)> {
if let Some(var) = scope.get_variable(ident.span.str()).as_deref() {
match var {
VariableData::Scoreboard { objective } => {
let Ok(ComptimeValue::String(target)) =
index.comptime_eval(scope, &VoidHandler)
else {
return Err(TranspileError::IllegalIndexing(IllegalIndexing {
reason: IllegalIndexingReason::InvalidComptimeType {
expected: ExpectedType::String,
},
expression: index.span(),
}));
};
let (cmd, value) = get_data_location(
&DataLocation::ScoreboardValue {
objective: objective.to_string(),
target,
},
transpiler,
);
Ok((false, cmd, value))
}
VariableData::ScoreboardArray { objective, targets } => {
let Ok(ComptimeValue::Integer(idx)) = index.comptime_eval(scope, &VoidHandler)
else {
return Err(TranspileError::IllegalIndexing(IllegalIndexing {
reason: IllegalIndexingReason::InvalidComptimeType {
expected: ExpectedType::Integer,
},
expression: index.span(),
}));
};
#[expect(clippy::option_if_let_else)]
if let Some(target) = usize::try_from(idx)
.ok()
.and_then(|index| targets.get(index))
{
let (cmd, value) = get_data_location(
&DataLocation::ScoreboardValue {
objective: objective.to_string(),
target: target.to_string(),
},
transpiler,
);
Ok((false, cmd, value))
} else {
Err(TranspileError::IllegalIndexing(IllegalIndexing {
reason: IllegalIndexingReason::IndexOutOfBounds {
index: usize::try_from(idx).unwrap_or(usize::MAX),
length: targets.len(),
},
expression: index.span(),
}))
}
}
VariableData::BooleanStorageArray {
storage_name,
paths,
} => {
let Ok(ComptimeValue::Integer(idx)) = index.comptime_eval(scope, &VoidHandler)
else {
return Err(TranspileError::IllegalIndexing(IllegalIndexing {
reason: IllegalIndexingReason::InvalidComptimeType {
expected: ExpectedType::Integer,
},
expression: index.span(),
}));
};
#[expect(clippy::option_if_let_else)]
if let Some(path) = usize::try_from(idx).ok().and_then(|index| paths.get(index))
{
let (cmd, value) = get_data_location(
&DataLocation::Storage {
storage_name: storage_name.to_string(),
path: path.to_string(),
r#type: StorageType::Boolean,
},
transpiler,
);
Ok((false, cmd, value))
} else {
Err(TranspileError::IllegalIndexing(IllegalIndexing {
reason: IllegalIndexingReason::IndexOutOfBounds {
index: usize::try_from(idx).unwrap_or(usize::MAX),
length: paths.len(),
},
expression: index.span(),
}))
}
}
VariableData::Tag { tag_name } => {
let Ok(ComptimeValue::String(entity)) =
index.comptime_eval(scope, &VoidHandler)
else {
return Err(TranspileError::IllegalIndexing(IllegalIndexing {
reason: IllegalIndexingReason::InvalidComptimeType {
expected: ExpectedType::String,
},
expression: index.span(),
}));
};
let (cmd, value) = get_data_location(
&DataLocation::Tag {
tag_name: tag_name.clone(),
entity,
},
transpiler,
);
Ok((false, cmd, value))
}
_ => Err(TranspileError::UnexpectedExpression(UnexpectedExpression(
Box::new(Expression::Primary(Primary::Identifier(ident.to_owned()))),
))),
@ -194,7 +345,7 @@ fn print_function(
("@a".into(), first)
};
let mut contains_macro = matches!(target, MacroString::MacroString(_));
let mut contains_macro = target.contains_macros();
let (mut cmds, parts) = match message_expression {
Expression::Primary(primary) => match primary {
@ -362,6 +513,24 @@ fn print_function(
cmds.extend(cur_cmds);
parts.push(part);
}
Expression::Primary(Primary::Indexed(indexed)) => {
match indexed.object().as_ref() {
Primary::Identifier(ident) => {
let (cur_contains_macro, cur_cmds, part) =
get_indexed_part(
ident,
indexed.index(),
transpiler,
scope,
)?;
contains_macro |= cur_contains_macro;
cmds.extend(cur_cmds);
parts.push(part);
}
_ => todo!("other expression in indexed"),
}
}
_ => todo!("other expression in template string literal"),
}
}

17
src/transpile/util.rs
View File

@ -14,6 +14,8 @@ use crate::{
},
};
use super::expression::ComptimeValue;
/// String that can contain macros
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
@ -291,7 +293,7 @@ impl TemplateStringLiteral {
.iter()
.map(|part| match part {
TemplateStringLiteralPart::Text(text) => Ok(MacroStringPart::String(
crate::util::unescape_macro_string(text.span.str()).to_string(),
crate::util::unescape_template_string(text.span.str()).into_owned(),
)),
TemplateStringLiteralPart::Expression { expression, .. } => {
match expression.as_ref() {
@ -305,6 +307,17 @@ impl TemplateStringLiteral {
VariableData::MacroParameter { macro_name, .. } => Ok(
MacroStringPart::MacroUsage(macro_name.to_owned()),
),
VariableData::ComptimeValue { value, .. } => {
let value = value.read().unwrap().as_ref().map_or_else(
|| "null".into(),
ComptimeValue::to_macro_string,
);
match value.as_str() {
Ok(s) => Ok(MacroStringPart::String(s.into_owned())),
Err(_) => todo!("comptime value resulting in macro string with macros")
}
}
_ => todo!("other identifiers in template strings"),
}
} else {
@ -325,7 +338,7 @@ impl TemplateStringLiteral {
Ok(macro_string)
} else {
Ok(MacroString::String(self.str_content()))
Ok(MacroString::String(self.as_str(scope, handler)?.into_owned()))
}
}
}

21
src/util.rs
View File

@ -20,15 +20,16 @@ pub fn escape_str(s: &str) -> Cow<'_, str> {
}
}
/// Unescapes '\`', `\`, `\n`, `\r` and `\t` in a string.
/// Unescapes '\`', `\`, `\n`, `\r` and `\t`, `\$` in a string.
#[must_use]
pub fn unescape_macro_string(s: &str) -> Cow<'_, str> {
pub fn unescape_template_string(s: &str) -> Cow<'_, str> {
if s.contains('\\') || s.contains('`') {
Cow::Owned(
s.replace("\\n", "\n")
.replace("\\r", "\r")
.replace("\\t", "\t")
.replace("\\`", "`")
.replace("\\$", "$")
.replace("\\\\", "\\"),
)
} else {
@ -147,33 +148,33 @@ mod tests {
#[test]
fn test_unescape_macro_string() {
assert_eq!(unescape_macro_string("Hello, world!"), "Hello, world!");
assert_eq!(unescape_template_string("Hello, world!"), "Hello, world!");
assert_eq!(
unescape_macro_string(r#"Hello, "world"!"#),
unescape_template_string(r#"Hello, "world"!"#),
r#"Hello, "world"!"#
);
assert_eq!(
unescape_macro_string(r"Hello, \world\!"),
unescape_template_string(r"Hello, \world\!"),
r"Hello, \world\!"
);
assert_eq!(
unescape_macro_string(r"Hello, \nworld\!"),
unescape_template_string(r"Hello, \nworld\!"),
"Hello, \nworld\\!"
);
assert_eq!(
unescape_macro_string(r"Hello, \rworld\!"),
unescape_template_string(r"Hello, \rworld\!"),
"Hello, \rworld\\!"
);
assert_eq!(
unescape_macro_string(r"Hello, \tworld\!"),
unescape_template_string(r"Hello, \tworld\!"),
"Hello, \tworld\\!"
);
assert_eq!(
unescape_macro_string(r"Hello, \`world\!"),
unescape_template_string(r"Hello, \`world\!"),
r"Hello, `world\!"
);
assert_eq!(
unescape_macro_string(r"Hello, \\world\!"),
unescape_template_string(r"Hello, \\world\!"),
r"Hello, \world\!"
);
}