// Copyright 2023 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // ANCHOR: solution use thiserror::Error; // ANCHOR: types use std::iter::Peekable; use std::str::Chars; /// An arithmetic operator. #[derive(Debug, PartialEq, Clone, Copy)] enum Op { Add, Sub, } /// A token in the expression language. #[derive(Debug, PartialEq)] enum Token { Number(String), Identifier(String), Operator(Op), } /// An expression in the expression language. #[derive(Debug, PartialEq)] enum Expression { /// A reference to a variable. Var(String), /// A literal number. Number(u32), /// A binary operation. Operation(Box, Op, Box), } // ANCHOR_END: types fn tokenize(input: &str) -> Tokenizer { return Tokenizer(input.chars().peekable()); } #[derive(Debug, Error)] enum TokenizerError { #[error("Unexpected character '{0}' in input")] UnexpectedCharacter(char), } struct Tokenizer<'a>(Peekable>); impl<'a> Tokenizer<'a> { fn collect_number(&mut self, first_char: char) -> Token { let mut num = String::from(first_char); while let Some(&c @ '0'..='9') = self.0.peek() { num.push(c); self.0.next(); } Token::Number(num) } fn collect_identifier(&mut self, first_char: char) -> Token { let mut ident = String::from(first_char); while let Some(&c @ ('a'..='z' | '_' | '0'..='9')) = self.0.peek() { ident.push(c); self.0.next(); } Token::Identifier(ident) } } impl<'a> Iterator for Tokenizer<'a> { type Item = Result; fn next(&mut self) -> Option> { let c = self.0.next()?; match c { '0'..='9' => Some(Ok(self.collect_number(c))), 'a'..='z' | '_' => Some(Ok(self.collect_identifier(c))), '+' => Some(Ok(Token::Operator(Op::Add))), '-' => Some(Ok(Token::Operator(Op::Sub))), _ => Some(Err(TokenizerError::UnexpectedCharacter(c))), } } } #[derive(Debug, Error)] enum ParserError { #[error("Tokenizer error: {0}")] TokenizerError(#[from] TokenizerError), #[error("Unexpected end of input")] UnexpectedEOF, #[error("Unexpected token {0:?}")] UnexpectedToken(Token), #[error("Invalid number")] InvalidNumber(#[from] std::num::ParseIntError), } fn parse(input: &str) -> Result { let mut tokens = tokenize(input); fn parse_expr<'a>( tokens: &mut Tokenizer<'a>, ) -> Result { let tok = tokens.next().ok_or(ParserError::UnexpectedEOF)??; let expr = match tok { Token::Number(num) => { let v = num.parse()?; Expression::Number(v) } Token::Identifier(ident) => Expression::Var(ident), Token::Operator(_) => return Err(ParserError::UnexpectedToken(tok)), }; // Look ahead to parse a binary operation if present. Ok(match tokens.next() { None => expr, Some(Ok(Token::Operator(op))) => Expression::Operation( Box::new(expr), op, Box::new(parse_expr(tokens)?), ), Some(Err(e)) => return Err(e.into()), Some(Ok(tok)) => return Err(ParserError::UnexpectedToken(tok)), }) } parse_expr(&mut tokens) } fn main() -> anyhow::Result<()> { let expr = parse("10+foo+20-30")?; println!("{expr:?}"); Ok(()) } // ANCHOR_END: solution /* // ANCHOR: panics fn tokenize(input: &str) -> Tokenizer { return Tokenizer(input.chars().peekable()); } struct Tokenizer<'a>(Peekable>); impl<'a> Tokenizer<'a> { fn collect_number(&mut self, first_char: char) -> Token { let mut num = String::from(first_char); while let Some(&c @ '0'..='9') = self.0.peek() { num.push(c); self.0.next(); } Token::Number(num) } fn collect_identifier(&mut self, first_char: char) -> Token { let mut ident = String::from(first_char); while let Some(&c @ ('a'..='z' | '_' | '0'..='9')) = self.0.peek() { ident.push(c); self.0.next(); } Token::Identifier(ident) } } impl<'a> Iterator for Tokenizer<'a> { type Item = Token; fn next(&mut self) -> Option { let c = self.0.next()?; match c { '0'..='9' => Some(self.collect_number(c)), 'a'..='z' => Some(self.collect_identifier(c)), '+' => Some(Token::Operator(Op::Add)), '-' => Some(Token::Operator(Op::Sub)), _ => panic!("Unexpected character {c}"), } } } fn parse(input: &str) -> Expression { let mut tokens = tokenize(input); fn parse_expr<'a>(tokens: &mut Tokenizer<'a>) -> Expression { let Some(tok) = tokens.next() else { panic!("Unexpected end of input"); }; let expr = match tok { Token::Number(num) => { let v = num.parse().expect("Invalid 32-bit integer'"); Expression::Number(v) } Token::Identifier(ident) => Expression::Var(ident), Token::Operator(_) => panic!("Unexpected token {tok:?}"), }; // Look ahead to parse a binary operation if present. match tokens.next() { None => expr, Some(Token::Operator(op)) => Expression::Operation( Box::new(expr), op, Box::new(parse_expr(tokens)), ), Some(tok) => panic!("Unexpected token {tok:?}"), } } parse_expr(&mut tokens) } fn main() { let expr = parse("10+foo+20-30"); println!("{expr:?}"); } // ANCHOR_END: panics */