diff --git a/calc/CalcEvaluator.scala b/calc/CalcEvaluator.scala new file mode 100644 index 0000000..5b1fca9 --- /dev/null +++ b/calc/CalcEvaluator.scala @@ -0,0 +1,123 @@ +// Copyright 2024-2025 DCal Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package distcompiler.calc + +import cats.syntax.all.given + +import distcompiler.* +import dsl.* +import distcompiler.Builtin.{Error, SourceMarker} +import distcompiler.calc.tokens.* + +object CalcEvaluator extends PassSeq: + import distcompiler.dsl.* + import Reader.* + import CalcReader.* + + def inputWellformed: Wellformed = distcompiler.calc.wellformed + + private val simplifyPass = passDef: + wellformed := inputWellformed.makeDerived: + Node.Top ::=! Expression + + pass(once = false, strategy = pass.bottomUp) + .rules: + on( + field(tok(Expression)) *> onlyChild( + tok(Add).withChildren: + field(tok(Expression) *> onlyChild(tok(Number))) + ~ field(tok(Expression) *> onlyChild(tok(Number))) + ~ eof + ) + ).rewrite: (left, right) => + val leftNum = left.unparent().sourceRange.decodeString().toInt + val rightNum = right.unparent().sourceRange.decodeString().toInt + + splice( + Expression( + Number( + (leftNum + rightNum).toString() + ) + ) + ) + | on( + field(tok(Expression)) *> onlyChild( + tok(Sub).withChildren: + field(tok(Expression) *> onlyChild(tok(Number))) + ~ field(tok(Expression) *> onlyChild(tok(Number))) + ~ eof + ) + ).rewrite: (left, right) => + val leftNum = left.unparent().sourceRange.decodeString().toInt + val rightNum = right.unparent().sourceRange.decodeString().toInt + + splice( + Expression( + Number( + (leftNum - rightNum).toString() + ) + ) + ) + | on( + field(tok(Expression)) *> onlyChild( + tok(Mul).withChildren: + field(tok(Expression) *> onlyChild(tok(Number))) + ~ field(tok(Expression) *> onlyChild(tok(Number))) + ~ eof + ) + ).rewrite: (left, right) => + val leftNum = left.unparent().sourceRange.decodeString().toInt + val rightNum = right.unparent().sourceRange.decodeString().toInt + + splice( + Expression( + Number( + (leftNum * rightNum).toString() + ) + ) + ) + | on( + field(tok(Expression)) *> onlyChild( + tok(Div).withChildren: + field(tok(Expression) *> onlyChild(tok(Number))) + ~ field(tok(Expression) *> onlyChild(tok(Number))) + ~ eof + ) + ).rewrite: (left, right) => + val leftNum = left.unparent().sourceRange.decodeString().toInt + val rightNum = right.unparent().sourceRange.decodeString().toInt + + splice( + Expression( + Number( + (leftNum / rightNum).toString() + ) + ) + ) + + private val removeLayerPass = passDef: + wellformed := prevWellformed.makeDerived: + Node.Top ::=! Number + + pass(once = true, strategy = pass.topDown) + .rules: + on( + tok(Expression).withChildren: + field(tok(Number)) + ~ eof + ).rewrite: (number) => + splice( + number.unparent() + ) diff --git a/calc/CalcParser.scala b/calc/CalcParser.scala new file mode 100644 index 0000000..fde8b10 --- /dev/null +++ b/calc/CalcParser.scala @@ -0,0 +1,127 @@ +// Copyright 2024-2025 DCal Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package distcompiler.calc + +import cats.syntax.all.given + +import distcompiler.* +import dsl.* +import distcompiler.Builtin.{Error, SourceMarker} +import distcompiler.calc.tokens.* + +object CalcParser extends PassSeq: + import distcompiler.dsl.* + import Reader.* + import CalcReader.* + + def inputWellformed: Wellformed = + CalcReader.wellformed.makeDerived: + Add ::= fields( + Expression, + Expression + ) + + Sub ::= fields( + Expression, + Expression + ) + + Mul ::= fields( + Expression, + Expression + ) + + Div ::= fields( + Expression, + Expression + ) + + Expression ::=! choice( + Number, + Add, + Sub, + Mul, + Div + ) + + private val mulDivPass = passDef: + wellformed := inputWellformed.makeDerived: + Node.Top ::=! repeated(choice(Expression, AddOp, SubOp)) + + pass(once = false, strategy = pass.topDown) + .rules: + on( + field(tok(Expression)) + ~ skip(tok(MulOp)) + ~ field(tok(Expression)) + ~ trailing + ).rewrite: (left, right) => + splice( + Expression( + Mul( + left.unparent(), + right.unparent() + ) + ) + ) + | on( + field(tok(Expression)) + ~ skip(tok(DivOp)) + ~ field(tok(Expression)) + ~ trailing + ).rewrite: (left, right) => + splice( + Expression( + Div( + left.unparent(), + right.unparent() + ) + ) + ) + + private val addSubPass = passDef: + wellformed := prevWellformed.makeDerived: + Node.Top ::=! repeated(Expression) + + pass(once = false, strategy = pass.topDown) + .rules: + on( + field(tok(Expression)) + ~ skip(tok(AddOp)) + ~ field(tok(Expression)) + ~ trailing + ).rewrite: (left, right) => + splice( + Expression( + Add( + left.unparent(), + right.unparent() + ) + ) + ) + | on( + field(tok(Expression)) + ~ skip(tok(SubOp)) + ~ field(tok(Expression)) + ~ trailing + ).rewrite: (left, right) => + splice( + Expression( + Sub( + left.unparent(), + right.unparent() + ) + ) + ) diff --git a/calc/CalcReader.scala b/calc/CalcReader.scala new file mode 100644 index 0000000..1697eac --- /dev/null +++ b/calc/CalcReader.scala @@ -0,0 +1,94 @@ +// Copyright 2024-2025 DCal Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package distcompiler.calc + +import cats.syntax.all.given + +import distcompiler.* +import dsl.* +import distcompiler.Builtin.{Error, SourceMarker} +import Reader.* +import distcompiler.calc.tokens.* + +object CalcReader extends Reader: + override lazy val wellformed = Wellformed: + Node.Top ::= repeated(choice(Expression, AddOp, SubOp, MulOp, DivOp)) + + Number ::= Atom + AddOp ::= Atom + SubOp ::= Atom + MulOp ::= Atom + DivOp ::= Atom + + Expression ::= fields( + Number + ) + + private val digit: Set[Char] = ('0' to '9').toSet + private val whitespace: Set[Char] = Set(' ', '\n', '\t') + + private lazy val unexpectedEOF: Manip[SourceRange] = + consumeMatch: m => + addChild(Error("unexpected EOF", SourceMarker(m))) + *> Manip.pure(m) + + protected lazy val rules: Manip[SourceRange] = + commit: + bytes + .selecting[SourceRange] + .onOneOf(whitespace): + extendThisNodeWithMatch(rules) + .onOneOf(digit): + numberMode + .on('+'): + addChild(AddOp()) + *> rules + .on('-'): + addChild(SubOp()) + *> rules + .on('*'): + addChild(MulOp()) + *> rules + .on('/'): + addChild(DivOp()) + *> rules + .fallback: + bytes.selectOne: + consumeMatch: m => + addChild(Error("invalid byte", SourceMarker(m))) + *> rules + | consumeMatch: m => + on(theTop).check + *> Manip.pure(m) + | unexpectedEOF + + private lazy val numberMode: Manip[SourceRange] = + commit: + bytes + .selecting[SourceRange] + .onOneOf(digit)(numberMode) + .fallback: + consumeMatch: m => + m.decodeString().toIntOption match + case Some(value) => + addChild( + Expression( + Number(m) + ) + ) + *> rules + case None => + addChild(Error("invalid number format", SourceMarker(m))) + *> rules diff --git a/calc/CalcReader.test.scala b/calc/CalcReader.test.scala new file mode 100644 index 0000000..3911f74 --- /dev/null +++ b/calc/CalcReader.test.scala @@ -0,0 +1,307 @@ +// Copyright 2024-2025 DCal Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package distcompiler.calc + +import distcompiler.* +import Builtin.{Error, SourceMarker} + +class CalcReaderTests extends munit.FunSuite: + extension (str: String) + def read: Node.Top = + calc.read.fromSourceRange(SourceRange.entire(Source.fromString(str))) + + def parse: Node.Top = + val top = read + + CalcParser( + top, + tracer = Manip.RewriteDebugTracer(os.pwd / "dbg_calc_parser_passes") + ) + + os.write.over( + os.pwd / "dbg_calc_parser" / "test_output.dbg", + top.toPrettyWritable(CalcReader.wellformed), + createFolders = true + ) + + top + + def evaluate: Node.Top = + val top = parse + + CalcEvaluator( + top, + tracer = Manip.RewriteDebugTracer(os.pwd / "dbg_calc_evaluator_passes") + ) + + os.write.over( + os.pwd / "dbg_calc_evaluator" / "test_output.dbg", + top.toPrettyWritable(CalcReader.wellformed), + createFolders = true + ) + + top + + test("empty string"): + assertEquals("".read, Node.Top()) + + test("only whitespace"): + assertEquals(" \n\t".read, Node.Top()) + + test("error: invalid character"): + assertEquals("k".read, Node.Top(Error("invalid byte", SourceMarker("k")))) + + test("single number"): + assertEquals( + "5".read, + Node.Top( + tokens.Expression( + tokens.Number("5") + ) + ) + ) + + test("read basic addition"): + assertEquals( + "5 + 11".read, + Node.Top( + tokens.Expression( + tokens.Number("5") + ), + tokens.AddOp(), + tokens.Expression( + tokens.Number("11") + ) + ) + ) + + test("read basic multiplication"): + assertEquals( + "5 * 11".read, + Node.Top( + tokens.Expression( + tokens.Number("5") + ), + tokens.MulOp(), + tokens.Expression( + tokens.Number("11") + ) + ) + ) + + test("read full calculation"): + assertEquals( + "5 + 11 * 4".read, + Node.Top( + tokens.Expression( + tokens.Number("5") + ), + tokens.AddOp(), + tokens.Expression( + tokens.Number("11") + ), + tokens.MulOp(), + tokens.Expression( + tokens.Number("4") + ) + ) + ) + + test("simple addition parse"): + assertEquals( + "5 + 11".parse, + Node.Top( + tokens + .Expression( + tokens + .Add( + tokens.Expression( + tokens.Number("5") + ), + tokens.Expression( + tokens.Number("11") + ) + ) + .at("5 + 11") + ) + .at("5 + 11") + ) + ) + + test("simple multiplication parse"): + assertEquals( + "5 * 11".parse, + Node.Top( + tokens + .Expression( + tokens + .Mul( + tokens.Expression( + tokens.Number("5") + ), + tokens.Expression( + tokens.Number("11") + ) + ) + .at("5 * 11") + ) + .at("5 * 11") + ) + ) + + test("full calculation parse"): + assertEquals( + "5 + 11 * 4".parse, + Node.Top( + tokens + .Expression( + tokens + .Add( + tokens.Expression( + tokens.Number("5") + ), + tokens + .Expression( + tokens + .Mul( + tokens.Expression( + tokens.Number("11") + ), + tokens.Expression( + tokens.Number("4") + ) + ) + .at("11 * 4") + ) + .at("11 * 4") + ) + .at("5 + 11 * 4") + ) + .at("5 + 11 * 4") + ) + ) + + test("full calculation 4 parse"): + assertEquals( + "5 * 4 + 4 / 2 - 6 * 2".parse, + Node.Top( + tokens + .Expression( + tokens + .Sub( + tokens + .Expression( + tokens + .Add( + tokens + .Expression( + tokens + .Mul( + tokens.Expression( + tokens.Number("5") + ), + tokens.Expression( + tokens.Number("4") + ) + ) + .at("5 * 4") + ) + .at("5 * 4"), + tokens + .Expression( + tokens + .Div( + tokens.Expression( + tokens.Number("4") + ), + tokens.Expression( + tokens.Number("2") + ) + ) + .at("4 / 2") + ) + .at("4 / 2") + ) + .at("5 * 4 + 4 / 2") + ) + .at("5 * 4 + 4 / 2"), + tokens + .Expression( + tokens + .Mul( + tokens.Expression( + tokens.Number("6") + ), + tokens.Expression( + tokens.Number("2") + ) + ) + .at("6 * 2") + ) + .at("6 * 2") + ) + .at("5 * 4 + 4 / 2 - 6 * 2") + ) + .at("5 * 4 + 4 / 2 - 6 * 2") + ) + ) + + test("addition calculation"): + assertEquals( + "5 + 11".evaluate, + Node.Top( + tokens.Number("16") + ) + ) + + test("multiplication calculation"): + assertEquals( + "5 * 11".evaluate, + Node.Top( + tokens.Number("55") + ) + ) + + test("full calculation"): + assertEquals( + "5 + 11 * 4".evaluate, + Node.Top( + tokens.Number("49") + ) + ) + + test("full calculation 2"): + assertEquals( + "5 * 4 + 4 / 2".evaluate, + Node.Top( + tokens.Number("22") + ) + ) + + test("full calculation 3"): + assertEquals( + "5 * 4 + 4 / 2 - 6".evaluate, + Node.Top( + tokens.Number("16") + ) + ) + + test("full calculation 4"): + assertEquals( + "5 * 4 + 4 / 2 - 6 * 2".evaluate, + Node.Top( + tokens.Number("10") + ) + ) diff --git a/calc/README.md b/calc/README.md new file mode 100644 index 0000000..c87b7d1 --- /dev/null +++ b/calc/README.md @@ -0,0 +1,79 @@ +# ```calc/``` + +## Overview +A parser and evaluator for arithmetic expressions given in string inputs; supporting addition, subtraction, multiplication, and division operations. + + +## Motivation +The calculator exists as a practical example demonstrating how DCal can be leveraged to parse languages and manipulate ASTs, with many of DCal's core features such as ```Wellformed```, ```PassSeq```, and ```SeqPattern``` being heavily used. + + +## Components + +### 1. ```package.scala``` +Defines all of the ```Token``` types that are used and provides a wellformed definition representing how the AST should be structured once fully built. + + +### 2. ```CalcReader.scala``` +```CalcReader``` is the lexer that converts input strings into tokens. + +It contains a wellformed definition of the initial token types with ```Number``` tokens that're wrapped around ```Expression``` and ```Op``` tokens for operations. + +The ```rules``` method uses byte-level pattern matching to create tokens for numbers and operators and skip anything else. + + +### 3. ```CalcParser.scala``` +```CalcParser``` is the parser, transforming the flat list of tokens into a structured AST. + +The wellformed definition adds new ```Operation``` token types that have 2 ```Expression``` children. Also, ```Expression``` tokens have a new definition, being able to wrap both ```Number``` tokens as well as ```Operation``` tokens. + +```mulDivPass``` and ```addSubPass``` both create nested expressions and splicing the old ```Op``` tokens that were previously defined. Both methods do this by pattern matching on the sequence of ```(Expression, Op, Expression)``` and replacing this sequence with + +``` +Expression( + Operation( + Expression, + Expression + ) +) +``` + +```mulDivPass``` is executed before ```addSubPass``` to create precedence, allowing multiplication and division operations to be nested deeper than addition and subtraction operations in the AST. + + +### 4. ```CalcEvaluator.scala``` +```CalcEvaluator``` simplifies the AST and computes the value of the arithmetic expression. + +The wellformed definition is imported from ```package.scala```, picking up with the AST structure of where ```CalcParser``` left off. + +```simplifyPass``` splices all expressions repeatedly until there's only a single expression node at the top of the AST structure. The pass uses a bottom-up strategy to begin with simplifying the base-case expressions with no nesting as it goes up the AST. + +The pass sequence pattern matches on + +``` +Expression( + Operation( + Expression( + Number + ), + Expression( + Number + ) + ) +) +``` + +and replaces the sequence with ```Expression(Number)```. The remaining ```Expression``` token at the end of the pass contains the value of arithmetic expression. + +```removeLayerPass``` splices the ```Expression``` token at the top of the AST and replaces it with the ```Number``` token that was wrapped inside. Pattern matching is done on the sequence of ```Expression(Number)``` and replaces it with just ```Number```. + + +## Usage +To learn how to use the calculator, ```CalcReader.test.scala``` contains methods (```parse```, ```read```, ```evaluate```) that execute the different components of the calculator. + + +## Example +The following images demonstrates the state of the AST after each pass with the input "5 + 3 * 4". Viewing the state of the AST after each pass can also be done by running calculator test cases with the tracer enabled. + +![example1](img/example1.jpg) +![exampel2](img/example2.jpg) diff --git a/calc/img/example1.jpg b/calc/img/example1.jpg new file mode 100644 index 0000000..6ba2279 Binary files /dev/null and b/calc/img/example1.jpg differ diff --git a/calc/img/example2.jpg b/calc/img/example2.jpg new file mode 100644 index 0000000..ac2ef1b Binary files /dev/null and b/calc/img/example2.jpg differ diff --git a/calc/package.scala b/calc/package.scala new file mode 100644 index 0000000..0c13d6d --- /dev/null +++ b/calc/package.scala @@ -0,0 +1,92 @@ +// Copyright 2024-2025 DCal Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package distcompiler.calc + +import cats.syntax.all.given + +import scala.collection.mutable + +import distcompiler.* +import dsl.* +import distcompiler.calc.tokens.* + +object tokens: + object Expression extends Token: + override def showSource: Boolean = false + + object AddOp extends Token: + override def showSource: Boolean = true + + object DivOp extends Token: + override def showSource: Boolean = true + + object MulOp extends Token: + override def showSource: Boolean = true + + object SubOp extends Token: + override def showSource: Boolean = true + + object Add extends Token: + override def showSource: Boolean = true + + object Sub extends Token: + override def showSource: Boolean = true + + object Mul extends Token: + override def showSource: Boolean = true + + object Div extends Token: + override def showSource: Boolean = true + + object Number extends Token: + override def showSource: Boolean = true + +val wellformed: Wellformed = + Wellformed: + Node.Top ::= repeated(tokens.Expression) + + tokens.Number ::= Atom + + tokens.Add ::= fields( + tokens.Expression, + tokens.Expression + ) + + tokens.Sub ::= fields( + tokens.Expression, + tokens.Expression + ) + + tokens.Mul ::= fields( + tokens.Expression, + tokens.Expression + ) + + tokens.Div ::= fields( + tokens.Expression, + tokens.Expression + ) + + tokens.Expression ::= choice( + tokens.Number, + tokens.Add, + tokens.Sub, + tokens.Mul, + tokens.Div + ) + +object read: + def fromSourceRange(sourceRange: SourceRange): Node.Top = + CalcReader(sourceRange)