Cleaned up version for v0.9.

ChangeLog.md

@@ -2,4 +2,11 @@
 
 ## 0.8
 
-- Added the frontend including the Lexer, Parser, Rewriter and Analyser.
+- Added the frontend including the Lexer, Parser, Rewriter and Analyser.
+
+## 0.9
+
+- Added the intermediate representation language, Espresso, with its own lexer and parser.
+- Added IR generation.
+- Added Control Flow Graph and variable liveness analysis.
+- Added x86_64 assembler code generation for the core of the language.

Latte.cabal

@@ -4,7 +4,7 @@ cabal-version: 1.12
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: e6604d4d52f5a5ece6040d707c4974ac74620fa8b272f50a16cecf5e8883b406
+-- hash: 028249ea0ebe143be13f7c63bd5fe2393ecd494c17d63a038438551cb0e68895
 
 name:           Latte
 version:        0.9.0.0
@@ -31,6 +31,9 @@ library
       ErrM
       Error
       Espresso.CodeGen.Generator
+      Espresso.CodeGen.GenM
+      Espresso.CodeGen.Labels
+      Espresso.CodeGen.Operators
       Espresso.ControlFlow.CFG
       Espresso.ControlFlow.Liveness
       Espresso.ControlFlow.Phi
@@ -39,6 +42,8 @@ library
       Espresso.Syntax.Lexer
       Espresso.Syntax.Parser
       Espresso.Syntax.Printer
+      Espresso.Types
+      Espresso.Utilities
       Identifiers
       LatteIO
       SemanticAnalysis.Analyser
@@ -52,13 +57,19 @@ library
       Syntax.Printer
       Syntax.Rewriter
       Utilities
-      X86_64.Consts
-      X86_64.Emit
-      X86_64.Generator
+      X86_64.CodeGen.Consts
+      X86_64.CodeGen.Emit
+      X86_64.CodeGen.Epilogue
+      X86_64.CodeGen.Generator
+      X86_64.CodeGen.GenM
+      X86_64.CodeGen.Module
+      X86_64.CodeGen.Prologue
+      X86_64.CodeGen.RegisterAllocation
+      X86_64.CodeGen.Stack
       X86_64.Loc
+      X86_64.Optimisation.Peephole
       X86_64.Registers
       X86_64.Size
-      X86_64.Stack
   other-modules:
       Paths_Latte
   hs-source-dirs:
@@ -73,6 +84,7 @@ library
     , hspec >=2.7 && <2.8
     , mtl >=2.2 && <2.3
     , process >=1.6 && <1.7
+    , regex >=1.1 && <1.2
   default-language: Haskell2010
 
 executable espi
@@ -93,6 +105,7 @@ executable espi
     , hspec >=2.7 && <2.8
     , mtl >=2.2 && <2.3
     , process >=1.6 && <1.7
+    , regex >=1.1 && <1.2
   default-language: Haskell2010
 
 executable latc_x86_64
@@ -113,6 +126,7 @@ executable latc_x86_64
     , hspec >=2.7 && <2.8
     , mtl >=2.2 && <2.3
     , process >=1.6 && <1.7
+    , regex >=1.1 && <1.2
   default-language: Haskell2010
 
 test-suite Latte-exec-test
@@ -133,6 +147,7 @@ test-suite Latte-exec-test
     , hspec >=2.7 && <2.8
     , mtl >=2.2 && <2.3
     , process >=1.6 && <1.7
+    , regex >=1.1 && <1.2
   default-language: Haskell2010
 
 test-suite Latte-test
@@ -155,4 +170,5 @@ test-suite Latte-test
     , hspec >=2.7 && <2.8
     , mtl >=2.2 && <2.3
     , process >=1.6 && <1.7
+    , regex >=1.1 && <1.2
   default-language: Haskell2010

README.md

@@ -1,23 +1,126 @@
-# Latte v0.8
+# Latte v0.9
 
 Compiler of the [Latte programming language](https://www.mimuw.edu.pl/~ben/Zajecia/Mrj2020/Latte/description.html) written in Haskell.
 
 ## Compiling the project
 
-Use `stack build` to compile the project. Use the `latc` executable to compile `.lat` source files.
+Use `stack build` to compile the project.
 
 Used version of GHC is 8.8.4 (LTS 16.22). For the breakdown of used packages consult `package.yaml`.
 
+## Running the project
+
+Use the `latc_x86_64` executable to compile `.lat` source files. Additionally, `espi` executable containing an IR interpreter is generated - this is not intended for end users but rather as a development tool.
+
+### Flags
+
+- `-v` - enable verbose mode
+- `-g` - generate intermediate steps
+
+When running on a file `<p>.lat`, where `<p> = <dir_path/<file_path>` is some path:
+
+1. when the flag `-g` is not specified, two files are created: `<p>.s` containing generated x86_64 assembly and an executable `<p>`;
+
+2. when `-g` is specified the following intermediate representations are generated additionally:
+
+- `<p>.esp`, `<p>.cfg` - Espresso intermediate code and text representation of its Control Flow Graph.
+- `<p>.1.opt.esp`, `<p>.1.opt.cfg` - Optimised Espresso code and its CFG.
+- `<p>.2.phi.esp`, `<p>.2.phi.cfg` - Optimised Espresso code with unfolded phony `phi` function usage and its CFG.
+- `<p>.3.liv.esp`, `<p>.3.liv.cfg` - Same code as above but with liveness annotations in form of comments on every instruction and the CFG with liveness annotations for the begin and end of each node.
+- `<p>.noopt.s` - Generated assembly before peephole optimisation phase.
+
 ## Testing the project
 
-Use `stack test` to run all included tests. The `lattest` directory contains all tests provided on
+Use `stack test` to run all included tests. The `lattests` directory contains all tests provided on
 the [assignment page](https://www.mimuw.edu.pl/~ben/Zajecia/Mrj2020/latte-en.html) (these are unchanged)
-and additional custom tests.
+and additional custom tests. The `esptests` directory contains a handful of tests for the IR language Espresso.
+
+There are two test suites, `Latte-test` and `Latte-test-exec`.
+
+- `Latte-test` tests parsing, semantic analysis and IR code generation using an interpreter for the generated IR.
+These are quick to run but do not test anything related to x86_64 assembly generation.
+
+- `Latte-test-exec` tests the entire compiler by running the executable on all valid tests, asserting they compile and then running the generated executables. It creates a temporary work directory in `lattests` that gets cleaned up after the test.
 
 ## Features
 
 - Full lexer and parser for Latte with objects, virtual methods and arrays.
 - Full semantic analysis with type checks and reachability analysis.
+- Internal IR language - Espresso - with a separate lexer/parser and a small interpreter.
+- Compilation to Espresso and generation of additional annotations - Control Flow Graph and variable liveness analysis.
+- x86_64 assembly code generation with register handling done locally via register/variable descriptions.
+- Peephole optimisations of the generated assembly to fix common trivial inefficiencies.
+
+## Unimplemented extensions
+
+Objects, arrays and virtual methods are implemented in the static analysis phase, but not in Espresso or x86_64 codegen. These will be implemented in the next version of the compiler.
+
+## Known issues
+
+1. The way strings work is currently inconsistent with how objects will work in general. For example, the compiler assumes a default value for a string is `null`, but there is no way to express a string `null` literal in Latte code (the grammar does not allow `(string) null`). One can achieve it by declaring an uninitialised string variable, as the default value for such variables is `null`. It is planned to change in the final version where `strings` will most likely be defined as actual object types with special handling for string literals.
+
+2. The generated code is wasteful when it comes to string literals. For example the code:
+
+```
+string x = "foo";
+string y = "bar";
+string z = x + y;
+printString(z);
+```
+
+causes three string allocations for each of the variables `x`, `y`, `z`. This is planned to change in the final version where constant propagation will be implemented.
+
+3. There is a slight issue with string allocation. Codegen for a string literal:
+```
+// Espresso code
+%v_0 := "literal";
+call void foo(string& %v_0);
+```
+
+looks like this:
+
+```as
+__const_1:
+  .string "literal"
+
+...
+
+lea __const_1(%rip), %r10
+movq %r10, %rdi # moving %v_0
+movl $7, %esi
+movq %r10, %r12 # moving %v_0 <---
+call lat_new_string
+movq %rax, %rdi # moving %v_0
+call __cl_TopLevel.foo
+```
+the indicated `mov` instruction is redundant. The compiler sees that %v_0 is alive after the `IStr` instruction (namely used in the call to `foo`) so it tries to preserve it through the call to `lat_new_string` (since `%r10` is caller-saved). This is wasteful, since the value inside `%r10` at that point is the address of the string literal constant, but the logical value of `%v_0` is the result of the call to `lat_new_string`. Fixing this is nontrivial, so it will be done if time permits for the next version.
+
+4. Conditional jumps where locals have to be persisted between blocks result in inefficient codegen. For example, the way a `<=` conditional is generated in Espresso is:
+
+```
+%v_cond := %v_0 <= %v_1;
+jump if %v_cond then .L_then else .L_else;
+```
+Assume only %v_0 is alive at the end of this block. These two instructions are independently translated. First, the boolean value is created (assume `%v_0` in `%eax` and `%v_1` in `%edx`):
+```
+cmpl %eax, %edx
+setle %dl
+```
+Then the conditional jump:
+```
+testb %dl, %dl
+movl %eax, 8(%rbp) # save %v_0 on the stack
+jz .L_else
+jmp .L_then
+```
+But clearly this can be more efficiently realised with:
+```
+cmpl %eax, %edx
+movl %eax, 8(%rbp) # save %v_0 on the stack
+jg .L_else
+jmp .L_then
+```
+Some of these are fixed by peephole optimisations, but that approach fails when there are the save-on-stack `mov`s in between the `set`-`test`-`jz` sequence. This is non-trivial to fix, so it will be done in the next version of the compiler.
 
 ## Custom extensions
 
@@ -26,6 +129,10 @@ their type to the compile-time type of the initialiser expression. It cannot be
 used in declarations without an initialiser. The motivation behind this is mainly so that `for` loop rewrite works correctly,
 but it is also useful as a language feature so it is exposed to the user.
 
+## Runtime
+
+The runtime is small and contains the basic library functions `readInt`, `readString`, `printInt`, `printString` and `error` as well as internal core functions for string allocation and manipulation. It is written in C and included in `lib/runtime.c`. 
+
 ## Compilation process
 
 After lexing and parsing that is automated using BNFC the compilation proceeds in phases.
@@ -104,12 +211,41 @@ The control flow rules currently concern themselves with function return stateme
 to have a `return` statement on each possible execution path. The Analyser tracks reachability of statements and combines
 branch reachability of `if` and `while` statements. An important optimisation is that if a condition of a conditional
 statement is trivially true (or false) the branch is considered to be always (or never) entered. Trivially true or false
-means that it is either a true or false literal, but since this phase is performed after the Rewriter all constant boolean expressions
-are already collapsed to a single literal.
+means that it is either a true or false literal, but since this phase is performed after the Rewriter all constant boolean expressions are already collapsed to a single literal.
+
+### Phase four - Espresso codegen
+
+The modules in `Espresso.CodeGen` process the Latte code with annotations from semantic analysis and generates the intermediate representation in a quadruple code language Espresso. The grammar for the language can be found in `src/Espresso/Syntax/Espresso.cf`.
+
+A program in Espresso starts with a `.metadata` table that contains type information about all classes and functions defined in the Latte code, plus runtime functions. Then a sequence of `.method` definitions follows as a sequence of quadruples including labels, jumps and operations. For a detailed scription of the instruction set refer to the grammar file.
+
+The instruction set includes the phony function `phi` akin to LLVM's `phi`. It sets a value based on the label from which a jump was performed. The code generated by `Espresso.CodeGen` _is not_ in SSA form, but it uses `phi` for setting the return value of a method.
+
+Code generation ensures that there is no fall-through between labels, each basic block ends with a conditional or unconditional jump. Therefore, the blocks can be reordered arbitrarily.
+
+### Phase five - Espresso analysis
+
+The generated code needs additional annotations, mainly liveness information for all values and instructions. These are done by modules in `Espresso.ControlFlow`. First the code is divided into basic blocks and a Control Flow Graph is constructed. Then the phony function usage is unfolded, since it is untranslateable into assembly directly. Then liveness analysis is performed on the new CFG graph.
+
+This phase will also contain optimisation steps in the future version of the compiler.
+
+### Phase six - x86_64 assembly codegen
+
+Assembly generations proceeds by simulating the state of the target machine with register/value descriptions. Locals are persisted on the stack between basic blocks, while registers are greedily allocated within the block based on variable next use data computed in the previous phase. This phase leaves a lot of garbage code, like empty `addq $0, %rsp` instructions, but these are easily cleared in the next phase.
+
+### Phase seven - peephole optimisations
+
+The result code is analysed by matching a number of patterns of common unoptimal code and fixing them locally. The process is repeated until a fixpoint is reached, i.e. no more optimisations are applicable.
+
+### Phase eight - assembly and linking
+
+As the final phase, `gcc` is used to compile the generated assembly and link it with the runtime.
 
 ## Grammar conflicts
 
-The grammar contains 3 shift/reduce conflicts.
+### Latte
+
+The Latte grammar contains 3 shift/reduce conflicts.
 
 The first conflict is the standard issue with single-statement `if` statements that makes statements of the following form ambiguous:
 ```
@@ -125,9 +261,16 @@ and an instantiation of a type with immediate indexing into it. The conflict is
 
 The third conflict is between a parenthesised single expression and a casted `null` expression, which is correctly resolved in favour of the `null` expression.
 
+### Espresso
+
+There is 1 shift/reduce conflict between `VNegInt` and `UnOpNeg`. `VNegInt` is a negative integer required to allow passing literal negative values as arguments without creating values for them, which would be tedious. Therefore the expression:
+```
+%v_0 := -42;
+```
+is ambiguous between `IUnOp` on `42` or `ISet` on `-42`. This is inconsequential and can be resolved either way without changing semantics.
+
 ## Sources
 
-A few parts of the code were directly copied or heavily inspired by my earlier work on the Harper language (https://github.com/V0ldek/Harper),
-most notably the control flow analysis monoid based approach.
+A few parts of the code were directly copied or heavily inspired by my earlier work on the Harper language (https://github.com/V0ldek/Harper), most notably the control flow analysis monoid based approach.
 
 The grammar rules for `null` literals are a slightly modified version of rules proposed by Krzysztof Małysa.

lib/runtime.c

@@ -33,10 +33,10 @@
 // Default version of CHK_SYSERR_VAL that tests against a -1 value.
 #define CHK_SYSERR(x, name) CHK_SYSERR_VAL(x, -1, name)
 
-#define NOTNULL(x)                                    \
-    if ((x) == NULL)                                  \
-    {                                                 \
-        TERMINATE("internal error. null reference."); \
+#define NOTNULL(x)                                      \
+    if ((x) == NULL)                                    \
+    {                                                   \
+        TERMINATE("internal error. null reference.\n"); \
     }
 
 typedef struct lat_string
@@ -96,14 +96,14 @@ const lat_string *lat_read_string()
 
 void lat_error()
 {
-    TERMINATE("runtime error");
+    TERMINATE("runtime error\n");
 }
 
 void lat_nullchk(const void *ptr)
 {
     if (ptr == NULL)
     {
-        TERMINATE("runtime error. attempt to dereference a null.");
+        TERMINATE("runtime error. attempt to dereference a null.\n");
     }
 }
 

package.yaml

@@ -28,6 +28,7 @@ dependencies:
 - filepath >= 1.4 && < 1.5
 - process >= 1.6 && < 1.7
 - hspec >= 2.7 && < 2.8
+- regex >= 1.1 && < 1.2
 
 library:
   source-dirs: src

src/Compiler.hs

@@ -1,14 +1,14 @@
 module Compiler where
 
 import           Control.Monad                 (when)
-import           Data.Bifunctor
+import           Data.Bifunctor                (Bifunctor (first))
 import qualified Data.Map                      as Map
 import           ErrM                          (toEither)
 import           Espresso.CodeGen.Generator    (generateEspresso)
 import           Espresso.ControlFlow.CFG
 import           Espresso.ControlFlow.Liveness (Liveness, analyseLiveness,
                                                 emptyLiveness)
-import           Espresso.ControlFlow.Phi
+import           Espresso.ControlFlow.Phi      (unfoldPhi)
 import qualified Espresso.Syntax.Abs           as Esp
 import           Espresso.Syntax.Printer       as PrintEsp (Print, printTree,
                                                             printTreeWithInstrComments)
@@ -25,7 +25,8 @@ import           Syntax.Rewriter               (rewrite)
 import           System.FilePath               (dropExtension, takeDirectory,
                                                 takeFileName, (<.>), (</>))
 import           Utilities                     (unlessM)
-import           X86_64.Generator
+import           X86_64.CodeGen.Generator      (generate)
+import qualified X86_64.Optimisation.Peephole  as Peephole
 
 data Verbosity = Quiet | Verbose deriving (Eq, Ord, Show)
 
@@ -76,7 +77,9 @@ run opt = do
     genStep opt (espressoWithLivenessFile directory fileName) espressoWithLiveness
     printStringV v "Generating x86_64 assembly..."
     let assembly = generate cfgsWithLiveness
-    genOutput opt (assemblyFile directory fileName) assembly
+    genOutput opt (unoptAssemblyFile directory fileName) assembly
+    let optAssembly = unlines $ Peephole.optimise (lines assembly)
+    genOutput opt (assemblyFile directory fileName) optAssembly
 
 analysePhase :: (Monad m, LatteIO m) => Options -> String -> m (Metadata SemData)
 analysePhase opt latSrc = do
@@ -172,6 +175,9 @@ showEspWithLiveness meta mthds = PrintEsp.printTreeWithInstrComments (Esp.Progra
 assemblyFile :: FilePath -> FilePath -> FilePath
 assemblyFile dir file = dir </> file <.> "s"
 
+unoptAssemblyFile :: FilePath -> FilePath -> FilePath
+unoptAssemblyFile dir file = dir </> file <.> "noopt" <.> "s"
+
 espressoFile :: FilePath -> FilePath -> FilePath
 espressoFile dir file = dir </> file <.> "esp"