Initial project.

.gitignore

+/.idea
+/target
+Cargo.lock

Cargo.toml

+[package]
+name = "kaleidoscope-client"
+version = "0.1.0"
+authors = ["Eduardo Souza <ledusou@gmail.com>"]
+edition = "2018"
+#debug = true
+#opt-level = 0
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+
+mu-aot-llvm = { git = "https://gitlab.anu.edu.au/u1075163/mu-aot-llvm", branch = "master", version = "*"}
+mu          = { git = "https://gitlab.anu.edu.au/mu/mu-impl-fast", branch = "rust-2018", version = "*" }
+libloading  = "*"
+llvm-sys    = "*"

in0.txt

+4*10 + 10 - 8;
+
+13+25
\ No newline at end of file

in1.txt

+def foo(x y) x+foo(y, 4.0);
\ No newline at end of file

in2.txt

+def foo(x y) x*y;
+
+foo(3 2);
\ No newline at end of file

in3.txt

+extern sin(a);
+
+sin(1.23)
\ No newline at end of file

in4.txt

+# Compute the x'th fibonacci number.
+def fib(x)
+  if x < 3 then
+    1
+  else
+    fib(x-1)+fib(x-2)
+
+# This expression will compute the 40th number.
+fib(40.0)
\ No newline at end of file

src/generator.rs

+use crate::ast::*;
+use crate::parser::*;
+use llvm::prelude::LLVMValueRef;
+use std::collections::HashMap;
+use std::ffi::{CStr, CString};
+use std::{iter, ptr};
+
+extern crate llvm_sys as llvm;
+
+use self::llvm::analysis::LLVMVerifierFailureAction::{
+    LLVMAbortProcessAction, LLVMPrintMessageAction,
+};
+use self::llvm::analysis::{LLVMVerifyFunction, LLVMVerifyModule};
+use self::llvm::execution_engine::{
+    LLVMCreateExecutionEngineForModule, LLVMCreateJITCompilerForModule,
+    LLVMCreateSimpleMCJITMemoryManager, LLVMDisposeExecutionEngine, LLVMExecutionEngineRef,
+    LLVMFreeMachineCodeForFunction, LLVMGenericValueToFloat, LLVMInitializeMCJITCompilerOptions,
+    LLVMLinkInMCJIT, LLVMMCJITCompilerOptions, LLVMOpaqueExecutionEngine, LLVMRunFunction,
+};
+use self::llvm::prelude::{LLVMBool, LLVMBuilderRef, LLVMContextRef};
+use self::llvm::target::{LLVM_InitializeNativeAsmPrinter, LLVM_InitializeNativeTarget};
+use self::llvm::LLVMRealPredicate::LLVMRealOLT;
+use self::llvm::{LLVMModule, LLVMType, LLVMValue};
+use crate::lexer::Token::CHAR;
+use llvm::core::*;
+use std::borrow::Borrow;
+use std::os::raw::{c_char, c_int, c_uint};
+use self::llvm::target_machine::{LLVMTargetMachineEmitToFile, LLVMGetDefaultTargetTriple, LLVMGetFirstTarget, LLVMCodeGenOptLevel, LLVMRelocMode, LLVMCodeModel, LLVMCreateTargetMachine, LLVMCodeGenFileType};
+
+pub struct Context {
+    pub context: LLVMContextRef,
+    pub builder: LLVMBuilderRef,
+    pub named_values: HashMap<String, LLVMValueRef>,
+    pub ty: *mut LLVMType,
+    pub curr_function: i32,
+}
+
+impl Context {
+    pub unsafe fn new() -> Context {
+        let context = llvm::core::LLVMContextCreate();
+        let builder = llvm::core::LLVMCreateBuilderInContext(context);
+        let named_values = HashMap::new();
+        let ty = LLVMDoubleType();
+
+        Context {
+            context,
+            builder,
+            named_values,
+            ty,
+            curr_function : 0,
+        }
+    }
+
+    pub unsafe fn dump(&self) {
+        LLVMContextDispose(self.context);
+    }
+}
+
+unsafe fn run_function(
+    function: Result<*mut LLVMValue, String>,
+    module: *mut LLVMModule,
+    context: &mut Context,
+) {
+    let mut ee = 0 as LLVMExecutionEngineRef;
+    let mut error = 0 as *mut c_char;
+    let mut options = std::mem::uninitialized();
+
+    if LLVMVerifyModule(module, LLVMAbortProcessAction, &mut error) > 0 {
+        let cstr_buf = std::ffi::CStr::from_ptr(error);
+        let result = String::from_utf8_lossy(cstr_buf.to_bytes()).into_owned();
+        let message = LLVMCreateMessage(error);
+        let message = CString::from_raw(message);
+        println!("{:?}", message);
+        return;
+    }
+
+    LLVMInitializeMCJITCompilerOptions(
+        &mut options,
+        std::mem::size_of::<LLVMMCJITCompilerOptions>(),
+    );
+
+    LLVMLinkInMCJIT();
+    LLVM_InitializeNativeTarget();
+    LLVM_InitializeNativeAsmPrinter();
+    LLVM_InitializeNativeTarget();
+
+    if LLVMCreateJITCompilerForModule(&mut ee, module, options.OptLevel, &mut error) > 0 {
+        let cstr_buf = std::ffi::CStr::from_ptr(error);
+        let result = String::from_utf8_lossy(cstr_buf.to_bytes()).into_owned();
+        let message = LLVMCreateMessage(error);
+        let message = CString::from_raw(message);
+        println!("{:?}", message);
+        return;
+    }
+
+    let mut args = vec![];
+
+    let (f, result) = match function {
+        Ok(f) => {
+            (
+                f,
+                LLVMRunFunction(ee, f, args.len() as c_uint, args.as_mut_ptr()),
+            )
+        }
+        Err(e) => {
+            println!("{}", e);
+            return;
+        }
+    };
+
+    let triple = LLVMGetDefaultTargetTriple();
+    let target = LLVMGetFirstTarget();
+    let cpu = CString::new("x86-64").expect("invalid cpu");
+    let feature = CString::new("").expect("invalid feature");
+    let opt_level = LLVMCodeGenOptLevel::LLVMCodeGenLevelNone;
+    let reloc_mode = LLVMRelocMode::LLVMRelocDefault;
+    let code_model = LLVMCodeModel::LLVMCodeModelDefault;
+    let target_machine = LLVMCreateTargetMachine(target, triple, cpu.as_ptr(), feature.as_ptr(), opt_level, reloc_mode, code_model);
+    let file_type = LLVMCodeGenFileType::LLVMAssemblyFile;
+    let output_file = CString::new("output.S").expect("invalid file");
+
+    let mut error_str = ptr::null_mut();
+    let res = LLVMTargetMachineEmitToFile(target_machine, module, output_file.as_ptr() as *mut i8, file_type, &mut error_str);
+    if res == 1 {
+        let x = CStr::from_ptr(error_str);
+        let result = String::from_utf8_lossy(x.to_bytes()).into_owned();
+        let message = LLVMCreateMessage(error_str);
+        let message = CString::from_raw(message);
+        println!("{:?}", message);
+        return;
+    }
+
+    let result = LLVMGenericValueToFloat(LLVMDoubleType(), result);
+//    LLVMDisposeExecutionEngine(ee);
+
+    println!("{}", result);
+}
+
+pub unsafe fn code_gen(filename: String, nodes: ASTNodes) {
+    let mut context = Context::new();
+
+    let mut module_number = 0;
+
+    let module = llvm::core::LLVMModuleCreateWithName(filename.as_str().as_ptr() as *const _);
+
+    for prototype in nodes.prototypes {
+        let out_file = CString::new(format!("out_{}.ll", module_number)).unwrap();
+        prototype.code_gen(&module, &mut context);
+
+        llvm::core::LLVMPrintModuleToFile(module, out_file.as_ptr(), ptr::null_mut());
+    }
+
+    for function in nodes.functions {
+
+        let out_file = CString::new(format!("out_{}.ll", module_number)).unwrap();
+
+        let result = function.code_gen(&module, &mut context);
+
+        llvm::core::LLVMPrintModuleToFile(module, out_file.as_ptr(), ptr::null_mut());
+
+        // execute module
+        if function.proto().name() == "" {
+            run_function(result, module, &mut context);
+        }
+        let module = llvm::core::LLVMModuleCreateWithName(filename.as_str().as_ptr() as *const _);
+        let context = Context::new();
+
+    }
+
+    //LLVMDumpModule(module);
+}
+
+trait Code_gen {
+    unsafe fn code_gen(
+        &self,
+        module: &*mut LLVMModule,
+        context: &mut Context,
+    ) -> Result<LLVMValueRef, String>;
+}
+
+impl Code_gen for Prototype {
+    unsafe fn code_gen(
+        &self,
+        module: &*mut LLVMModule,
+        context: &mut Context,
+    ) -> Result<LLVMValueRef, String> {
+        let function = match get_function_by_name(*module, self.name().as_str()) {
+            Some(function) => {
+                if LLVMCountParams(function) as usize != self.args().len() {
+                    return Err(String::from(
+                        "Redefinition of function with different number of arguments.",
+                    ));
+                }
+
+                if LLVMCountBasicBlocks(function) > 0 {
+                    return Err(String::from("Redefinition of function"));
+                }
+
+                function
+            }
+            None => {
+                let mut param_types = iter::repeat(context.ty)
+                    .take(self.args().len())
+                    .collect::<Vec<_>>();
+                let fty = LLVMFunctionType(
+                    context.ty,
+                    param_types.as_mut_ptr(),
+                    param_types.len() as c_uint,
+                    false as LLVMBool,
+                );
+                let name;
+                if self.name().as_str().eq("") {
+                    let fname = format!("f_anonn_{}", context.curr_function);
+                    context.curr_function += 1;
+                    name = CString::new(fname.as_str()).unwrap();
+                } else {
+                    name = CString::new(self.name().clone().as_str()).unwrap();
+                }
+
+                let function = LLVMAddFunction(*module, name.as_ptr() as *const c_char, fty);
+
+                for i in 0..self.args().len() {
+                    let param = LLVMGetParam(function, i as u32);
+                    let param_name = self.args().get(i).unwrap().as_str();
+
+                    LLVMSetValueName2(
+                        param,
+                        param_name.as_ptr() as *const c_char,
+                        param_name.len(),
+                    )
+                }
+                function
+            }
+        };
+
+        let basic_blocs_count = LLVMCountBasicBlocks(function);
+
+        Ok(function)
+    }
+}
+
+pub fn get_function_by_name(function: *mut LLVMModule, name: &str) -> Option<*mut LLVMValue> {
+    let name = CString::new(name).unwrap();
+    let function = unsafe { LLVMGetNamedFunction(function, name.as_ptr() as *const c_char) };
+    if !function.is_null() {
+        unsafe { Some(function) }
+    } else {
+        None
+    }
+}
+
+impl Code_gen for Function {
+    unsafe fn code_gen(
+        &self,
+        module: &*mut LLVMModule,
+        context: &mut Context,
+    ) -> Result<LLVMValueRef, String> {
+        context.named_values.clear();
+
+        let function = match self.proto().code_gen(module, context) {
+            Ok(function) => function,
+            e @ Err(_) => return e,
+        };
+
+        let mut basic_block = LLVMAppendBasicBlockInContext(
+            context.context,
+            function,
+            CString::new("entry").unwrap().as_ptr(),
+        );
+
+        LLVMPositionBuilderAtEnd(context.builder, basic_block);
+
+        // set function parameters
+        for (param, arg) in get_params(function).iter().zip(self.proto().args()) {
+            context.named_values.insert(arg.clone(), *param);
+        }
+
+        let body = match self.body().code_gen(module, context) {
+            Ok(value) => value,
+            e @ Err(_) => {
+                LLVMDeleteFunction(function);
+                return e;
+            }
+        };
+
+        LLVMBuildRet(context.builder, body);
+
+        LLVMVerifyFunction(function, LLVMAbortProcessAction);
+
+        Ok(function)
+    }
+}
+
+unsafe fn get_params(function: *mut LLVMValue) -> Vec<LLVMValueRef> {
+    let params_count = LLVMCountParams(function);
+    let mut buf: Vec<LLVMValueRef> = Vec::with_capacity(params_count as usize);
+    let p = buf.as_mut_ptr();
+    unsafe {
+        std::mem::forget(buf);
+        LLVMGetParams(function, p);
+        Vec::from_raw_parts(p, params_count as usize, params_count as usize)
+    }
+}

src/lexer.rs

+use std::{fs, io};
+use std::ops::Add;
+
+#[derive(Debug)]
+pub enum Token {
+    EOF,
+    DEF,
+    EXTERN,
+    ID(String),
+    NUMBER(f64),
+    CHAR(char),
+    IF,
+    THEN,
+    ELSE
+}
+
+pub fn read_file() -> (String, Vec<Token>) {
+    loop {
+        println!("Enter filename or type '.quit' to close: ");
+
+        let mut filename = String::new();
+
+//        let result = match io::stdin().read_line(&mut filename) {
+//            Ok(_) => (),
+//            Err(e) => {
+//                println!("{}", e);
+//                continue;
+//            }
+//        };
+
+        let exit_code = String::from(".quit");
+
+        if filename.trim().eq(&exit_code) {
+            return (String::new(), vec![]);
+        }
+
+        let mut filename = String::from("test.txt");
+
+        if !filename.ends_with(".txt") {
+            filename = String::from(filename.trim());
+            filename = filename.add(".txt");
+        }
+
+        match fs::read_to_string(filename.trim()) {
+            Ok(contents) => {
+                return (String::from(filename.trim_end_matches(".txt")), tokenize(contents));
+            }
+            Err(e) => {
+                println!("{}", e);
+                continue;
+            }
+        };
+    }
+}
+
+pub fn tokenize(input: String) -> Vec<Token> {
+    let mut result = vec![];
+
+    let mut index: i32 = 0;
+    let input_str = input.as_bytes();
+
+    while index < input.len() as i32 {
+        let c = input_str[index as usize] as char;
+        index = index + 1;
+
+        if c.is_ascii_whitespace() {
+            continue;
+        }
+
+        // parse keywords or identifiers
+        if c.is_alphabetic() {
+            result.push(parse_identifier_or_keyword(c, input_str, &mut index));
+            continue;
+        }
+
+        // recognize numbers
+        if c.is_numeric() || c == '.' {
+            result.push(parse_number(c, input_str, &mut index));
+            continue;
+        }
+
+        // recognize comments
+        if c == '#' {
+            parse_comments(input_str, &mut index);
+            continue;
+        }
+
+        // recognized character
+        result.push(Token::CHAR(c));
+    }
+
+    result.push(Token::EOF);
+    result
+}
+
+fn parse_identifier_or_keyword(c: char, input_str: &[u8], index: &mut i32) -> Token {
+    let mut result = String::new();
+    result.push(c);
+
+    while *index < input_str.len() as i32 {
+        let c = input_str[*index as usize] as char;
+        *index = *index + 1;
+
+        if c.is_alphanumeric() {
+            result.push(c);
+        } else {
+            *index = *index - 1;
+            break;
+        }
+    }
+
+    let def_token = String::from("def");
+    let extern_token = String::from("extern");
+    let if_token = String::from("if");
+    let then_token = String::from("then");
+    let else_token = String::from("else");
+
+
+    if result.eq(&def_token) {
+        return Token::DEF;
+    }
+
+    if result.eq(&extern_token) {
+        return Token::EXTERN;
+    }
+
+    if result.eq(&if_token) {
+        return Token::IF;
+    }
+
+    if result.eq(&then_token) {
+        return Token::THEN;
+    }
+
+    if result.eq(&else_token) {
+        return Token::ELSE;
+    }
+
+    Token::ID(result)
+}
+//
+fn parse_number(c: char, string_as_bytes: &[u8], index: &mut i32) -> Token {
+    let mut result = String::new();
+    result.push(c);
+
+    while *index < string_as_bytes.len() as i32 {
+        let c = string_as_bytes[*index as usize] as char;
+        *index = *index + 1;
+
+        if c.is_numeric() || c == '.' {
+            result.push(c);
+        } else {
+            *index = *index - 1;
+            break;
+        }
+    }
+
+    let value = result.parse::<f64>().expect("Invalid number.");
+    Token::NUMBER(value)
+}
+//
+fn parse_comments(string_as_bytes: &[u8], pos: &mut i32) {
+    while *pos < string_as_bytes.len() as i32 {
+        let c = string_as_bytes[*pos as usize] as char;
+        *pos = *pos + 1;
+
+        if c == '\n' || c == '\r' {
+            break;
+        }
+    }
+}

src/lib.rs

+//use std::io::{Read, empty};
+
+extern crate mu;
+use self::mu::ast::types::*;
+
+#[macro_use]
+pub mod ir_macros;
+
+pub mod lexer;
+pub mod parser;
+pub mod generator;
+pub mod ast;
+
+pub mod mu_generator;

src/main.rs

+extern crate mu;
+use self::mu::ast::types::*;
+
+#[macro_use]
+pub mod ir_macros;
+
+pub mod lexer;
+pub mod parser;
+pub mod generator;
+pub mod ast;
+
+pub mod mu_generator;
+
+fn main() {
+    const USE_LLVM : bool = false;
+
+    // process chars into tokens
+    let (filename, tokens) = lexer::read_file();
+
+    if !tokens.is_empty() {
+        println!("Tokens: {:?}", tokens);
+    }
+    // parse tokens into tree
+    let nodes = parser::parse_program(tokens);
+
+    if USE_LLVM {
+        unsafe {
+            generator::code_gen(filename, nodes);
+        }
+    } else {
+        unsafe {
+            mu_generator::code_gen(filename, nodes);
+        }
+    }
+}

src/mu_generator.rs

+extern crate llvm_sys as llvm;
+use crate::ast::*;
+use crate::parser::*;
+use mu::vm::api;
+use mu::vm::api::api_c::*;
+use std::os::raw::{c_char, c_void};
+use std::ptr::null;
+
+extern crate mu;
+use mu::ast::inst::*;
+use mu::ast::ir::*;
+use mu::ast::op::*;
+use mu::ast::types::*;
+use mu::linkutils::*;
+use mu::runtime::*;
+use mu::utils::LinkedHashMap;
+use mu::vm::*;
+
+extern crate mu_aot_llvm;
+
+extern crate libloading as ll;
+
+use std::sync::Arc;
+pub type P<T> = Arc<T>;
+
+#[macro_use]
+use crate::ir_macros;
+use self::mu::linkutils;
+use std::collections::{HashMap, HashSet};
+use std::ops::Add;
+use std::path::PathBuf;
+
+pub struct Mu_VM {
+    pub vm: VM,