To protect your data, the CISO officer has suggested users to enable GitLab 2FA as soon as possible.

inst_sel.rs 201 KB
Newer Older
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
// Copyright 2017 The Australian National University
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// 
//     http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

15
16
#![warn(unused_imports)]
#![warn(unreachable_code)]
17

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
use ast::ir::*;
use ast::ptr::*;
use ast::inst::*;
use ast::op;
use ast::op::*;
use ast::types::*;
use vm::VM;
use runtime::mm;
use runtime::mm::objectmodel::OBJECT_HEADER_SIZE;

use runtime::ValueLocation;
use runtime::thread;
use runtime::entrypoints;
use runtime::entrypoints::RuntimeEntrypoint;

use compiler::CompilerPass;

use compiler::backend::PROLOGUE_BLOCK_NAME;
36
use compiler::backend::EPILOGUE_BLOCK_NAME;
37
38
39
40
41
42

use compiler::backend::aarch64::*;
use compiler::machine_code::CompiledFunction;
use compiler::frame::Frame;

use std::collections::HashMap;
43
44
use std::collections::LinkedList;

45
46
47
48
49
50
51
52
53
54
55
56
use std::any::Any;

const INLINE_FASTPATH : bool = false;

pub struct InstructionSelection {
    name: &'static str,
    backend: Box<CodeGenerator>,

    current_fv_id: MuID,
    current_callsite_id: usize,
    current_frame: Option<Frame>,
    current_block: Option<MuName>,
57
    current_block_in_ir: Option<MuName>,
58
    current_func_start: Option<ValueLocation>,
59
60
61
62
63
64

    // A list of all callsites, with the corresponding exception block (if there is one)

    // Technically this is a map in that each Key is unique, but we will never try and add duplicate
    // keys, or look things up, so a list of pairs is faster than a Map.
    current_callsites: LinkedList<(MuName, MuID)>,
65
    // key: block id, val: block location
66
    current_exn_blocks: HashMap<MuID, MuName>,
67
    current_xr_value: Option<P<Value>>, // A temporary that holds to saved XR value (if needed)
68
69
70
71
    current_constants: HashMap<MuID, P<Value>>,
    current_constants_locs: HashMap<MuID, P<Value>>
}

72
// TODO: Move all functions that are in here that don't need access to 'self' (or only call functions that don't need access to self (even if called on self)) to Mod.rs
73
74
75
76
77
78
79
80
81
82
83
impl <'a> InstructionSelection {
    #[cfg(feature = "aot")]
    pub fn new() -> InstructionSelection {
        InstructionSelection {
            name: "Instruction Selection (x64)",
            backend: Box::new(ASMCodeGen::new()),

            current_fv_id: 0,
            current_callsite_id: 0,
            current_frame: None,
            current_block: None,
84
85
86
87
88
89
            current_block_in_ir: None,  // it is possible the block is newly created in instruction selection
                                        // but sometimes we want to know its control flow
                                        // so we need to track what block it is from the IR

                                        // FIXME: ideally we should not create new blocks in instruction selection
                                        // see Issue #6
90
            current_func_start: None,
91
            current_callsites: LinkedList::new(),
92
            current_exn_blocks: HashMap::new(),
93
            current_xr_value: None,
94
95
96
            current_constants: HashMap::new(),
            current_constants_locs: HashMap::new()
        }
97

98
99
100
101
102
103
104
105
106
107
    }

    #[cfg(feature = "jit")]
    pub fn new() -> InstructionSelection {
        unimplemented!()
    }

    // in this pass, we assume that
    // * we do not need to backup/restore caller-saved registers
    // if any of these assumption breaks, we will need to re-emit the code
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
108
    fn instruction_select(&mut self, node: &'a TreeNode, f_content: &FunctionContent, f_context: &mut FunctionContext, vm: &VM) {
109
110
111
112
113
        trace!("instsel on node#{} {}", node.id(), node);

        match node.v {
            TreeNode_::Instruction(ref inst) => {
                match inst.v {
114
                    // TODO: Optimise if cond is a flag from a binary operation?
115
                    Instruction_::Branch2 { cond, ref true_dest, ref false_dest, .. } => {
116
117
                        trace!("instsel on BRANCH2");
                        let (fallthrough_dest, branch_dest, branch_if_true) = {
118
119
120
121
                            let cur_block = f_content.get_block_by_name(self.current_block_in_ir.as_ref().unwrap().clone());
                            let next_block_in_trace = cur_block.control_flow.get_hottest_succ().unwrap();

                            if next_block_in_trace == true_dest.target {
122
123
124
125
126
127
                                (true_dest, false_dest, false)
                            } else {
                                (false_dest, true_dest, true)
                            }
                        };

128
                        let ref ops = inst.ops;
129
130
131
132
133
134
135
136

                        self.process_dest(&ops, fallthrough_dest, f_content, f_context, vm);
                        self.process_dest(&ops, branch_dest, f_content, f_context, vm);

                        let branch_target = f_content.get_block(branch_dest.target).name().unwrap();

                        let ref cond = ops[cond];

137
                        if self.match_cmp_res(cond) {
138
                            trace!("emit cmp_res-branch2");
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
                            // Emit a CBNZ for 128-bit comparisons that are not symmetric
                            let use_cbnz = self.is_int128_asym_cmp(cond);
                            let tmp_cond =
                                if use_cbnz { Some(make_temporary(f_context, UINT1_TYPE.clone(), vm)) }
                                else { None };
                            let cond_box =
                                if use_cbnz { Some(Box::new(tmp_cond.as_ref().unwrap().clone())) }
                                else { None };

                            let mut cmpop = self.emit_cmp_res(cond, cond_box, f_content, f_context, vm);

                            if use_cbnz {
                                if !branch_if_true {
                                    self.backend.emit_cbz(tmp_cond.as_ref().unwrap(), branch_target);
                                } else {
                                    self.backend.emit_cbnz(tmp_cond.as_ref().unwrap(), branch_target);
                                }
156

157
                            } else {
158
159
160
                                if !branch_if_true {
                                    cmpop = cmpop.invert();
                                }
161

162
163
164
165
166
167
168
169
170
171
172
173
                                let cond = get_condition_codes(cmpop);

                                if cmpop == op::CmpOp::FFALSE {
                                    ; // Do nothing
                                } else if cmpop == op::CmpOp::FTRUE {
                                    self.backend.emit_b(branch_target);
                                } else {
                                    self.backend.emit_b_cond(cond[0], branch_target.clone());

                                    if cond.len() == 2 {
                                        self.backend.emit_b_cond(cond[1], branch_target);
                                    }
174
175
                                }
                            }
176
                        } else {
177
                            let cond_reg = self.emit_ireg(cond, f_content, f_context, vm);
178

179
180
181
182
                            if branch_if_true {
                                self.backend.emit_tbnz(&cond_reg, 0, branch_target.clone());
                            } else {
                                self.backend.emit_tbz(&cond_reg, 0, branch_target.clone());
183
                            }
184
                        };
185
186
187

                        // it is possible that the fallthrough block is scheduled somewhere else
                        // we need to explicitly jump to it
188
                        self.finish_block();
189
190
191
192
193
                        let fallthrough_temp_block = format!("{}_{}_branch_fallthrough", self.current_fv_id, node.id());
                        self.start_block(fallthrough_temp_block, &vec![]);

                        let fallthrough_target = f_content.get_block(fallthrough_dest.target).name().unwrap();
                        self.backend.emit_b(fallthrough_target);
194
195
196
197
198
199
                    },

                    Instruction_::Select { cond, true_val, false_val } => {
                        use ast::op::CmpOp::*;

                        trace!("instsel on SELECT");
200
                        let ref ops = inst.ops;
201
202
203
204
205

                        let ref cond = ops[cond];
                        let ref true_val = ops[true_val];
                        let ref false_val = ops[false_val];

206
                        let tmp_res = self.get_result_value(node, 0);
207
208
209
210

                        // moving integers/pointers
                        // generate compare
                        let cmpop = if self.match_cmp_res(cond) {
211
                            self.emit_cmp_res(cond, None, f_content, f_context, vm)
212
                        } else if self.match_ireg(cond) {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
213
                            let tmp_cond = self.emit_ireg(cond, f_content, f_context, vm);
214
215
216
217
218
219
                            self.backend.emit_cmp_imm(&tmp_cond, 0, false);
                            NE
                        } else {
                            panic!("expected ireg, found {}", cond)
                        };

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
220
221
                        let tmp_true = self.emit_reg(true_val, f_content, f_context, vm);
                        let tmp_false = self.emit_reg(false_val, f_content, f_context, vm);
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249

                        let cond = get_condition_codes(cmpop);

                        if self.match_ireg(true_val) {
                            if cmpop == FFALSE {
                                self.backend.emit_mov(&tmp_res, &tmp_false);
                            } else if cmpop == FTRUE {
                                self.backend.emit_mov(&tmp_res, &tmp_true);
                            } else {
                                self.backend.emit_csel(&tmp_res, &tmp_true, &tmp_false, cond[0]);

                                if cond.len() == 2 {
                                    self.backend.emit_csel(&tmp_res, &tmp_true, &tmp_res, cond[1]);
                                }
                            }
                        } else if self.match_fpreg(true_val) {
                            if cmpop == FFALSE {
                                self.backend.emit_fmov(&tmp_res, &tmp_false);
                            } else if cmpop == FTRUE {
                                self.backend.emit_fmov(&tmp_res, &tmp_true);
                            } else {
                                self.backend.emit_fcsel(&tmp_res, &tmp_true, &tmp_false, cond[0]);

                                if cond.len() == 2 {
                                    self.backend.emit_fcsel(&tmp_res, &tmp_true, &tmp_res, cond[1]);
                                }
                            }
                        } else {
250
                            // moving vectors
251
252
253
254
255
256
257
258
                            unimplemented!()
                        }
                    },

                    Instruction_::CmpOp(op, op1, op2) => {
                        use ast::op::CmpOp::*;

                        trace!("instsel on CMPOP");
259
                        let ref ops = inst.ops;
260
261
262
                        let ref op1 = ops[op1];
                        let ref op2 = ops[op2];

263
                        let tmp_res = self.get_result_value(node, 0);
264
265
266
267

                        debug_assert!(tmp_res.ty.get_int_length().is_some());
                        debug_assert!(tmp_res.ty.get_int_length().unwrap() == 1);

268
                        let cmpop = self.emit_cmp_res_op(op, Some(Box::new(tmp_res.clone())), &op1, &op2, f_content, f_context, vm);
269
270
                        let cond = get_condition_codes(cmpop);

271
272
273
274
275
276
277
278
                        // emit_cmp_res_op will set tmp_res for 128-bit assymettric comparisons
                        if !self.is_int128_asym_cmp(node) {
                            if cmpop == FFALSE {
                                emit_mov_u64(self.backend.as_mut(), &tmp_res, 0);
                            } else if cmpop == FTRUE {
                                emit_mov_u64(self.backend.as_mut(), &tmp_res, 1);
                            } else {
                                self.backend.emit_cset(&tmp_res, cond[0]);
279

280
281
282
283
284
                                // Note: some compariosns can't be computed based on a single aarch64 flag
                                // insted they are computed as a condition OR NOT another condition.
                                if cond.len() == 2 {
                                    self.backend.emit_csinc(&tmp_res, &tmp_res, &WZR, invert_condition_code(cond[1]));
                                }
285
286
287
288
289
290
                            }
                        }
                    }

                    Instruction_::Branch1(ref dest) => {
                        trace!("instsel on BRANCH1");
291
                        let ref ops = inst.ops;
292
293
294
295
296
297
298
299
300
301
302
303

                        self.process_dest(&ops, dest, f_content, f_context, vm);

                        let target = f_content.get_block(dest.target).name().unwrap();

                        trace!("emit branch1");
                        // jmp
                        self.backend.emit_b(target);
                    },

                    Instruction_::Switch { cond, ref default, ref branches } => {
                        trace!("instsel on SWITCH");
304
                        let ref ops = inst.ops;
305
306
307
308

                        let ref cond = ops[cond];

                        if self.match_ireg(cond) {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
309
                            let tmp_cond = self.emit_ireg(cond, f_content, f_context, vm);
310
                            emit_zext(self.backend.as_mut(), &tmp_cond);
311
312
313
314
315
316
317
318
319
320
321
322

                            // emit each branch
                            for &(case_op_index, ref case_dest) in branches {
                                let ref case_op = ops[case_op_index];

                                // process dest
                                self.process_dest(&ops, case_dest, f_content, f_context, vm);

                                let target = f_content.get_block(case_dest.target).name().unwrap();

                                let mut imm_val = 0 as u64;
                                // Is one of the arguments a valid immediate?
323
324
                                let emit_imm = if match_node_int_imm(&case_op) {
                                    imm_val = node_imm_to_u64(&case_op);
325
326
327
328
329
330
331
332
333
334
                                    is_valid_arithmetic_imm(imm_val)
                                } else {
                                    false
                                };

                                if emit_imm {
                                    let imm_shift = imm_val > 4096;
                                    let imm_op2 = if imm_shift { imm_val >> 12 } else { imm_val };
                                    self.backend.emit_cmp_imm(&tmp_cond, imm_op2 as u16, imm_shift);
                                } else {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
335
                                    let tmp_case_op = self.emit_ireg(case_op, f_content, f_context, vm);
336
                                    emit_zext(self.backend.as_mut(), &tmp_case_op);
337
338
339
340
341
                                    self.backend.emit_cmp(&tmp_cond, &tmp_case_op);
                                }

                                self.backend.emit_b_cond("EQ", target);

342
                                self.finish_block();
343
                                self.start_block(format!("{}_switch_not_met_case_{}", node.id(), case_op_index), &vec![]);
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
                            }

                            // emit default
                            self.process_dest(&ops, default, f_content, f_context, vm);

                            let default_target = f_content.get_block(default.target).name().unwrap();
                            self.backend.emit_b(default_target);
                        } else {
                            panic!("expecting cond in switch to be ireg: {}", cond);
                        }
                    }

                    Instruction_::ExprCall { ref data, is_abort } => {
                        trace!("instsel on EXPRCALL");

                        if is_abort {
                            unimplemented!()
                        }

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
363
                        self.emit_mu_call(
364
365
366
367
368
369
370
371
372
373
                            inst, // inst: &Instruction,
                            data, // calldata: &CallData,
                            None, // resumption: Option<&ResumptionData>,
                            node, // cur_node: &TreeNode, 
                            f_content, f_context, vm);
                    },

                    Instruction_::Call { ref data, ref resume } => {
                        trace!("instsel on CALL");

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
374
                        self.emit_mu_call(
375
376
377
378
379
380
381
382
383
384
385
386
387
388
                            inst,
                            data,
                            Some(resume),
                            node,
                            f_content, f_context, vm);
                    },

                    Instruction_::ExprCCall { ref data, is_abort } => {
                        trace!("instsel on EXPRCCALL");

                        if is_abort {
                            unimplemented!()
                        }

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
389
                        self.emit_c_call_ir(inst, data, None, node, f_content, f_context, vm);
390
391
392
393
394
                    }

                    Instruction_::CCall { ref data, ref resume } => {
                        trace!("instsel on CCALL");

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
395
                        self.emit_c_call_ir(inst, data, Some(resume), node, f_content, f_context, vm);
396
397
                    }

398
                    Instruction_::Return(ref vals) => {
399
400
                        trace!("instsel on RETURN");

401
                        // prepare return regs
402
                        let ref ops = inst.ops;
403
                        // TODO: Are vals in the same order as the return types in the functions signature?
404

405
                        let ret_tys = vals.iter().map(|i| node_type(&ops[*i])).collect();
406
                        let ret_type = self.combine_return_types(&ret_tys);
407

408
                        let n = ret_tys.len(); // number of return values
409
410
                        let xr_value = self.current_xr_value.as_ref().unwrap().clone();

411
412
                        if n == 0 {
                            // Do nothing
413
                        } else if n == 1 {
414
                            let ret_loc = self.compute_return_locations(&ret_type, &xr_value, &vm);
415
416
417
418
419
420
421
422
423
424
425
                            let ret_val = self.emit_node_value(&ops[vals[0]], f_content, f_context, vm);

                            if is_machine_reg(&ret_loc) && is_int_ex_reg(&ret_val) {
                                let (val_l, val_h) = split_int128(&ret_val, f_context, vm);
                                let ret_loc_h = get_register_from_id(ret_loc.id() + 2);
                                // nothing special needs to be done
                                emit_move_value_to_value(self.backend.as_mut(), &ret_loc, &val_l, f_context, vm);
                                emit_move_value_to_value(self.backend.as_mut(), &ret_loc_h, &val_h, f_context, vm);
                            } else {
                                emit_move_value_to_value(self.backend.as_mut(), &ret_loc, &ret_val, f_context, vm);
                            }
426
                        } else {
427
                            let ret_loc = self.compute_return_locations(&ret_type, &xr_value, &vm);
428

429
                            let mut i = 0;
430
                            for ret_index in vals {
431
432
433
434
435
436
437
438
                                let ret_val = self.emit_node_value(&ops[*ret_index], f_content, f_context, vm);
                                let ref ty = ret_val.ty;
                                let offset = self.get_field_offset(&ret_type, i, &vm);

                                match ty.v {
                                    MuType_::Vector(_, _) | MuType_::Tagref64 => unimplemented!(),
                                    MuType_::Void => panic!("Unexpected void"),
                                    MuType_::Struct(_) | MuType_::Array(_, _) => unimplemented!(),
439
440
                                    MuType_::Hybrid(_) => panic!("Can't return a hybrid"),
                                    // Integral, pointer or floating point type
441
442
443
444
445
446
                                    _ => self.insert_bytes(&ret_loc, &ret_val, offset as i64, f_context, vm),
                                }

                                i += 1;
                            }
                        }
447

448
                        self.backend.emit_b(EPILOGUE_BLOCK_NAME.to_string());
449
450
451
452
                    },

                    Instruction_::BinOp(op, op1, op2) => {
                        trace!("instsel on BINOP");
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
453
                        self.emit_binop(node, inst, op, BinOpStatus { flag_n: false, flag_z: false, flag_c: false, flag_v: false }, op1, op2, f_content, f_context, vm);
454
455
456
457
                    },

                    Instruction_::BinOpWithStatus(op, status, op1, op2) => {
                        trace!("instsel on BINOP_STATUS");
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
458
                        self.emit_binop(node, inst, op, status, op1, op2, f_content, f_context, vm);
459
460
461
462
463
                    }

                    Instruction_::ConvOp { operation, ref from_ty, ref to_ty, operand } => {
                        trace!("instsel on CONVOP");

464
                        let ref ops = inst.ops;
465
466
467

                        let ref op = ops[operand];

468
                        let tmp_res = self.get_result_value(node, 0);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
469
                        let tmp_op = self.emit_reg(op, f_content, f_context, vm);
470
471
472
473
474
475

                        let from_ty_size = get_bit_size(&from_ty, vm);
                        let to_ty_size = get_bit_size(&to_ty, vm);

                        match operation {
                            op::ConvOp::TRUNC => {
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
                                // src is in one register
                                if self.match_ireg(op) {
                                    self.backend.emit_mov(&tmp_res, &cast_value(&tmp_op, &to_ty));
                                } else if self.match_ireg_ex(op) {
                                    // Move the lower word
                                    if from_ty_size != to_ty_size {
                                        let (op_l, _) = self.emit_ireg_ex(op, f_content, f_context, vm);
                                        self.backend.emit_mov(&tmp_res, &cast_value(&op_l, &to_ty));
                                    } else {
                                        self.emit_move_node_to_value(&tmp_res, op, f_content, f_context, vm);
                                    }
                                } else {
                                    panic!("unexpected op (expect ireg): {}", op);
                                }

491
                            },
492

493
494
                            op::ConvOp::ZEXT => {
                                if from_ty_size != to_ty_size {
495
496
497
498
499
500
501
502
503
504
505
506
                                    if to_ty_size <= 64 {
                                        self.backend.emit_ubfx(&tmp_res, &cast_value(&tmp_op, &to_ty), 0, from_ty_size as u8);
                                    } else if to_ty_size == 128 {
                                        let (res_l, res_h) = split_int128(&tmp_res, f_context, vm);

                                        // res_l = ZEXT src
                                        self.backend.emit_ubfx(&res_l, &cast_value(&tmp_op, &UINT64_TYPE), 0, from_ty_size as u8);
                                        self.backend.emit_mov(&res_h, &XZR); // res_h = 0

                                    } else {
                                        panic!("unexpected int length {}", to_ty_size);
                                    }
507
                                } else {
508
509
                                    // Trivial, just do a move
                                    emit_move_value_to_value(self.backend.as_mut(), &tmp_res, &tmp_op, f_context, vm);
510
511
                                }
                            },
512
513


514
515
                            op::ConvOp::SEXT => {
                                if from_ty_size != to_ty_size {
516
517
518
519
520
521
522
523
524
525
526
527
528
                                    if to_ty_size <= 64 {
                                        self.backend.emit_sbfx(&tmp_res, &cast_value(&tmp_op, &to_ty), 0, from_ty_size as u8);
                                    } else if to_ty_size == 128 {
                                        let (res_l, res_h) = split_int128(&tmp_res, f_context, vm);

                                        // res_l = SEXT src
                                        self.backend.emit_sbfx(&res_l, &cast_value(&tmp_op, &UINT64_TYPE), 0, from_ty_size as u8);
                                        self.backend.emit_asr_imm(&res_h, &tmp_op, 63); // res_h = ASHR src, 63

                                    } else {
                                        panic!("unexpected int length {}", to_ty_size);
                                    }

529
                                } else {
530
531
                                    // Trivial, just do a move
                                    emit_move_value_to_value(self.backend.as_mut(), &tmp_res, &tmp_op, f_context, vm);
532
533
534
535
536
537
538
539
                                }
                            },
                            op::ConvOp::REFCAST | op::ConvOp::PTRCAST => {
                                // just a mov (and hopefully reg alloc will coalesce it)
                                self.backend.emit_mov(&tmp_res, &tmp_op);
                            },

                            op::ConvOp::UITOFP => {
540
541
542
543
544
                                if from_ty_size == 128 {
                                    if to_ty_size == 64 {
                                        self.emit_runtime_entry(&entrypoints::UITOFP_U128_DOUBLE,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
545
                                            Some(node), f_context, vm);
546
547
548
549
                                    } else {
                                        self.emit_runtime_entry(&entrypoints::UITOFP_U128_FLOAT,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
550
                                            Some(node), f_context, vm);
551
552
553
554
                                    }
                                } else {
                                    self.backend.emit_ucvtf(&tmp_res, &tmp_op);
                                }
555
556
557
                            },

                            op::ConvOp::SITOFP => {
558
559
560
561
562
                                if from_ty_size == 128 {
                                    if to_ty_size == 64 {
                                        self.emit_runtime_entry(&entrypoints::SITOFP_I128_DOUBLE,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
563
                                            Some(node), f_context, vm);
564
565
566
567
                                    } else {
                                        self.emit_runtime_entry(&entrypoints::SITOFP_I128_FLOAT,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
568
                                            Some(node), f_context, vm);
569
570
571
                                    }                                } else {
                                    self.backend.emit_scvtf(&tmp_res, &tmp_op);
                                }
572
573
574
                            },

                            op::ConvOp::FPTOUI => {
575
576
577
578
579
                                if to_ty_size == 128 {
                                    if from_ty_size == 64 {
                                        self.emit_runtime_entry(&entrypoints::FPTOUI_DOUBLE_U128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
580
                                            Some(node), f_context, vm);
581
582
583
584
                                    } else {
                                        self.emit_runtime_entry(&entrypoints::FPTOUI_FLOAT_U128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
585
                                            Some(node), f_context, vm);
586
587
588
589
                                    }
                                } else {
                                    self.backend.emit_fcvtzu(&tmp_res, &tmp_op);
                                }
590
591
592
                            },

                            op::ConvOp::FPTOSI => {
593
594
595
596
597
                                if to_ty_size == 128 {
                                    if from_ty_size == 64 {
                                        self.emit_runtime_entry(&entrypoints::FPTOSI_DOUBLE_I128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
598
                                            Some(node), f_context, vm);
599
600
601
602
                                    } else {
                                        self.emit_runtime_entry(&entrypoints::FPTOSI_FLOAT_I128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
603
                                            Some(node), f_context, vm);
604
605
606
607
                                    }
                                } else {
                                    self.backend.emit_fcvtzs(&tmp_res, &tmp_op);
                                }
608
609
610
611
612
613
614
615
616
617
618
                            },

                            op::ConvOp::BITCAST => {
                                self.backend.emit_fmov(&tmp_res, &tmp_op);
                            },
                            op::ConvOp::FPTRUNC | op::ConvOp::FPEXT => {
                                self.backend.emit_fcvt(&tmp_res, &tmp_op);
                            },
                        }
                    }

619
                    Instruction_::Load { order, mem_loc, .. } => {
620
                        trace!("instsel on LOAD");
621
                        let ref ops = inst.ops;
622
623
                        let ref loc_op = ops[mem_loc];

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
624
                        let resolved_loc = self.emit_node_addr_to_value(loc_op, f_content, f_context, vm);
625
                        let res = self.get_result_value(node, 0);
626

627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
                        if self.match_ireg(node) || self.match_fpreg(node) {
                            // Whether to use a load acquire
                            let use_acquire = match order {
                                MemoryOrder::Relaxed | MemoryOrder::NotAtomic => false,
                                MemoryOrder::Consume | MemoryOrder::Acquire | MemoryOrder::SeqCst => true,
                                _ => panic!("didnt expect order {:?} with load inst", order)
                            };


                            if use_acquire {
                                // Can only have a base for a LDAR
                                let temp_loc = emit_mem_base(self.backend.as_mut(), &resolved_loc, f_context, vm);
                                match res.ty.v {
                                    // Have to load a temporary GPR first
                                    MuType_::Float => {
                                        let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                        self.backend.emit_ldar(&temp, &temp_loc);
                                        self.backend.emit_fmov(&res, &temp);
                                    }
                                    MuType_::Double => {
                                        let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                        self.backend.emit_ldar(&temp, &temp_loc);
                                        self.backend.emit_fmov(&res, &temp);
                                    }
                                    // Can load the register directly
                                    _ => self.backend.emit_ldar(&res, &temp_loc)
                                };
                            } else {
655
                                let temp_loc = emit_mem(self.backend.as_mut(), &resolved_loc, get_type_alignment(&res.ty, vm), f_context, vm);
656
657
658
659
660
661
662
                                self.backend.emit_ldr(&res, &temp_loc, false);
                            }
                        } else if self.match_ireg_ex(node) {
                            let (res_l, res_h) = split_int128(&res, f_context, vm);

                            match order {
                                MemoryOrder::NotAtomic => {
663
                                    let temp_loc = emit_mem(self.backend.as_mut(), &resolved_loc, get_type_alignment(&res.ty, vm), f_context, vm);
664
                                    self.backend.emit_ldp(&res_l, &res_h, &temp_loc);
665
                                }
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685

                                // Aarch64 dosn't have a load acquire pair instruction
                                // So instead we have to write a loop using load/store exclusive pairs
                                _ => {
                                    // Whether to use a load exclusive acquire
                                    let use_acquire = match order {
                                        MemoryOrder::Relaxed  => false,
                                        MemoryOrder::Consume | MemoryOrder::Acquire | MemoryOrder::SeqCst => true,
                                        _ => panic!("didnt expect order {:?} with atomic load inst", order)
                                    };
                                    // Whether to use a store exclusive release
                                    let use_release = match order {
                                        MemoryOrder::Relaxed | MemoryOrder::Consume | MemoryOrder::Acquire  => false,
                                        MemoryOrder::SeqCst => true,
                                        _ => panic!("didnt expect order {:?} with atomic load inst", order)
                                    };

                                    // Exclusive loads/stores, only supports a base address
                                    let temp_loc = emit_mem_base(self.backend.as_mut(), &resolved_loc, f_context, vm);

686
                                    self.finish_block();
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711

                                    let blk_load_start = format!("{}_load_start", node.id());

                                    // load_start:
                                    self.start_block(blk_load_start.clone(), &vec![temp_loc.clone()]);


                                    // Load the value:
                                    if use_acquire {
                                        self.backend.emit_ldaxp(&res_l, &res_h, &temp_loc);
                                    } else {
                                        self.backend.emit_ldxp(&res_l, &res_h, &temp_loc);
                                    }

                                    let success = make_temporary(f_context, UINT1_TYPE.clone(), vm);

                                    // Store the value we just read back to memory
                                    if use_release {
                                        self.backend.emit_stlxp(&temp_loc, &success, &res_l, &res_h);
                                    } else {
                                        self.backend.emit_stxp(&temp_loc, &success, &res_l, &res_h);
                                    }

                                    // If the store failed, then branch back to 'load_start:'
                                    self.backend.emit_cbnz(&success, blk_load_start.clone())
712
                                }
713
                            }
714
                        } else {
715
                            unimplemented!();
716
717
718
                        }
                    }

719
                    Instruction_::Store { order, mem_loc, value, .. } => {
720
                        trace!("instsel on STORE");
721
                        let ref ops = inst.ops;
722
723
724
                        let ref loc_op = ops[mem_loc];
                        let ref val_op = ops[value];

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
725
                        let resolved_loc = self.emit_node_addr_to_value(loc_op, f_content, f_context, vm);
726

727
728
729
730
731
732
733
                        if self.match_ireg(val_op) || self.match_fpreg(val_op) {
                            // Whether to use a store release or not
                            let use_release = match order {
                                MemoryOrder::Relaxed | MemoryOrder::NotAtomic => false,
                                MemoryOrder::Release | MemoryOrder::SeqCst => true,
                                _ => panic!("didnt expect order {:?} with load inst", order)
                            };
734

735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
                            let val = self.emit_reg(val_op, f_content, f_context, vm);

                            if use_release {
                                // Can only have a base for a STLR
                                let temp_loc = emit_mem_base(self.backend.as_mut(), &resolved_loc, f_context, vm);

                                match val.ty.v {
                                    // Have to store a temporary GPR
                                    MuType_::Float => {
                                        let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                        self.backend.emit_fmov(&temp, &val);
                                              self.backend.emit_stlr(&temp_loc, &temp);
                                    }
                                    MuType_::Double => {
                                        let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                        self.backend.emit_fmov(&temp, &val);
                                        self.backend.emit_stlr(&temp_loc, &temp);
                                    }
                                    // Can load the register directly
                                    _ => self.backend.emit_stlr(&temp_loc, &val)
                                };
                            } else {
757
                                let temp_loc = emit_mem(self.backend.as_mut(), &resolved_loc, get_type_alignment(&val.ty, vm), f_context, vm);
758
759
760
761
762
763
764
                                self.backend.emit_str(&temp_loc, &val);
                            }
                        } else if self.match_ireg_ex(val_op) {
                            let (val_l, val_h) = self.emit_ireg_ex(val_op, f_content, f_context, vm);

                            match order {
                                MemoryOrder::NotAtomic => {
765
                                    let temp_loc = emit_mem(self.backend.as_mut(), &resolved_loc, 16, f_context, vm);
766
                                    self.backend.emit_stp(&temp_loc, &val_l, &val_h);
767
                                }
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787

                                // Aarch64 dosn't have a store release pair instruction
                                // So instead we have to write a loop using load/store exclusive pairs
                                _ => {
                                    // Whether to use a load exclusive acquire
                                    let use_acquire = match order {
                                        MemoryOrder::Relaxed | MemoryOrder::Release => false,
                                        MemoryOrder::SeqCst => true,
                                        _ => panic!("didnt expect order {:?} with atomic store inst", order)
                                    };
                                    // Whether to use a store exclusive release
                                    let use_release = match order {
                                        MemoryOrder::Relaxed  => false,
                                        MemoryOrder::Release | MemoryOrder::SeqCst => true,
                                        _ => panic!("didnt expect order {:?} with atomic store inst", order)
                                    };

                                    // Exclusive loads/stores, only supports a base address
                                    let temp_loc = emit_mem_base(self.backend.as_mut(), &resolved_loc, f_context, vm);

788
                                    self.finish_block();
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812

                                    let blk_store_start = format!("{}_store_start", node.id());

                                    // store_start:
                                    self.start_block(blk_store_start.clone(), &vec![temp_loc.clone()]);

                                    let success = make_temporary(f_context, UINT1_TYPE.clone(), vm);
                                    let discard_reg = cast_value(&success, &UINT64_TYPE);
                                    // Load a value (discard it)
                                    if use_acquire {
                                        self.backend.emit_ldaxp(&XZR, &discard_reg, &temp_loc);
                                    } else {
                                        self.backend.emit_ldxp(&XZR, &discard_reg, &temp_loc);
                                    }

                                    // Store the value
                                    if use_release {
                                        self.backend.emit_stlxp(&temp_loc, &success, &val_l, &val_h);
                                    } else {
                                        self.backend.emit_stxp(&temp_loc, &success, &val_l, &val_h);
                                    }

                                    // If the store failed, then branch back to 'store_start:'
                                    self.backend.emit_cbnz(&success, blk_store_start.clone())
813
                                }
814
                            }
815
                        } else {
816
                            unimplemented!();
817
                        }
818

819
820
                    }

821
                    Instruction_::CmpXchg{is_weak, success_order, fail_order, mem_loc, expected_value, desired_value, ..} => {
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
                        // Note: this uses the same operations as GCC (for the C++ atomic cmpxchg)
                        // Clang is slightly different and ignores the 'fail_order'
                        let use_acquire = match fail_order {
                            MemoryOrder::Acquire | MemoryOrder::SeqCst => true,
                            MemoryOrder::Relaxed => match success_order {
                                MemoryOrder::Acquire | MemoryOrder::AcqRel | MemoryOrder::SeqCst => true,
                                MemoryOrder::Relaxed | MemoryOrder::Release => false,
                                _ => panic!("didnt expect success order {:?} for cmpxchg", success_order)
                            },
                            _ => panic!("didnt expect fail order {:?} for cmpxchg", fail_order)
                        };
                        let use_release = match fail_order {
                            MemoryOrder::Acquire => match success_order {
                                MemoryOrder::Relaxed | MemoryOrder::Release | MemoryOrder::AcqRel | MemoryOrder::SeqCst => true,
                                MemoryOrder::Acquire => false,
                                _ => panic!("didnt expect success order {:?} for cmpxchg", success_order)
                            },
                            MemoryOrder::SeqCst => true,
                            MemoryOrder::Relaxed => match success_order {
                                MemoryOrder::Release | MemoryOrder::AcqRel | MemoryOrder::SeqCst => true,
                                MemoryOrder::Relaxed | MemoryOrder::Acquire => false,
                                _ => panic!("didnt expect success order {:?} for cmpxchg", success_order)
                            },
                            _ => panic!("didnt expect fail order {:?} for cmpxchg", fail_order)
                        };


849
                        let ref ops = inst.ops;
850
851
852
853
854
855
856
857
858
859
860
861
                        let loc = self.emit_node_addr_to_value(&ops[mem_loc], f_content, f_context, vm);
                        let expected = self.emit_reg(&ops[expected_value], f_content, f_context, vm);
                        let desired = self.emit_reg(&ops[desired_value], f_content, f_context, vm);

                        let res_value = self.get_result_value(node, 0);
                        let res_success = self.get_result_value(node, 1);


                        let blk_cmpxchg_start = format!("{}_cmpxchg_start", node.id());
                        let blk_cmpxchg_failed = format!("{}_cmpxchg_failed", node.id());
                        let blk_cmpxchg_succeded = format!("{}_cmpxchg_succeded", node.id());

862
                        self.finish_block();
863

864
                        // cmpxchg_start:
865
866
867
                        self.start_block(blk_cmpxchg_start.clone(), &vec![loc.clone(),expected.clone(), desired.clone()]);

                        if use_acquire {
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
                            match res_value.ty.v {
                                // Have to load a temporary GPR first
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_ldaxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
                                }
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_ldaxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
                                }
                                // Can load the register directly
                                _ => self.backend.emit_ldaxr(&res_value, &loc)
                            };
883
                        } else {
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
                            match res_value.ty.v {
                                // Have to load a temporary GPR first
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_ldxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
                                }
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_ldxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
                                }
                                // Can load the register directly
                                _ => self.backend.emit_ldxr(&res_value, &loc)
                            };
899
900
                        }

901
                        if is_int_reg(&expected) {
902
903
904
905
906
907
908
                            self.backend.emit_cmp(&res_value, &expected);
                        } else {
                            self.backend.emit_fcmp(&res_value, &expected);
                        }
                        self.backend.emit_b_cond("NE", blk_cmpxchg_failed.clone());

                        if use_release {
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
                            match desired.ty.v {
                                // Have to store a temporary GPR
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stlxr(&loc, &res_success, &temp);
                                }
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stlxr(&loc, &res_success, &temp);
                                }
                                // Can load the register directly
                                _ => self.backend.emit_stlxr(&loc, &res_success, &desired)
                            };
924
                        } else {
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
                            match desired.ty.v {
                                // Have to store a temporary GPR
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stxr(&loc, &res_success, &temp);
                                }
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stxr(&loc, &res_success, &temp);
                                }
                                // Can load the register directly
                                _ => self.backend.emit_stxr(&loc, &res_success, &desired)
                            };
940
941
942
943
944
945
946
947
948
                        }

                        if !is_weak {
                            // Store failed, try again
                            self.backend.emit_cbnz(&res_success, blk_cmpxchg_start.clone());
                        }

                        self.backend.emit_b(blk_cmpxchg_succeded.clone());

949
                        self.finish_block();
950

951
                        // cmpxchg_failed:
952
953
954
955
956
957
                        self.start_block(blk_cmpxchg_failed.clone(), &vec![res_success.clone(), res_value.clone()]);

                        self.backend.emit_clrex();
                        // Set res_success to 1 (the same value STXR/STLXR uses to indicate failure)
                        self.backend.emit_mov_imm(&res_success, 1);

958
                        self.finish_block();
959

960
                        // cmpxchg_succeded:
961
962
963
964
                        self.start_block(blk_cmpxchg_succeded.clone(), &vec![res_success.clone(), res_value.clone()]);
                        // this NOT is needed as STXR/STLXR returns sucess as '0', wheras the Mu spec says it should be 1
                        self.backend.emit_eor_imm(&res_success, &res_success, 1);
                    }
965
966
967
968
969
970
                    Instruction_::GetIRef(_)
                    | Instruction_::GetFieldIRef { .. }
                    | Instruction_::GetElementIRef{..}
                    | Instruction_::GetVarPartIRef { .. }
                    | Instruction_::ShiftIRef { .. } => {
                        trace!("instsel on GET/FIELD/VARPARTIREF, SHIFTIREF");
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
971
                        let mem_addr = self.emit_get_mem_from_inst(node, f_content, f_context, vm);
972
                        let tmp_res = self.get_result_value(node, 0);
973
                        emit_calculate_address(self.backend.as_mut(), &tmp_res, &mem_addr, f_context, vm);
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
                    }

                    Instruction_::Fence(order) => {
                        trace!("instsel on FENCE");

                        // Whether to emit a load fence or a normal one
                        let use_load = match order {
                            MemoryOrder::Release | MemoryOrder::SeqCst | MemoryOrder::AcqRel => false,
                            MemoryOrder::Acquire => true,
                            _ => panic!("didnt expect order {:?} with load inst", order)
                        };

                        if use_load {
                            // Data Memory Barrirer for Inner Shariable Domain (for Load accesses only)
                            self.backend.emit_dmb("ISHLD");
                        } else {
                            // Data Memory Barrirer for Inner Shariable Domain
                            self.backend.emit_dmb("ISH");
                        }
                    }
994

995
996
997
                    // TODO: Implement this similar to a return (where theres a common exit block)
                    // and change SWAP_BACK_TO_NATIV_STACK and swap_to_mu_stack so they don't handle the callee saved registers
                    // (this instruction should then guarentee that they are restored (in the same way a Return does)
998
999
1000
1001
1002
                    Instruction_::ThreadExit => {
                        trace!("instsel on THREADEXIT");
                        // emit a call to swap_back_to_native_stack(sp_loc: Address)

                        // get thread local and add offset to get sp_loc
1003
                        let tl = self.emit_get_threadlocal(f_context, vm);
1004
1005
                        self.backend.emit_add_imm(&tl, &tl, *thread::NATIVE_SP_LOC_OFFSET as u16, false);

1006
                        self.emit_runtime_entry(&entrypoints::SWAP_BACK_TO_NATIVE_STACK, vec![tl.clone()], None, Some(node), f_context, vm);
1007
1008
                    }

1009

1010
1011
1012
                    Instruction_::CommonInst_GetThreadLocal => {
                        trace!("instsel on GETTHREADLOCAL");
                        // get thread local
1013
                        let tl = self.emit_get_threadlocal(f_context, vm);
1014

1015
                        let tmp_res = self.get_result_value(node, 0);
1016
1017

                        // load [tl + USER_TLS_OFFSET] -> tmp_res
1018
                        emit_load_base_offset(self.backend.as_mut(), &tmp_res, &tl, *thread::USER_TLS_OFFSET as i64, f_context, vm);
1019
1020
                    }

1021

1022
1023
                    Instruction_::CommonInst_SetThreadLocal(op) => {
                        trace!("instsel on SETTHREADLOCAL");
1024
                        let ref ops = inst.ops;
1025
1026
1027
1028
                        let ref op = ops[op];

                        debug_assert!(self.match_ireg(op));

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1029
                        let tmp_op = self.emit_ireg(op, f_content, f_context, vm);
1030
1031

                        // get thread local
1032
                        let tl = self.emit_get_threadlocal(f_context, vm);
1033
1034

                        // store tmp_op -> [tl + USER_TLS_OFFSTE]
1035
                        emit_store_base_offset(self.backend.as_mut(), &tl, *thread::USER_TLS_OFFSET as i64, &tmp_op, f_context, vm);
1036
1037
1038
1039
1040
1041
                    }

                    Instruction_::CommonInst_Pin(op) => {
                        trace!("instsel on PIN");
                        if !mm::GC_MOVES_OBJECT {
                            // non-moving GC: pin is a nop (move from op to result)
1042
                            let ref ops = inst.ops;
1043
1044
                            let ref op = ops[op];

1045
                            let tmp_res = self.get_result_value(node, 0);
1046

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1047
                            self.emit_move_node_to_value(&tmp_res, op, f_content, f_context, vm);
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
                        } else {
                            unimplemented!()
                        }
                    }

                    Instruction_::CommonInst_Unpin(_) => {
                        trace!("instsel on UNPIN");
                        if !mm::GC_MOVES_OBJECT {
                            // do nothing
                        } else {
                            unimplemented!()
                        }
                    }

                    Instruction_::Move(op) => {
                        trace!("instsel on MOVE (internal IR)");
1064
                        let ref ops = inst.ops;
1065
1066
                        let ref op = ops[op];

1067
                        let tmp_res = self.get_result_value(node, 0);
1068

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1069
                        self.emit_move_node_to_value(&tmp_res, op, f_content, f_context, vm);
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
                    }

                    Instruction_::New(ref ty) => {
                        trace!("instsel on NEW");
                        if cfg!(debug_assertions) {
                            match ty.v {
                                MuType_::Hybrid(_) => panic!("cannot use NEW for hybrid, use NEWHYBRID instead"),
                                _ => {}
                            }
                        }

                        let ty_info = vm.get_backend_type_info(ty.id());
                        let size = ty_info.size;
                        let ty_align = ty_info.alignment;

1085
                        let const_size = make_value_int_const(size as u64, vm);
1086

1087
                        let tmp_allocator = self.emit_get_allocator(f_context, vm);
1088
                        let tmp_res = self.emit_alloc_sequence(tmp_allocator.clone(), const_size, ty_align, node, f_context, vm);
1089
1090

                        // ASM: call muentry_init_object(%allocator, %tmp_res, %encode)
1091
                        let encode = make_value_int_const(mm::get_gc_type_encode(ty_info.gc_type.id), vm);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1092
                        self.emit_runtime_entry(
1093
1094
1095
                            &entrypoints::INIT_OBJ,
                            vec![tmp_allocator.clone(), tmp_res.clone(), encode],
                            None,
1096
                            Some(node), f_context, vm
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
                        );
                    }

                    Instruction_::NewHybrid(ref ty, var_len) => {
                        trace!("instsel on NEWHYBRID");
                        if cfg!(debug_assertions) {
                            match ty.v {
                                MuType_::Hybrid(_) => {},
                                _ => panic!("NEWHYBRID is only for allocating hybrid types, use NEW for others")
                            }
                        }

                        let ty_info = vm.get_backend_type_info(ty.id());
                        let ty_align = ty_info.alignment;
                        let fix_part_size = ty_info.size;
                        let var_ty_size = match ty.v {
                            MuType_::Hybrid(ref name) => {
                                let map_lock = HYBRID_TAG_MAP.read().unwrap();
                                let hybrid_ty_ = map_lock.get(name).unwrap();
                                let var_ty = hybrid_ty_.get_var_ty();

                                vm.get_backend_type_info(var_ty.id()).size
                            },
                            _ => panic!("only expect HYBRID type here")
                        };

                        // actual size = fix_part_size + var_ty_size * len
                        let (actual_size, length) = {
1125
                            let ref ops = inst.ops;
1126
1127
                            let ref var_len = ops[var_len];

1128
1129
                            if match_node_int_imm(var_len) {
                                let var_len = node_imm_to_u64(var_len);
1130
1131
                                let actual_size = fix_part_size + var_ty_size * (var_len as usize);
                                (
1132
1133
                                    make_value_int_const(actual_size as u64, vm),
                                    make_value_int_const(var_len as u64, vm)
1134
1135
                                )
                            } else {
1136
                                let tmp_actual_size = make_temporary(f_context, UINT64_TYPE.clone(), vm);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1137
                                let tmp_var_len = self.emit_ireg(var_len, f_content, f_context, vm);
1138
1139

                                // tmp_actual_size = tmp_var_len*var_ty_size
1140
                                emit_mul_u64(self.backend.as_mut(), &tmp_actual_size, &tmp_var_len, f_context, vm, var_ty_size as u64);
1141
                                // tmp_actual_size = tmp_var_len*var_ty_size + fix_part_size
1142
                                emit_add_u64(self.backend.as_mut(), &tmp_actual_size, &tmp_actual_size, f_context, vm, fix_part_size as u64);
1143
1144
1145
1146
                                (tmp_actual_size, tmp_var_len)
                            }
                        };

1147
                        let tmp_allocator = self.emit_get_allocator(f_context, vm);
1148
                        let tmp_res = self.emit_alloc_sequence(tmp_allocator.clone(), actual_size, ty_align, node, f_context, vm);
1149
1150

                        // ASM: call muentry_init_object(%allocator, %tmp_res, %encode)
1151
                        let encode = make_value_int_const(mm::get_gc_type_encode(ty_info.gc_type.id), vm);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1152
                        self.emit_runtime_entry(
1153
1154
1155
                            &entrypoints::INIT_HYBRID,
                            vec![tmp_allocator.clone(), tmp_res.clone(), encode, length],
                            None,
1156
                            Some(node), f_context, vm
1157
1158
1159
1160
1161
1162
                        );
                    }

                    // Runtime Entry
                    Instruction_::Throw(op_index) => {
                        trace!("instsel on THROW");
1163
                        let ref ops = inst.ops;
1164
1165
                        let ref exception_obj = ops[op_index];

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1166
                        self.emit_runtime_entry(
1167
1168
1169
                            &entrypoints::THROW_EXCEPTION,
                            vec![exception_obj.clone_value()],
                            None,
1170
                            Some(node), f_context, vm);
1171
1172
1173
1174
1175
                    }

                    // Runtime Entry
                    Instruction_::PrintHex(index) => {
                        trace!("instsel on PRINTHEX");
1176
                        let ref ops = inst.ops;
1177
1178
                        let ref op = ops[index];

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1179
                        self.emit_runtime_entry(
1180
1181
1182
                            &entrypoints::PRINT_HEX,
                            vec![op.clone_value()],
                            None,
1183
                            Some(node), f_context, vm
1184
1185
1186
                        );
                    }

1187
1188
1189
1190
                    // Runtime Entry
                    Instruction_::SetRetval(index) => {
                        trace!("instsel on SETRETVAL");

1191
                        let ref ops = inst.ops;
1192
1193
1194
1195
1196
1197
                        let ref op  = ops[index];

                        self.emit_runtime_entry(
                            &entrypoints::SET_RETVAL,
                            vec![op.clone_value()],
                            None,
1198
                            Some(node), f_context, vm