To protect your data, the CISO officer has suggested users to enable GitLab 2FA as soon as possible.

inst_sel.rs 219 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
// Copyright 2017 The Australian National University
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// 
//     http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

15
16
#![warn(unused_imports)]
#![warn(unreachable_code)]
17

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
use ast::ir::*;
use ast::ptr::*;
use ast::inst::*;
use ast::op;
use ast::op::*;
use ast::types::*;
use vm::VM;
use runtime::mm;
use runtime::mm::objectmodel::OBJECT_HEADER_SIZE;

use runtime::ValueLocation;
use runtime::thread;
use runtime::entrypoints;
use runtime::entrypoints::RuntimeEntrypoint;

use compiler::CompilerPass;

use compiler::backend::PROLOGUE_BLOCK_NAME;
36
use compiler::backend::EPILOGUE_BLOCK_NAME;
37
38

use compiler::backend::aarch64::*;
39
use compiler::backend::make_block_name;
40
41
42
43
use compiler::machine_code::CompiledFunction;
use compiler::frame::Frame;

use std::collections::HashMap;
44
use std::collections::LinkedList;
45
use std::mem;
46
47
48
49
50
51
52
53
54
use std::any::Any;

const INLINE_FASTPATH : bool = false;

pub struct InstructionSelection {
    name: &'static str,
    backend: Box<CodeGenerator>,

    current_fv_id: MuID,
55
    current_fv_name: MuName,
56
57
58
    current_callsite_id: usize,
    current_frame: Option<Frame>,
    current_block: Option<MuName>,
59
    current_block_in_ir: Option<MuName>,
60
    current_func_start: Option<ValueLocation>,
61
62
63
64
65

    // A list of all callsites, with the corresponding exception block (if there is one)

    // Technically this is a map in that each Key is unique, but we will never try and add duplicate
    // keys, or look things up, so a list of pairs is faster than a Map.
66
    current_callsites: LinkedList<(MuName, MuID, usize)>,
67
    // key: block id, val: block location
68
    current_exn_blocks: HashMap<MuID, MuName>,
69
    current_xr_value: Option<P<Value>>, // A temporary that holds to saved XR value (if needed)
70
71
72
73
    current_constants: HashMap<MuID, P<Value>>,
    current_constants_locs: HashMap<MuID, P<Value>>
}

74
// TODO: Move all functions that are in here that don't need access to 'self' (or only call functions that don't need access to self (even if called on self)) to Mod.rs
75
76
77
78
79
80
81
82
impl <'a> InstructionSelection {
    #[cfg(feature = "aot")]
    pub fn new() -> InstructionSelection {
        InstructionSelection {
            name: "Instruction Selection (x64)",
            backend: Box::new(ASMCodeGen::new()),

            current_fv_id: 0,
83
            current_fv_name: String::new(),
84
85
86
            current_callsite_id: 0,
            current_frame: None,
            current_block: None,
87
88
89
90
91
92
            current_block_in_ir: None,  // it is possible the block is newly created in instruction selection
                                        // but sometimes we want to know its control flow
                                        // so we need to track what block it is from the IR

                                        // FIXME: ideally we should not create new blocks in instruction selection
                                        // see Issue #6
93
            current_func_start: None,
94
            current_callsites: LinkedList::new(),
95
            current_exn_blocks: HashMap::new(),
96
            current_xr_value: None,
97
98
99
            current_constants: HashMap::new(),
            current_constants_locs: HashMap::new()
        }
100

101
102
103
104
105
106
107
108
109
110
    }

    #[cfg(feature = "jit")]
    pub fn new() -> InstructionSelection {
        unimplemented!()
    }

    // in this pass, we assume that
    // * we do not need to backup/restore caller-saved registers
    // if any of these assumption breaks, we will need to re-emit the code
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
111
    fn instruction_select(&mut self, node: &'a TreeNode, f_content: &FunctionContent, f_context: &mut FunctionContext, vm: &VM) {
112
113
114
115
116
        trace!("instsel on node#{} {}", node.id(), node);

        match node.v {
            TreeNode_::Instruction(ref inst) => {
                match inst.v {
117
                    // TODO: Optimise if cond is a flag from a binary operation?
118
                    Instruction_::Branch2 { cond, ref true_dest, ref false_dest, .. } => {
119
120
                        trace!("instsel on BRANCH2");
                        let (fallthrough_dest, branch_dest, branch_if_true) = {
121
122
123
124
                            let cur_block = f_content.get_block_by_name(self.current_block_in_ir.as_ref().unwrap().clone());
                            let next_block_in_trace = cur_block.control_flow.get_hottest_succ().unwrap();

                            if next_block_in_trace == true_dest.target {
125
126
127
128
129
130
                                (true_dest, false_dest, false)
                            } else {
                                (false_dest, true_dest, true)
                            }
                        };

131
                        let ref ops = inst.ops;
132
133
134
135

                        self.process_dest(&ops, fallthrough_dest, f_content, f_context, vm);
                        self.process_dest(&ops, branch_dest, f_content, f_context, vm);

136
                        let branch_target = f_content.get_block(branch_dest.target).name();
137
138
139

                        let ref cond = ops[cond];

140
                        if self.match_cmp_res(cond) {
141
                            trace!("emit cmp_res-branch2");
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
                            // Emit a CBNZ for 128-bit comparisons that are not symmetric
                            let use_cbnz = self.is_int128_asym_cmp(cond);
                            let tmp_cond =
                                if use_cbnz { Some(make_temporary(f_context, UINT1_TYPE.clone(), vm)) }
                                else { None };
                            let cond_box =
                                if use_cbnz { Some(Box::new(tmp_cond.as_ref().unwrap().clone())) }
                                else { None };

                            let mut cmpop = self.emit_cmp_res(cond, cond_box, f_content, f_context, vm);

                            if use_cbnz {
                                if !branch_if_true {
                                    self.backend.emit_cbz(tmp_cond.as_ref().unwrap(), branch_target);
                                } else {
                                    self.backend.emit_cbnz(tmp_cond.as_ref().unwrap(), branch_target);
                                }
159

160
                            } else {
161
162
163
                                if !branch_if_true {
                                    cmpop = cmpop.invert();
                                }
164

165
166
167
168
169
170
171
172
173
174
175
176
                                let cond = get_condition_codes(cmpop);

                                if cmpop == op::CmpOp::FFALSE {
                                    ; // Do nothing
                                } else if cmpop == op::CmpOp::FTRUE {
                                    self.backend.emit_b(branch_target);
                                } else {
                                    self.backend.emit_b_cond(cond[0], branch_target.clone());

                                    if cond.len() == 2 {
                                        self.backend.emit_b_cond(cond[1], branch_target);
                                    }
177
178
                                }
                            }
179
                        } else {
180
                            let cond_reg = self.emit_ireg(cond, f_content, f_context, vm);
181

182
183
184
185
                            if branch_if_true {
                                self.backend.emit_tbnz(&cond_reg, 0, branch_target.clone());
                            } else {
                                self.backend.emit_tbz(&cond_reg, 0, branch_target.clone());
186
                            }
187
                        };
188
189
190

                        // it is possible that the fallthrough block is scheduled somewhere else
                        // we need to explicitly jump to it
191
                        self.finish_block();
192
                        let fallthrough_temp_block = make_block_name(node, "branch_fallthrough", );
193
194
                        self.start_block(fallthrough_temp_block, &vec![]);

195
                        let fallthrough_target = f_content.get_block(fallthrough_dest.target).name();
196
                        self.backend.emit_b(fallthrough_target);
197
198
199
200
201
202
                    },

                    Instruction_::Select { cond, true_val, false_val } => {
                        use ast::op::CmpOp::*;

                        trace!("instsel on SELECT");
203
                        let ref ops = inst.ops;
204
205
206
207
208

                        let ref cond = ops[cond];
                        let ref true_val = ops[true_val];
                        let ref false_val = ops[false_val];

209
                        let tmp_res = self.get_result_value(node, 0);
210
211
212
213

                        // moving integers/pointers
                        // generate compare
                        let cmpop = if self.match_cmp_res(cond) {
214
                            self.emit_cmp_res(cond, None, f_content, f_context, vm)
215
                        } else if self.match_ireg(cond) {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
216
                            let tmp_cond = self.emit_ireg(cond, f_content, f_context, vm);
217
218
219
220
221
222
                            self.backend.emit_cmp_imm(&tmp_cond, 0, false);
                            NE
                        } else {
                            panic!("expected ireg, found {}", cond)
                        };

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
223
224
                        let tmp_true = self.emit_reg(true_val, f_content, f_context, vm);
                        let tmp_false = self.emit_reg(false_val, f_content, f_context, vm);
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252

                        let cond = get_condition_codes(cmpop);

                        if self.match_ireg(true_val) {
                            if cmpop == FFALSE {
                                self.backend.emit_mov(&tmp_res, &tmp_false);
                            } else if cmpop == FTRUE {
                                self.backend.emit_mov(&tmp_res, &tmp_true);
                            } else {
                                self.backend.emit_csel(&tmp_res, &tmp_true, &tmp_false, cond[0]);

                                if cond.len() == 2 {
                                    self.backend.emit_csel(&tmp_res, &tmp_true, &tmp_res, cond[1]);
                                }
                            }
                        } else if self.match_fpreg(true_val) {
                            if cmpop == FFALSE {
                                self.backend.emit_fmov(&tmp_res, &tmp_false);
                            } else if cmpop == FTRUE {
                                self.backend.emit_fmov(&tmp_res, &tmp_true);
                            } else {
                                self.backend.emit_fcsel(&tmp_res, &tmp_true, &tmp_false, cond[0]);

                                if cond.len() == 2 {
                                    self.backend.emit_fcsel(&tmp_res, &tmp_true, &tmp_res, cond[1]);
                                }
                            }
                        } else {
253
                            // moving vectors
254
255
256
257
258
259
260
261
                            unimplemented!()
                        }
                    },

                    Instruction_::CmpOp(op, op1, op2) => {
                        use ast::op::CmpOp::*;

                        trace!("instsel on CMPOP");
262
                        let ref ops = inst.ops;
263
264
265
                        let ref op1 = ops[op1];
                        let ref op2 = ops[op2];

266
                        let tmp_res = self.get_result_value(node, 0);
267
268
269
270

                        debug_assert!(tmp_res.ty.get_int_length().is_some());
                        debug_assert!(tmp_res.ty.get_int_length().unwrap() == 1);

271
                        let cmpop = self.emit_cmp_res_op(op, Some(Box::new(tmp_res.clone())), &op1, &op2, f_content, f_context, vm);
272
273
                        let cond = get_condition_codes(cmpop);

274
275
276
277
278
279
280
281
                        // emit_cmp_res_op will set tmp_res for 128-bit assymettric comparisons
                        if !self.is_int128_asym_cmp(node) {
                            if cmpop == FFALSE {
                                emit_mov_u64(self.backend.as_mut(), &tmp_res, 0);
                            } else if cmpop == FTRUE {
                                emit_mov_u64(self.backend.as_mut(), &tmp_res, 1);
                            } else {
                                self.backend.emit_cset(&tmp_res, cond[0]);
282

283
284
285
286
287
                                // Note: some compariosns can't be computed based on a single aarch64 flag
                                // insted they are computed as a condition OR NOT another condition.
                                if cond.len() == 2 {
                                    self.backend.emit_csinc(&tmp_res, &tmp_res, &WZR, invert_condition_code(cond[1]));
                                }
288
289
290
291
292
293
                            }
                        }
                    }

                    Instruction_::Branch1(ref dest) => {
                        trace!("instsel on BRANCH1");
294
                        let ref ops = inst.ops;
295
296
297

                        self.process_dest(&ops, dest, f_content, f_context, vm);

298
                        let target = f_content.get_block(dest.target).name();
299
300
301
302
303
304
305
306

                        trace!("emit branch1");
                        // jmp
                        self.backend.emit_b(target);
                    },

                    Instruction_::Switch { cond, ref default, ref branches } => {
                        trace!("instsel on SWITCH");
307
                        let ref ops = inst.ops;
308
309
310
311

                        let ref cond = ops[cond];

                        if self.match_ireg(cond) {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
312
                            let tmp_cond = self.emit_ireg(cond, f_content, f_context, vm);
313
                            emit_zext(self.backend.as_mut(), &tmp_cond);
314
315
316
317
318
319
320
321

                            // emit each branch
                            for &(case_op_index, ref case_dest) in branches {
                                let ref case_op = ops[case_op_index];

                                // process dest
                                self.process_dest(&ops, case_dest, f_content, f_context, vm);

322
                                let target = f_content.get_block(case_dest.target).name();
323
324
325

                                let mut imm_val = 0 as u64;
                                // Is one of the arguments a valid immediate?
326
327
                                let emit_imm = if match_node_int_imm(&case_op) {
                                    imm_val = node_imm_to_u64(&case_op);
328
329
330
331
332
333
334
335
336
337
                                    is_valid_arithmetic_imm(imm_val)
                                } else {
                                    false
                                };

                                if emit_imm {
                                    let imm_shift = imm_val > 4096;
                                    let imm_op2 = if imm_shift { imm_val >> 12 } else { imm_val };
                                    self.backend.emit_cmp_imm(&tmp_cond, imm_op2 as u16, imm_shift);
                                } else {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
338
                                    let tmp_case_op = self.emit_ireg(case_op, f_content, f_context, vm);
339
                                    emit_zext(self.backend.as_mut(), &tmp_case_op);
340
341
342
343
344
                                    self.backend.emit_cmp(&tmp_cond, &tmp_case_op);
                                }

                                self.backend.emit_b_cond("EQ", target);

345
                                self.finish_block();
346
                                self.start_block(make_block_name(node, format!("switch_not_met_case_{}", case_op_index).as_str()), &vec![]);
347
348
349
350
351
                            }

                            // emit default
                            self.process_dest(&ops, default, f_content, f_context, vm);

352
                            let default_target = f_content.get_block(default.target).name();
353
354
355
356
357
358
359
360
361
362
363
364
365
                            self.backend.emit_b(default_target);
                        } else {
                            panic!("expecting cond in switch to be ireg: {}", cond);
                        }
                    }

                    Instruction_::ExprCall { ref data, is_abort } => {
                        trace!("instsel on EXPRCALL");

                        if is_abort {
                            unimplemented!()
                        }

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
366
                        self.emit_mu_call(
367
368
369
370
371
372
373
374
375
376
                            inst, // inst: &Instruction,
                            data, // calldata: &CallData,
                            None, // resumption: Option<&ResumptionData>,
                            node, // cur_node: &TreeNode, 
                            f_content, f_context, vm);
                    },

                    Instruction_::Call { ref data, ref resume } => {
                        trace!("instsel on CALL");

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
377
                        self.emit_mu_call(
378
379
380
381
382
383
384
385
386
387
388
389
390
391
                            inst,
                            data,
                            Some(resume),
                            node,
                            f_content, f_context, vm);
                    },

                    Instruction_::ExprCCall { ref data, is_abort } => {
                        trace!("instsel on EXPRCCALL");

                        if is_abort {
                            unimplemented!()
                        }

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
392
                        self.emit_c_call_ir(inst, data, None, node, f_content, f_context, vm);
393
394
395
396
397
                    }

                    Instruction_::CCall { ref data, ref resume } => {
                        trace!("instsel on CCALL");

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
398
                        self.emit_c_call_ir(inst, data, Some(resume), node, f_content, f_context, vm);
399
400
                    }

401
                    Instruction_::Return(ref vals) => {
402
403
                        trace!("instsel on RETURN");

404
                        // prepare return regs
405
                        let ref ops = inst.ops;
406
                        // TODO: Are vals in the same order as the return types in the functions signature?
407

408
                        let ret_tys = vals.iter().map(|i| node_type(&ops[*i])).collect();
409
                        let ret_type = self.combine_return_types(&ret_tys);
410

411
                        let n = ret_tys.len(); // number of return values
412
413
                        let xr_value = self.current_xr_value.as_ref().unwrap().clone();

414
415
                        if n == 0 {
                            // Do nothing
416
                        } else if n == 1 {
417
                            let ret_loc = self.compute_return_locations(&ret_type, &xr_value, &vm);
418
419
420
421
422
423
424
425
426
427
428
                            let ret_val = self.emit_node_value(&ops[vals[0]], f_content, f_context, vm);

                            if is_machine_reg(&ret_loc) && is_int_ex_reg(&ret_val) {
                                let (val_l, val_h) = split_int128(&ret_val, f_context, vm);
                                let ret_loc_h = get_register_from_id(ret_loc.id() + 2);
                                // nothing special needs to be done
                                emit_move_value_to_value(self.backend.as_mut(), &ret_loc, &val_l, f_context, vm);
                                emit_move_value_to_value(self.backend.as_mut(), &ret_loc_h, &val_h, f_context, vm);
                            } else {
                                emit_move_value_to_value(self.backend.as_mut(), &ret_loc, &ret_val, f_context, vm);
                            }
429
                        } else {
430
                            let ret_loc = self.compute_return_locations(&ret_type, &xr_value, &vm);
431

432
                            let mut i = 0;
433
                            for ret_index in vals {
434
435
436
437
438
                                let ret_val = self.emit_node_value(&ops[*ret_index], f_content, f_context, vm);
                                let ref ty = ret_val.ty;
                                let offset = self.get_field_offset(&ret_type, i, &vm);

                                match ty.v {
439
                                    MuType_::Vector(_, _) => unimplemented!(),
440
441
                                    MuType_::Void => panic!("Unexpected void"),
                                    MuType_::Struct(_) | MuType_::Array(_, _) => unimplemented!(),
442
443
                                    MuType_::Hybrid(_) => panic!("Can't return a hybrid"),
                                    // Integral, pointer or floating point type
444
445
446
447
448
449
                                    _ => self.insert_bytes(&ret_loc, &ret_val, offset as i64, f_context, vm),
                                }

                                i += 1;
                            }
                        }
450

451
                        self.backend.emit_b(EPILOGUE_BLOCK_NAME.to_string());
452
453
454
455
                    },

                    Instruction_::BinOp(op, op1, op2) => {
                        trace!("instsel on BINOP");
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
456
                        self.emit_binop(node, inst, op, BinOpStatus { flag_n: false, flag_z: false, flag_c: false, flag_v: false }, op1, op2, f_content, f_context, vm);
457
458
459
460
                    },

                    Instruction_::BinOpWithStatus(op, status, op1, op2) => {
                        trace!("instsel on BINOP_STATUS");
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
461
                        self.emit_binop(node, inst, op, status, op1, op2, f_content, f_context, vm);
462
463
464
465
466
                    }

                    Instruction_::ConvOp { operation, ref from_ty, ref to_ty, operand } => {
                        trace!("instsel on CONVOP");

467
                        let ref ops = inst.ops;
468
469
470

                        let ref op = ops[operand];

471
                        let tmp_res = self.get_result_value(node, 0);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
472
                        let tmp_op = self.emit_reg(op, f_content, f_context, vm);
473
474
475
476
477
478

                        let from_ty_size = get_bit_size(&from_ty, vm);
                        let to_ty_size = get_bit_size(&to_ty, vm);

                        match operation {
                            op::ConvOp::TRUNC => {
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
                                // src is in one register
                                if self.match_ireg(op) {
                                    self.backend.emit_mov(&tmp_res, &cast_value(&tmp_op, &to_ty));
                                } else if self.match_ireg_ex(op) {
                                    // Move the lower word
                                    if from_ty_size != to_ty_size {
                                        let (op_l, _) = self.emit_ireg_ex(op, f_content, f_context, vm);
                                        self.backend.emit_mov(&tmp_res, &cast_value(&op_l, &to_ty));
                                    } else {
                                        self.emit_move_node_to_value(&tmp_res, op, f_content, f_context, vm);
                                    }
                                } else {
                                    panic!("unexpected op (expect ireg): {}", op);
                                }

494
                            },
495

496
497
                            op::ConvOp::ZEXT => {
                                if from_ty_size != to_ty_size {
498
499
500
501
502
503
504
505
506
507
508
509
                                    if to_ty_size <= 64 {
                                        self.backend.emit_ubfx(&tmp_res, &cast_value(&tmp_op, &to_ty), 0, from_ty_size as u8);
                                    } else if to_ty_size == 128 {
                                        let (res_l, res_h) = split_int128(&tmp_res, f_context, vm);

                                        // res_l = ZEXT src
                                        self.backend.emit_ubfx(&res_l, &cast_value(&tmp_op, &UINT64_TYPE), 0, from_ty_size as u8);
                                        self.backend.emit_mov(&res_h, &XZR); // res_h = 0

                                    } else {
                                        panic!("unexpected int length {}", to_ty_size);
                                    }
510
                                } else {
511
512
                                    // Trivial, just do a move
                                    emit_move_value_to_value(self.backend.as_mut(), &tmp_res, &tmp_op, f_context, vm);
513
514
                                }
                            },
515
516


517
518
                            op::ConvOp::SEXT => {
                                if from_ty_size != to_ty_size {
519
520
521
522
523
524
525
526
527
528
529
530
531
                                    if to_ty_size <= 64 {
                                        self.backend.emit_sbfx(&tmp_res, &cast_value(&tmp_op, &to_ty), 0, from_ty_size as u8);
                                    } else if to_ty_size == 128 {
                                        let (res_l, res_h) = split_int128(&tmp_res, f_context, vm);

                                        // res_l = SEXT src
                                        self.backend.emit_sbfx(&res_l, &cast_value(&tmp_op, &UINT64_TYPE), 0, from_ty_size as u8);
                                        self.backend.emit_asr_imm(&res_h, &tmp_op, 63); // res_h = ASHR src, 63

                                    } else {
                                        panic!("unexpected int length {}", to_ty_size);
                                    }

532
                                } else {
533
534
                                    // Trivial, just do a move
                                    emit_move_value_to_value(self.backend.as_mut(), &tmp_res, &tmp_op, f_context, vm);
535
536
537
538
539
540
541
542
                                }
                            },
                            op::ConvOp::REFCAST | op::ConvOp::PTRCAST => {
                                // just a mov (and hopefully reg alloc will coalesce it)
                                self.backend.emit_mov(&tmp_res, &tmp_op);
                            },

                            op::ConvOp::UITOFP => {
543
544
545
546
547
                                if from_ty_size == 128 {
                                    if to_ty_size == 64 {
                                        self.emit_runtime_entry(&entrypoints::UITOFP_U128_DOUBLE,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
548
                                            Some(node), f_context, vm);
549
550
551
552
                                    } else {
                                        self.emit_runtime_entry(&entrypoints::UITOFP_U128_FLOAT,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
553
                                            Some(node), f_context, vm);
554
555
                                    }
                                } else {
556
                                    emit_zext(self.backend.as_mut(), &tmp_op);
557
558
                                    self.backend.emit_ucvtf(&tmp_res, &tmp_op);
                                }
559
560
561
                            },

                            op::ConvOp::SITOFP => {
562
563
564
565
566
                                if from_ty_size == 128 {
                                    if to_ty_size == 64 {
                                        self.emit_runtime_entry(&entrypoints::SITOFP_I128_DOUBLE,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
567
                                            Some(node), f_context, vm);
568
569
570
571
                                    } else {
                                        self.emit_runtime_entry(&entrypoints::SITOFP_I128_FLOAT,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
572
                                            Some(node), f_context, vm);
573
574
575
                                    }
                                } else {
                                    emit_sext(self.backend.as_mut(), &tmp_op);
576
577
                                    self.backend.emit_scvtf(&tmp_res, &tmp_op);
                                }
578
579
580
                            },

                            op::ConvOp::FPTOUI => {
581
582
583
584
585
                                if to_ty_size == 128 {
                                    if from_ty_size == 64 {
                                        self.emit_runtime_entry(&entrypoints::FPTOUI_DOUBLE_U128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
586
                                            Some(node), f_context, vm);
587
588
589
590
                                    } else {
                                        self.emit_runtime_entry(&entrypoints::FPTOUI_FLOAT_U128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
591
                                            Some(node), f_context, vm);
592
593
594
                                    }
                                } else {
                                    self.backend.emit_fcvtzu(&tmp_res, &tmp_op);
595
596
597
598
599
600
601
602
603
604
605

                                    // We have to emmit code to handle the case when the real result
                                    // overflows to_ty_size, but not to_ty_reg_size
                                    let to_ty_reg_size = check_op_len(&tmp_res.ty); // The size of the aarch64 register
                                    if to_ty_size != to_ty_reg_size {
                                        // Compare the bits of the result after the lower
                                        // to_ty_size bits
                                        self.backend.emit_tst_imm(&tmp_res, bits_ones(to_ty_reg_size-to_ty_size) << to_ty_size);

                                        // If the above condition is true, the an overflow occurred
                                        // So set tmp_res to !0 (i.e. all ones, the maximum value)
606
                                        self.backend.emit_csinv(&tmp_res, &tmp_res, &get_alias_for_length(XZR.id(), to_ty_size), "EQ");
607
                                    }
608
                                }
609
610
611
                            },

                            op::ConvOp::FPTOSI => {
612
613
                                if to_ty_size == 128 {
                                    if from_ty_size == 64 {
614
615
                                        self.emit_runtime_entry(&entrypoints::FPTOSI_DOUBLE_I128, vec![tmp_op.clone()],
                                                                Some(vec![tmp_res.clone()]), Some(node), f_context, vm);
616
                                    } else {
617
618
                                        self.emit_runtime_entry(&entrypoints::FPTOSI_FLOAT_I128, vec![tmp_op.clone()],
                                                                Some(vec![tmp_res.clone()]), Some(node), f_context, vm);
619
620
621
                                    }
                                } else {
                                    self.backend.emit_fcvtzs(&tmp_res, &tmp_op);
622
623
624
625
626
627
628
629

                                    // TODO This code is horrible and inefficient due to branches and duplication
                                    // is there a better way?

                                    // We have to emmit code to handle the case when the real result
                                    // overflows to_ty_size, but not to_ty_reg_size
                                    let to_ty_reg_size = check_op_len(&tmp_res.ty); // The size of the aarch64 register
                                    if to_ty_size != to_ty_reg_size {
630
631
632
                                        let blk_positive = make_block_name(node, "positive");
                                        let blk_negative = make_block_name(node, "negative");
                                        let blk_end      = make_block_name(node, "end");
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
                                        let tmp          = make_temporary(f_context, to_ty.clone(), vm);

                                        self.backend.emit_tbnz(&tmp_res, (to_ty_size - 1) as u8, blk_negative.clone());
                                        self.finish_block();

                                        self.start_block(blk_positive.clone(), &vec![]);
                                        {
                                            // check to see if the higher bits are the same as the
                                            // sign bit (which is 0), if their not there's an overflow
                                            self.backend.emit_tst_imm(&tmp_res, bits_ones(to_ty_reg_size - to_ty_size) << to_ty_size);
                                            self.backend.emit_mov_imm(&tmp, bits_ones(to_ty_size - 1));

                                            // if the above test fails (i.e. results in zero)
                                            // then set temp_res to tmp
                                            self.backend.emit_csel(&tmp_res, &tmp, &tmp_res, "EQ");

                                            self.backend.emit_b(blk_end.clone());
                                            self.finish_block();
                                        }
                                        self.start_block(blk_negative.clone(), &vec![]);
                                        {
                                            self.backend.emit_mvn(&tmp, &tmp_res);
                                            // check to see if the higher bits of temp are the same as the
                                            // sign bit (which is 1), if their not there's an overflow
                                            self.backend.emit_tst_imm(&tmp_res, bits_ones(to_ty_reg_size - to_ty_size) << to_ty_size);

                                            // Set just the sign bit (this is smallest representable signed number)
                                            self.backend.emit_mov_imm(&tmp, 1 << to_ty_size);

                                            // if the above test fails (i.e. results in zero), then set temp_res to tmp
                                            self.backend.emit_csel(&tmp_res, &tmp, &tmp_res, "EQ");
                                            self.finish_block();
                                        }
                                        self.start_block(blk_end.clone(), &vec![]);
                                    }
668
                                }
669
670
671
672
673
674
675
676
677
678
679
                            },

                            op::ConvOp::BITCAST => {
                                self.backend.emit_fmov(&tmp_res, &tmp_op);
                            },
                            op::ConvOp::FPTRUNC | op::ConvOp::FPEXT => {
                                self.backend.emit_fcvt(&tmp_res, &tmp_op);
                            },
                        }
                    }

680
                    Instruction_::Load { order, mem_loc, .. } => {
681
                        trace!("instsel on LOAD");
682
                        let ref ops = inst.ops;
683
684
                        let ref loc_op = ops[mem_loc];

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
685
                        let resolved_loc = self.emit_node_addr_to_value(loc_op, f_content, f_context, vm);
686
                        let res = self.get_result_value(node, 0);
687

688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
                        if self.match_ireg(node) || self.match_fpreg(node) {
                            // Whether to use a load acquire
                            let use_acquire = match order {
                                MemoryOrder::Relaxed | MemoryOrder::NotAtomic => false,
                                MemoryOrder::Consume | MemoryOrder::Acquire | MemoryOrder::SeqCst => true,
                                _ => panic!("didnt expect order {:?} with load inst", order)
                            };


                            if use_acquire {
                                // Can only have a base for a LDAR
                                let temp_loc = emit_mem_base(self.backend.as_mut(), &resolved_loc, f_context, vm);
                                match res.ty.v {
                                    // Have to load a temporary GPR first
                                    MuType_::Float => {
                                        let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                        self.backend.emit_ldar(&temp, &temp_loc);
                                        self.backend.emit_fmov(&res, &temp);
                                    }
                                    MuType_::Double => {
                                        let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                        self.backend.emit_ldar(&temp, &temp_loc);
                                        self.backend.emit_fmov(&res, &temp);
                                    }
                                    // Can load the register directly
                                    _ => self.backend.emit_ldar(&res, &temp_loc)
                                };
                            } else {
716
                                let temp_loc = emit_mem(self.backend.as_mut(), &resolved_loc, get_type_alignment(&res.ty, vm), f_context, vm);
717
718
719
720
721
722
723
                                self.backend.emit_ldr(&res, &temp_loc, false);
                            }
                        } else if self.match_ireg_ex(node) {
                            let (res_l, res_h) = split_int128(&res, f_context, vm);

                            match order {
                                MemoryOrder::NotAtomic => {
724
                                    let temp_loc = emit_mem(self.backend.as_mut(), &resolved_loc, get_type_alignment(&res.ty, vm), f_context, vm);
725
                                    self.backend.emit_ldp(&res_l, &res_h, &temp_loc);
726
                                }
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746

                                // Aarch64 dosn't have a load acquire pair instruction
                                // So instead we have to write a loop using load/store exclusive pairs
                                _ => {
                                    // Whether to use a load exclusive acquire
                                    let use_acquire = match order {
                                        MemoryOrder::Relaxed  => false,
                                        MemoryOrder::Consume | MemoryOrder::Acquire | MemoryOrder::SeqCst => true,
                                        _ => panic!("didnt expect order {:?} with atomic load inst", order)
                                    };
                                    // Whether to use a store exclusive release
                                    let use_release = match order {
                                        MemoryOrder::Relaxed | MemoryOrder::Consume | MemoryOrder::Acquire  => false,
                                        MemoryOrder::SeqCst => true,
                                        _ => panic!("didnt expect order {:?} with atomic load inst", order)
                                    };

                                    // Exclusive loads/stores, only supports a base address
                                    let temp_loc = emit_mem_base(self.backend.as_mut(), &resolved_loc, f_context, vm);

747
                                    self.finish_block();
748

749
                                    let blk_load_start = make_block_name(node, "load_start");
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772

                                    // load_start:
                                    self.start_block(blk_load_start.clone(), &vec![temp_loc.clone()]);


                                    // Load the value:
                                    if use_acquire {
                                        self.backend.emit_ldaxp(&res_l, &res_h, &temp_loc);
                                    } else {
                                        self.backend.emit_ldxp(&res_l, &res_h, &temp_loc);
                                    }

                                    let success = make_temporary(f_context, UINT1_TYPE.clone(), vm);

                                    // Store the value we just read back to memory
                                    if use_release {
                                        self.backend.emit_stlxp(&temp_loc, &success, &res_l, &res_h);
                                    } else {
                                        self.backend.emit_stxp(&temp_loc, &success, &res_l, &res_h);
                                    }

                                    // If the store failed, then branch back to 'load_start:'
                                    self.backend.emit_cbnz(&success, blk_load_start.clone())
773
                                }
774
                            }
775
                        } else {
776
                            unimplemented!();
777
778
779
                        }
                    }

780
                    Instruction_::Store { order, mem_loc, value, .. } => {
781
                        trace!("instsel on STORE");
782
                        let ref ops = inst.ops;
783
784
785
                        let ref loc_op = ops[mem_loc];
                        let ref val_op = ops[value];

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
786
                        let resolved_loc = self.emit_node_addr_to_value(loc_op, f_content, f_context, vm);
787

788
789
790
791
792
793
794
                        if self.match_ireg(val_op) || self.match_fpreg(val_op) {
                            // Whether to use a store release or not
                            let use_release = match order {
                                MemoryOrder::Relaxed | MemoryOrder::NotAtomic => false,
                                MemoryOrder::Release | MemoryOrder::SeqCst => true,
                                _ => panic!("didnt expect order {:?} with load inst", order)
                            };
795

796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
                            let val = self.emit_reg(val_op, f_content, f_context, vm);

                            if use_release {
                                // Can only have a base for a STLR
                                let temp_loc = emit_mem_base(self.backend.as_mut(), &resolved_loc, f_context, vm);

                                match val.ty.v {
                                    // Have to store a temporary GPR
                                    MuType_::Float => {
                                        let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                        self.backend.emit_fmov(&temp, &val);
                                              self.backend.emit_stlr(&temp_loc, &temp);
                                    }
                                    MuType_::Double => {
                                        let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                        self.backend.emit_fmov(&temp, &val);
                                        self.backend.emit_stlr(&temp_loc, &temp);
                                    }
                                    // Can load the register directly
                                    _ => self.backend.emit_stlr(&temp_loc, &val)
                                };
                            } else {
818
                                let temp_loc = emit_mem(self.backend.as_mut(), &resolved_loc, get_type_alignment(&val.ty, vm), f_context, vm);
819
820
821
822
823
824
825
                                self.backend.emit_str(&temp_loc, &val);
                            }
                        } else if self.match_ireg_ex(val_op) {
                            let (val_l, val_h) = self.emit_ireg_ex(val_op, f_content, f_context, vm);

                            match order {
                                MemoryOrder::NotAtomic => {
826
                                    let temp_loc = emit_mem(self.backend.as_mut(), &resolved_loc, 16, f_context, vm);
827
                                    self.backend.emit_stp(&temp_loc, &val_l, &val_h);
828
                                }
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848

                                // Aarch64 dosn't have a store release pair instruction
                                // So instead we have to write a loop using load/store exclusive pairs
                                _ => {
                                    // Whether to use a load exclusive acquire
                                    let use_acquire = match order {
                                        MemoryOrder::Relaxed | MemoryOrder::Release => false,
                                        MemoryOrder::SeqCst => true,
                                        _ => panic!("didnt expect order {:?} with atomic store inst", order)
                                    };
                                    // Whether to use a store exclusive release
                                    let use_release = match order {
                                        MemoryOrder::Relaxed  => false,
                                        MemoryOrder::Release | MemoryOrder::SeqCst => true,
                                        _ => panic!("didnt expect order {:?} with atomic store inst", order)
                                    };

                                    // Exclusive loads/stores, only supports a base address
                                    let temp_loc = emit_mem_base(self.backend.as_mut(), &resolved_loc, f_context, vm);

849
                                    self.finish_block();
850

851
                                    let blk_store_start = make_block_name(node, "store_start");
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873

                                    // store_start:
                                    self.start_block(blk_store_start.clone(), &vec![temp_loc.clone()]);

                                    let success = make_temporary(f_context, UINT1_TYPE.clone(), vm);
                                    let discard_reg = cast_value(&success, &UINT64_TYPE);
                                    // Load a value (discard it)
                                    if use_acquire {
                                        self.backend.emit_ldaxp(&XZR, &discard_reg, &temp_loc);
                                    } else {
                                        self.backend.emit_ldxp(&XZR, &discard_reg, &temp_loc);
                                    }

                                    // Store the value
                                    if use_release {
                                        self.backend.emit_stlxp(&temp_loc, &success, &val_l, &val_h);
                                    } else {
                                        self.backend.emit_stxp(&temp_loc, &success, &val_l, &val_h);
                                    }

                                    // If the store failed, then branch back to 'store_start:'
                                    self.backend.emit_cbnz(&success, blk_store_start.clone())
874
                                }
875
                            }
876
                        } else {
877
                            unimplemented!();
878
                        }
879

880
881
                    }

882
                    Instruction_::CmpXchg{is_weak, success_order, fail_order, mem_loc, expected_value, desired_value, ..} => {
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
                        // Note: this uses the same operations as GCC (for the C++ atomic cmpxchg)
                        // Clang is slightly different and ignores the 'fail_order'
                        let use_acquire = match fail_order {
                            MemoryOrder::Acquire | MemoryOrder::SeqCst => true,
                            MemoryOrder::Relaxed => match success_order {
                                MemoryOrder::Acquire | MemoryOrder::AcqRel | MemoryOrder::SeqCst => true,
                                MemoryOrder::Relaxed | MemoryOrder::Release => false,
                                _ => panic!("didnt expect success order {:?} for cmpxchg", success_order)
                            },
                            _ => panic!("didnt expect fail order {:?} for cmpxchg", fail_order)
                        };
                        let use_release = match fail_order {
                            MemoryOrder::Acquire => match success_order {
                                MemoryOrder::Relaxed | MemoryOrder::Release | MemoryOrder::AcqRel | MemoryOrder::SeqCst => true,
                                MemoryOrder::Acquire => false,
                                _ => panic!("didnt expect success order {:?} for cmpxchg", success_order)
                            },
                            MemoryOrder::SeqCst => true,
                            MemoryOrder::Relaxed => match success_order {
                                MemoryOrder::Release | MemoryOrder::AcqRel | MemoryOrder::SeqCst => true,
                                MemoryOrder::Relaxed | MemoryOrder::Acquire => false,
                                _ => panic!("didnt expect success order {:?} for cmpxchg", success_order)
                            },
                            _ => panic!("didnt expect fail order {:?} for cmpxchg", fail_order)
                        };


910
                        let ref ops = inst.ops;
911
912
913
914
915
916
917
                        let loc = self.emit_node_addr_to_value(&ops[mem_loc], f_content, f_context, vm);
                        let expected = self.emit_reg(&ops[expected_value], f_content, f_context, vm);
                        let desired = self.emit_reg(&ops[desired_value], f_content, f_context, vm);

                        let res_value = self.get_result_value(node, 0);
                        let res_success = self.get_result_value(node, 1);

918
919
920
                        let blk_cmpxchg_start =    make_block_name(node, "cmpxchg_start");
                        let blk_cmpxchg_failed =   make_block_name(node, "cmpxchg_failed");
                        let blk_cmpxchg_succeded = make_block_name(node, "cmpxchg_succeded");
921

922
                        self.finish_block();
923

924
                        // cmpxchg_start:
925
926
927
                        self.start_block(blk_cmpxchg_start.clone(), &vec![loc.clone(),expected.clone(), desired.clone()]);

                        if use_acquire {
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
                            match res_value.ty.v {
                                // Have to load a temporary GPR first
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_ldaxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
                                }
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_ldaxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
                                }
                                // Can load the register directly
                                _ => self.backend.emit_ldaxr(&res_value, &loc)
                            };
943
                        } else {
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
                            match res_value.ty.v {
                                // Have to load a temporary GPR first
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_ldxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
                                }
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_ldxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
                                }
                                // Can load the register directly
                                _ => self.backend.emit_ldxr(&res_value, &loc)
                            };
959
960
                        }

961
                        if is_int_reg(&expected) {
962
963
964
965
966
967
968
                            self.backend.emit_cmp(&res_value, &expected);
                        } else {
                            self.backend.emit_fcmp(&res_value, &expected);
                        }
                        self.backend.emit_b_cond("NE", blk_cmpxchg_failed.clone());

                        if use_release {
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
                            match desired.ty.v {
                                // Have to store a temporary GPR
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stlxr(&loc, &res_success, &temp);
                                }
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stlxr(&loc, &res_success, &temp);
                                }
                                // Can load the register directly
                                _ => self.backend.emit_stlxr(&loc, &res_success, &desired)
                            };
984
                        } else {
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
                            match desired.ty.v {
                                // Have to store a temporary GPR
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stxr(&loc, &res_success, &temp);
                                }
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stxr(&loc, &res_success, &temp);
                                }
                                // Can load the register directly
                                _ => self.backend.emit_stxr(&loc, &res_success, &desired)
                            };
1000
1001
1002
1003
1004
1005
1006
1007
1008
                        }

                        if !is_weak {
                            // Store failed, try again
                            self.backend.emit_cbnz(&res_success, blk_cmpxchg_start.clone());
                        }

                        self.backend.emit_b(blk_cmpxchg_succeded.clone());

1009
                        self.finish_block();
1010

1011
                        // cmpxchg_failed:
1012
1013
1014
1015
1016
1017
                        self.start_block(blk_cmpxchg_failed.clone(), &vec![res_success.clone(), res_value.clone()]);

                        self.backend.emit_clrex();
                        // Set res_success to 1 (the same value STXR/STLXR uses to indicate failure)
                        self.backend.emit_mov_imm(&res_success, 1);

1018
                        self.finish_block();
1019

1020
                        // cmpxchg_succeded:
1021
1022
1023
1024
                        self.start_block(blk_cmpxchg_succeded.clone(), &vec![res_success.clone(), res_value.clone()]);
                        // this NOT is needed as STXR/STLXR returns sucess as '0', wheras the Mu spec says it should be 1
                        self.backend.emit_eor_imm(&res_success, &res_success, 1);
                    }
1025
1026
1027
1028
1029
1030
                    Instruction_::GetIRef(_)
                    | Instruction_::GetFieldIRef { .. }
                    | Instruction_::GetElementIRef{..}
                    | Instruction_::GetVarPartIRef { .. }
                    | Instruction_::ShiftIRef { .. } => {
                        trace!("instsel on GET/FIELD/VARPARTIREF, SHIFTIREF");
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1031
                        let mem_addr = self.emit_get_mem_from_inst(node, f_content, f_context, vm);
1032
                        let tmp_res = self.get_result_value(node, 0);
1033
                        emit_calculate_address(self.backend.as_mut(), &tmp_res, &mem_addr, f_context, vm);
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
                    }

                    Instruction_::Fence(order) => {
                        trace!("instsel on FENCE");

                        // Whether to emit a load fence or a normal one
                        let use_load = match order {
                            MemoryOrder::Release | MemoryOrder::SeqCst | MemoryOrder::AcqRel => false,
                            MemoryOrder::Acquire => true,
                            _ => panic!("didnt expect order {:?} with load inst", order)
                        };

                        if use_load {
                            // Data Memory Barrirer for Inner Shariable Domain (for Load accesses only)
                            self.backend.emit_dmb("ISHLD");
                        } else {
                            // Data Memory Barrirer for Inner Shariable Domain
                            self.backend.emit_dmb("ISH");
                        }
                    }
1054

1055
1056
1057
                    // TODO: Implement this similar to a return (where theres a common exit block)
                    // and change SWAP_BACK_TO_NATIV_STACK and swap_to_mu_stack so they don't handle the callee saved registers
                    // (this instruction should then guarentee that they are restored (in the same way a Return does)
1058
1059
1060
1061
1062
                    Instruction_::ThreadExit => {
                        trace!("instsel on THREADEXIT");
                        // emit a call to swap_back_to_native_stack(sp_loc: Address)

                        // get thread local and add offset to get sp_loc
1063
                        let tl = self.emit_get_threadlocal(f_context, vm);
1064
1065
                        self.backend.emit_add_imm(&tl, &tl, *thread::NATIVE_SP_LOC_OFFSET as u16, false);

1066
                        self.emit_runtime_entry(&entrypoints::SWAP_BACK_TO_NATIVE_STACK, vec![tl.clone()], None, Some(node), f_context, vm);
1067
1068
                    }

1069

1070
1071
1072
                    Instruction_::CommonInst_GetThreadLocal => {
                        trace!("instsel on GETTHREADLOCAL");
                        // get thread local
1073
                        let tl = self.emit_get_threadlocal(f_context, vm);
1074

1075
                        let tmp_res = self.get_result_value(node, 0);
1076
1077

                        // load [tl + USER_TLS_OFFSET] -> tmp_res
1078
                        emit_load_base_offset(self.backend.as_mut(), &tmp_res, &tl, *thread::USER_TLS_OFFSET as i64, f_context, vm);
1079
1080
                    }

1081

1082
1083
                    Instruction_::CommonInst_SetThreadLocal(op) => {
                        trace!("instsel on SETTHREADLOCAL");
1084
                        let ref ops = inst.ops;
1085
1086
1087
1088
                        let ref op = ops[op];

                        debug_assert!(self.match_ireg(op));

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1089
                        let tmp_op = self.emit_ireg(op, f_content, f_context, vm);
1090
1091

                        // get thread local
1092
                        let tl = self.emit_get_threadlocal(f_context, vm);
1093
1094

                        // store tmp_op -> [tl + USER_TLS_OFFSTE]
1095
                        emit_store_base_offset(self.backend.as_mut(), &tl, *thread::USER_TLS_OFFSET as i64, &tmp_op, f_context, vm);
1096
1097
1098
1099
1100
1101
                    }

                    Instruction_::CommonInst_Pin(op) => {
                        trace!("instsel on PIN");
                        if !mm::GC_MOVES_OBJECT {
                            // non-moving GC: pin is a nop (move from op to result)
1102
                            let ref ops = inst.ops;
1103
1104
                            let ref op = ops[op];

1105
                            let tmp_res = self.get_result_value(node, 0);
1106

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1107
                            self.emit_move_node_to_value(&tmp_res, op, f_content, f_context, vm);
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
                        } else {
                            unimplemented!()
                        }
                    }

                    Instruction_::CommonInst_Unpin(_) => {
                        trace!("instsel on UNPIN");
                        if !mm::GC_MOVES_OBJECT {
                            // do nothing
                        } else {
                            unimplemented!()
                        }
                    }

                    Instruction_::Move(op) => {
                        trace!("instsel on MOVE (internal IR)");
1124
                        let ref ops = inst.ops;
1125
1126
                        let ref op = ops[op];

1127
                        let tmp_res = self.get_result_value(node, 0);
1128

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1129
                        self.emit_move_node_to_value(&tmp_res, op, f_content, f_context, vm);
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
                    }

                    Instruction_::New(ref ty) => {
                        trace!("instsel on NEW");
                        if cfg!(debug_assertions) {
                            match ty.v {
                                MuType_::Hybrid(_) => panic!("cannot use NEW for hybrid, use NEWHYBRID instead"),
                                _ => {}
                            }
                        }

                        let ty_info = vm.get_backend_type_info(ty.id());
                        let size = ty_info.size;
                        let ty_align = ty_info.alignment;

1145
                        let const_size = make_value_int_const(size as u64, vm);
1146

1147
                        let tmp_allocator = self.emit_get_allocator(f_context, vm);
1148
                        let tmp_res = self.emit_alloc_sequence(tmp_allocator.clone(), const_size, ty_align, node, f_context, vm);
1149
1150

                        // ASM: call muentry_init_object(%allocator, %tmp_res, %encode)
1151
                        let encode = make_value_int_const(mm::get_gc_type_encode(ty_info.gc_type.id), vm);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1152
                        self.emit_runtime_entry(
1153
1154
1155
                            &entrypoints::INIT_OBJ,
                            vec![tmp_allocator.clone(), tmp_res.clone(), encode],
                            None,
1156
                            Some(node), f_context, vm
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
                        );
                    }

                    Instruction_::NewHybrid(ref ty, var_len) => {
                        trace!("instsel on NEWHYBRID");
                        if cfg!(debug_assertions) {
                            match ty.v {
                                MuType_::Hybrid(_) => {},
                                _ => panic!("NEWHYBRID is only for allocating hybrid types, use NEW for others")
                            }
                        }

                        let ty_info = vm.get_backend_type_info(ty.id());
                        let ty_align = ty_info.alignment;
                        let fix_part_size = ty_info.size;
                        let var_ty_size = match ty.v {
                            MuType_::Hybrid(ref name) => {
                                let map_lock = HYBRID_TAG_MAP.read().unwrap();
                                let hybrid_ty_ = map_lock.get(name).unwrap();
                                let var_ty = hybrid_ty_.get_var_ty();

                                vm.get_backend_type_info(var_ty.id()).size
                            },
                            _ => panic!("only expect HYBRID type here")
                        };

                        // actual size = fix_part_size + var_ty_size * len
                        let (actual_size, length) = {
1185
                            let ref ops = inst.ops;
1186
1187
                            let ref var_len = ops[var_len];