To protect your data, the CISO officer has suggested users to enable GitLab 2FA as soon as possible.

inst_sel.rs 251 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
// Copyright 2017 The Australian National University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![warn(unused_imports)]
#![warn(unreachable_code)]

use ast::ir::*;
use ast::ptr::*;
use ast::inst::*;
use ast::op;
use ast::op::*;
use ast::types::*;
use utils::math::align_up;
use vm::VM;
use runtime::mm;
use runtime::mm::OBJECT_HEADER_SIZE;

use runtime::ValueLocation;
use runtime::thread;
use runtime::entrypoints;
use runtime::entrypoints::RuntimeEntrypoint;

use compiler::CompilerPass;

use compiler::backend::PROLOGUE_BLOCK_NAME;
use compiler::backend::EPILOGUE_BLOCK_NAME;

use compiler::backend::aarch64::*;
use compiler::backend::make_block_name;
use compiler::machine_code::CompiledFunction;
use compiler::frame::Frame;

use std::collections::HashMap;
use std::collections::LinkedList;
use std::mem;
use std::any::Any;
use num::integer::lcm;

const INLINE_FASTPATH: bool = false;

pub struct InstructionSelection {
    name: &'static str,
    backend: Box<CodeGenerator>,

    // The key is the function signature id, the value is the combined return type
    combined_return_types: HashMap<MuID, P<MuType>>,
    current_return_type: Option<P<MuType>>,
    current_fv_id: MuID,
    current_fv_name: MuName,
    current_callsite_id: usize,
    current_frame: Option<Frame>,
    current_block: Option<MuName>,
    current_block_in_ir: Option<MuName>,
    current_func_start: Option<ValueLocation>,

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
67
68
69
    // A list of all callsites, with the corresponding exception block (if there is one)

    // Technically this is a map in that each Key is unique, but we will never try and add duplicate
70
71
72
73
74
75
76
    // keys, or look things up, so a list of pairs is faster than a Map.
    current_callsites: LinkedList<(MuName, MuID, usize)>,
    // key: block id, val: block location
    current_exn_blocks: HashMap<MuID, MuName>,
    current_stack_arg_size: usize,
    current_xr_value: Option<P<Value>>, // A temporary that holds to saved XR value (if needed)
    current_constants: HashMap<MuID, P<Value>>,
77
    current_constants_locs: HashMap<MuID, P<Value>>
78
79
}

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
80
81
// TODO: Move all functions that are in here that don't need access to 'self'
// (or only call functions that don't need access to self (even if called on self)) to Mod.rs
82
83
84
85
86
87
88
89
90
91
92
93
94
95
impl<'a> InstructionSelection {
    #[cfg(feature = "aot")]
    pub fn new() -> InstructionSelection {
        InstructionSelection {
            name: "Instruction Selection (aarch64)",
            backend: Box::new(ASMCodeGen::new()),

            combined_return_types: HashMap::new(),
            current_return_type: None,
            current_fv_id: 0,
            current_fv_name: String::new(),
            current_callsite_id: 0,
            current_frame: None,
            current_block: None,
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
96
97
98
99
100
101
102
103

            current_block_in_ir: None,
            // it is possible the block is newly created in instruction selection
            // but sometimes we want to know its control flow
            // so we need to track what block it is from the IR

            // FIXME: ideally we should not create new blocks in instruction selection
            // see Issue #6
104
105
106
107
108
109
            current_func_start: None,
            current_callsites: LinkedList::new(),
            current_exn_blocks: HashMap::new(),
            current_stack_arg_size: 0,
            current_xr_value: None,
            current_constants: HashMap::new(),
110
            current_constants_locs: HashMap::new()
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
        }

    }

    #[cfg(feature = "jit")]
    pub fn new() -> InstructionSelection {
        unimplemented!()
    }

    // in this pass, we assume that
    // * we do not need to backup/restore caller-saved registers
    // if any of these assumption breaks, we will need to re-emit the code
    fn instruction_select(
        &mut self,
        node: &'a TreeNode,
        f_content: &FunctionContent,
        f_context: &mut FunctionContext,
128
        vm: &VM
129
130
131
    ) {
        trace!("instsel on node#{} {}", node.id(), node);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
132
        match node.v {
133
            TreeNode_::Instruction(ref inst) => {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
134
                match inst.v {
135
136
137
138
139
140
141
142
143
144
                    // TODO: Optimise if cond is a flag from a binary operation?
                    Instruction_::Branch2 {
                        cond,
                        ref true_dest,
                        ref false_dest,
                        ..
                    } => {
                        trace!("instsel on BRANCH2");
                        let (fallthrough_dest, branch_dest, branch_if_true) = {
                            let cur_block = f_content.get_block_by_name(
145
                                self.current_block_in_ir.as_ref().unwrap().clone()
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
                            );
                            let next_block_in_trace =
                                cur_block.control_flow.get_hottest_succ().unwrap();

                            if next_block_in_trace == true_dest.target {
                                (true_dest, false_dest, false)
                            } else {
                                (false_dest, true_dest, true)
                            }
                        };

                        let ref ops = inst.ops;

                        self.process_dest(&ops, fallthrough_dest, f_content, f_context, vm);
                        self.process_dest(&ops, branch_dest, f_content, f_context, vm);

                        let branch_target = f_content.get_block(branch_dest.target).name();

                        let ref cond = ops[cond];

                        if self.match_cmp_res(cond) {
                            trace!("emit cmp_res-branch2");
                            // Emit a CBNZ for 128-bit comparisons that are not symmetric
                            let use_cbnz = self.is_int128_asym_cmp(cond);
                            let tmp_cond = if use_cbnz {
                                Some(make_temporary(f_context, UINT1_TYPE.clone(), vm))
                            } else {
                                None
                            };
                            let cond_box = if use_cbnz {
                                Some(Box::new(tmp_cond.as_ref().unwrap().clone()))
                            } else {
                                None
                            };

                            let mut cmpop =
                                self.emit_cmp_res(cond, cond_box, f_content, f_context, vm);

                            if use_cbnz {
                                if !branch_if_true {
                                    self.backend
                                        .emit_cbz(tmp_cond.as_ref().unwrap(), branch_target);
                                } else {
                                    self.backend
                                        .emit_cbnz(tmp_cond.as_ref().unwrap(), branch_target);
                                }

                            } else {
                                if !branch_if_true {
                                    cmpop = cmpop.invert();
                                }

                                let cond = get_condition_codes(cmpop);

                                if cmpop == op::CmpOp::FFALSE {
; // Do nothing
                                } else if cmpop == op::CmpOp::FTRUE {
                                    self.backend.emit_b(branch_target);
                                } else {
                                    self.backend.emit_b_cond(cond[0], branch_target.clone());

                                    if cond.len() == 2 {
                                        self.backend.emit_b_cond(cond[1], branch_target);
                                    }
                                }
                            }
                        } else {
                            let cond_reg = self.emit_ireg(cond, f_content, f_context, vm);

                            if branch_if_true {
                                self.backend.emit_tbnz(&cond_reg, 0, branch_target.clone());
                            } else {
                                self.backend.emit_tbz(&cond_reg, 0, branch_target.clone());
                            }
                        };

                        // it is possible that the fallthrough block is scheduled somewhere else
                        // we need to explicitly jump to it
                        self.finish_block();
                        let fallthrough_temp_block =
                            make_block_name(&self.current_fv_name, node.id(), "branch_fallthrough");
                        self.start_block(fallthrough_temp_block);

                        let fallthrough_target =
                            f_content.get_block(fallthrough_dest.target).name();
                        self.backend.emit_b(fallthrough_target);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
232
                    }
233
234
235
236

                    Instruction_::Select {
                        cond,
                        true_val,
237
                        false_val
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
                    } => {
                        use ast::op::CmpOp::*;

                        trace!("instsel on SELECT");
                        let ref ops = inst.ops;

                        let ref cond = ops[cond];
                        let ref true_val = ops[true_val];
                        let ref false_val = ops[false_val];

                        let tmp_res = self.get_result_value(node, 0);

                        // moving integers/pointers
                        // generate compare
                        let cmpop = if self.match_cmp_res(cond) {
                            self.emit_cmp_res(cond, None, f_content, f_context, vm)
                        } else if self.match_ireg(cond) {
                            let tmp_cond = self.emit_ireg(cond, f_content, f_context, vm);
                            self.backend.emit_cmp_imm(&tmp_cond, 0, false);
                            NE
                        } else {
                            panic!("expected ireg, found {}", cond)
                        };

                        let tmp_true = self.emit_reg(true_val, f_content, f_context, vm);
                        let tmp_false = self.emit_reg(false_val, f_content, f_context, vm);

                        let cond = get_condition_codes(cmpop);

                        if self.match_ireg(true_val) {
                            if cmpop == FFALSE {
                                self.backend.emit_mov(&tmp_res, &tmp_false);
                            } else if cmpop == FTRUE {
                                self.backend.emit_mov(&tmp_res, &tmp_true);
                            } else {
                                self.backend
                                    .emit_csel(&tmp_res, &tmp_true, &tmp_false, cond[0]);

                                if cond.len() == 2 {
                                    self.backend
                                        .emit_csel(&tmp_res, &tmp_true, &tmp_res, cond[1]);
                                }
                            }
                        } else if self.match_fpreg(true_val) {
                            if cmpop == FFALSE {
                                self.backend.emit_fmov(&tmp_res, &tmp_false);
                            } else if cmpop == FTRUE {
                                self.backend.emit_fmov(&tmp_res, &tmp_true);
                            } else {
                                self.backend
                                    .emit_fcsel(&tmp_res, &tmp_true, &tmp_false, cond[0]);

                                if cond.len() == 2 {
                                    self.backend
                                        .emit_fcsel(&tmp_res, &tmp_true, &tmp_res, cond[1]);
                                }
                            }
                        } else {
                            // moving vectors
                            unimplemented!()
                        }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
299
                    }
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320

                    Instruction_::CmpOp(op, op1, op2) => {
                        use ast::op::CmpOp::*;

                        trace!("instsel on CMPOP");
                        let ref ops = inst.ops;
                        let ref op1 = ops[op1];
                        let ref op2 = ops[op2];

                        let tmp_res = self.get_result_value(node, 0);

                        debug_assert!(tmp_res.ty.get_int_length().is_some());
                        debug_assert!(tmp_res.ty.get_int_length().unwrap() == 1);

                        let cmpop = self.emit_cmp_res_op(
                            op,
                            Some(Box::new(tmp_res.clone())),
                            &op1,
                            &op2,
                            f_content,
                            f_context,
321
                            vm
322
323
324
325
326
327
328
329
330
331
332
333
                        );
                        let cond = get_condition_codes(cmpop);

                        // emit_cmp_res_op will set tmp_res for 128-bit assymettric comparisons
                        if !self.is_int128_asym_cmp(node) {
                            if cmpop == FFALSE {
                                emit_mov_u64(self.backend.as_mut(), &tmp_res, 0);
                            } else if cmpop == FTRUE {
                                emit_mov_u64(self.backend.as_mut(), &tmp_res, 1);
                            } else {
                                self.backend.emit_cset(&tmp_res, cond[0]);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
334
335
336
                                // Note: some compariosns can't be computed based on a single
                                // aarch64 flag insted they are computed as a condition OR NOT
                                // another condition.
337
338
339
340
341
                                if cond.len() == 2 {
                                    self.backend.emit_csinc(
                                        &tmp_res,
                                        &tmp_res,
                                        &WZR,
342
                                        invert_condition_code(cond[1])
343
344
345
346
                                    );
                                }
                            }
                        }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
347
                    }
348
349
350
351
352
353
354
355
356
357
358
359

                    Instruction_::Branch1(ref dest) => {
                        trace!("instsel on BRANCH1");
                        let ref ops = inst.ops;

                        self.process_dest(&ops, dest, f_content, f_context, vm);

                        let target = f_content.get_block(dest.target).name();

                        trace!("emit branch1");
                        // jmp
                        self.backend.emit_b(target);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
360
                    }
361
362
363
364

                    Instruction_::Switch {
                        cond,
                        ref default,
365
                        ref branches
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
                    } => {
                        trace!("instsel on SWITCH");
                        let ref ops = inst.ops;

                        let ref cond = ops[cond];

                        if self.match_ireg(cond) {
                            let tmp_cond = self.emit_ireg(cond, f_content, f_context, vm);
                            emit_zext(self.backend.as_mut(), &tmp_cond);

                            // emit each branch
                            for &(case_op_index, ref case_dest) in branches {
                                let ref case_op = ops[case_op_index];

                                // process dest
                                self.process_dest(&ops, case_dest, f_content, f_context, vm);

                                let target = f_content.get_block(case_dest.target).name();

                                let mut imm_val = 0 as u64;
                                // Is one of the arguments a valid immediate?
                                let emit_imm = if match_node_int_imm(&case_op) {
                                    imm_val = node_imm_to_u64(&case_op);
                                    is_valid_arithmetic_imm(imm_val)
                                } else {
                                    false
                                };

                                if emit_imm {
                                    let imm_shift = imm_val > 4096;
                                    let imm_op2 = if imm_shift { imm_val >> 12 } else { imm_val };
                                    self.backend
                                        .emit_cmp_imm(&tmp_cond, imm_op2 as u16, imm_shift);
                                } else {
                                    let tmp_case_op =
                                        self.emit_ireg(case_op, f_content, f_context, vm);
                                    emit_zext(self.backend.as_mut(), &tmp_case_op);
                                    self.backend.emit_cmp(&tmp_cond, &tmp_case_op);
                                }

                                self.backend.emit_b_cond("EQ", target);

                                self.finish_block();
                                let block_name = make_block_name(
                                    &self.current_fv_name,
                                    node.id(),
412
                                    format!("switch_not_met_case_{}", case_op_index).as_str()
413
414
415
416
417
418
419
420
421
422
423
424
                                );
                                self.start_block(block_name);
                            }

                            // emit default
                            self.process_dest(&ops, default, f_content, f_context, vm);

                            let default_target = f_content.get_block(default.target).name();
                            self.backend.emit_b(default_target);
                        } else {
                            panic!("expecting cond in switch to be ireg: {}", cond);
                        }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
425
                    }
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441

                    Instruction_::ExprCall { ref data, is_abort } => {
                        trace!("instsel on EXPRCALL");

                        if is_abort {
                            unimplemented!()
                        }

                        self.emit_mu_call(
                            false, // is tail
                            inst,  // inst: &Instruction,
                            data,  // calldata: &CallData,
                            None,  // resumption: Option<&ResumptionData>,
                            node,  // cur_node: &TreeNode,
                            f_content,
                            f_context,
442
                            vm
443
                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
444
                    }
445
446
447

                    Instruction_::Call {
                        ref data,
448
                        ref resume
449
450
451
452
453
454
455
456
457
458
459
                    } => {
                        trace!("instsel on CALL");

                        self.emit_mu_call(
                            false, // is tail
                            inst,
                            data,
                            Some(resume),
                            node,
                            f_content,
                            f_context,
460
                            vm
461
                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
462
                    }
463
464
465
466
467
468
469
470
471
472
473
474

                    Instruction_::TailCall(ref data) => {
                        trace!("instsel on TAILCALL");

                        self.emit_mu_call(
                            true, // is tail
                            inst,
                            data,
                            None,
                            node,
                            f_content,
                            f_context,
475
                            vm
476
                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
477
                    }
478
479
480
481
482
483
484
485
486

                    Instruction_::ExprCCall { ref data, is_abort } => {
                        trace!("instsel on EXPRCCALL");

                        if is_abort {
                            unimplemented!()
                        }

                        self.emit_c_call_ir(inst, data, None, node, f_content, f_context, vm);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
487
                    }
488
489
490

                    Instruction_::CCall {
                        ref data,
491
                        ref resume
492
493
494
495
496
497
498
499
500
501
                    } => {
                        trace!("instsel on CCALL");

                        self.emit_c_call_ir(
                            inst,
                            data,
                            Some(resume),
                            node,
                            f_content,
                            f_context,
502
                            vm
503
                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
504
                    }
505
506
507
508
509
510

                    Instruction_::Return(ref vals) => {
                        trace!("instsel on RETURN");

                        // prepare return regs
                        let ref ops = inst.ops;
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
511
512
                        // TODO: Are vals in the same order as the return types in the functions
                        // signature?
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536

                        let ret_tys: Vec<P<MuType>> =
                            vals.iter().map(|i| node_type(&ops[*i])).collect();
                        let ret_type = self.current_return_type.as_ref().unwrap().clone();

                        let n = ret_tys.len(); // number of return values
                        let xr_value = self.current_xr_value.as_ref().unwrap().clone();

                        if n == 0 {
                            // Do nothing
                        } else if n == 1 {
                            let ret_loc = self.compute_return_locations(&ret_type, &xr_value, &vm);
                            let ret_val =
                                self.emit_node_value(&ops[vals[0]], f_content, f_context, vm);

                            if is_machine_reg(&ret_loc) && is_int_ex_reg(&ret_val) {
                                let (val_l, val_h) = split_int128(&ret_val, f_context, vm);
                                let ret_loc_h = get_register_from_id(ret_loc.id() + 2);
                                // nothing special needs to be done
                                emit_move_value_to_value(
                                    self.backend.as_mut(),
                                    &ret_loc,
                                    &val_l,
                                    f_context,
537
                                    vm
538
539
540
541
542
543
                                );
                                emit_move_value_to_value(
                                    self.backend.as_mut(),
                                    &ret_loc_h,
                                    &val_h,
                                    f_context,
544
                                    vm
545
546
547
548
549
550
551
                                );
                            } else {
                                emit_move_value_to_value(
                                    self.backend.as_mut(),
                                    &ret_loc,
                                    &ret_val,
                                    f_context,
552
                                    vm
553
554
555
556
557
558
559
560
561
562
563
                                );
                            }
                        } else {
                            let ret_loc = self.compute_return_locations(&ret_type, &xr_value, &vm);

                            let mut i = 0;
                            for ret_index in vals {
                                let ret_val = self.emit_node_value(
                                    &ops[*ret_index],
                                    f_content,
                                    f_context,
564
                                    vm
565
566
567
568
                                );
                                let ref ty = ret_val.ty;
                                let offset = self.get_field_offset(&ret_type, i, &vm);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
569
570
571
572
573
                                match ty.v {
                                    MuType_::Vector(_, _) => unimplemented!(),
                                    MuType_::Void => panic!("Unexpected void"),
                                    MuType_::Struct(_) | MuType_::Array(_, _) => unimplemented!(),
                                    MuType_::Hybrid(_) => panic!("Can't return a hybrid"),
574
575
576
577
578
579
580
                                    // Integral, pointer or floating point type
                                    _ => {
                                        self.insert_bytes(
                                            &ret_loc,
                                            &ret_val,
                                            offset as i64,
                                            f_context,
581
                                            vm
582
                                        )
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
583
                                    }
584
585
586
587
588
589
590
591
592
                                }

                                i += 1;
                            }
                        }

                        let epilogue_block =
                            format!("{}:{}", self.current_fv_name, EPILOGUE_BLOCK_NAME);
                        self.backend.emit_b(epilogue_block);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
593
                    }
594
595
596
597
598
599
600
601
602
603
604

                    Instruction_::BinOp(op, op1, op2) => {
                        trace!("instsel on BINOP");
                        self.emit_binop(
                            node,
                            inst,
                            op,
                            BinOpStatus {
                                flag_n: false,
                                flag_z: false,
                                flag_c: false,
605
                                flag_v: false
606
607
608
609
610
                            },
                            op1,
                            op2,
                            f_content,
                            f_context,
611
                            vm
612
                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
613
                    }
614
615
616
617

                    Instruction_::BinOpWithStatus(op, status, op1, op2) => {
                        trace!("instsel on BINOP_STATUS");
                        self.emit_binop(node, inst, op, status, op1, op2, f_content, f_context, vm);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
618
                    }
619
620
621
622
623

                    Instruction_::ConvOp {
                        operation,
                        ref from_ty,
                        ref to_ty,
624
                        operand
625
626
627
628
629
630
631
632
633
634
635
636
637
                    } => {
                        trace!("instsel on CONVOP");

                        let ref ops = inst.ops;

                        let ref op = ops[operand];

                        let tmp_res = self.get_result_value(node, 0);
                        let tmp_op = self.emit_reg(op, f_content, f_context, vm);

                        let from_ty_size = get_bit_size(&from_ty, vm);
                        let to_ty_size = get_bit_size(&to_ty, vm);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
638
                        match operation {
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
                            op::ConvOp::TRUNC => {
                                // src is in one register
                                if self.match_ireg(op) {
                                    self.backend
                                        .emit_mov(&tmp_res, &cast_value(&tmp_op, &to_ty));
                                } else if self.match_ireg_ex(op) {
                                    // Move the lower word
                                    if from_ty_size != to_ty_size {
                                        let (op_l, _) =
                                            self.emit_ireg_ex(op, f_content, f_context, vm);
                                        self.backend.emit_mov(&tmp_res, &cast_value(&op_l, &to_ty));
                                    } else {
                                        self.emit_move_node_to_value(
                                            &tmp_res,
                                            op,
                                            f_content,
                                            f_context,
656
                                            vm
657
658
659
660
661
662
                                        );
                                    }
                                } else {
                                    panic!("unexpected op (expect ireg): {}", op);
                                }

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
663
                            }
664
665
666
667
668
669
670
671

                            op::ConvOp::ZEXT => {
                                if from_ty_size != to_ty_size {
                                    if to_ty_size <= 64 {
                                        self.backend.emit_ubfx(
                                            &tmp_res,
                                            &cast_value(&tmp_op, &to_ty),
                                            0,
672
                                            from_ty_size as u8
673
674
675
676
677
678
679
680
681
                                        );
                                    } else if to_ty_size == 128 {
                                        let (res_l, res_h) = split_int128(&tmp_res, f_context, vm);

                                        // res_l = ZEXT src
                                        self.backend.emit_ubfx(
                                            &res_l,
                                            &cast_value(&tmp_op, &UINT64_TYPE),
                                            0,
682
                                            from_ty_size as u8
683
684
685
686
687
688
689
690
691
692
693
694
695
                                        );
                                        self.backend.emit_mov(&res_h, &XZR); // res_h = 0

                                    } else {
                                        panic!("unexpected int length {}", to_ty_size);
                                    }
                                } else {
                                    // Trivial, just do a move
                                    emit_move_value_to_value(
                                        self.backend.as_mut(),
                                        &tmp_res,
                                        &tmp_op,
                                        f_context,
696
                                        vm
697
698
                                    );
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
699
                            }
700
701
702
703
704
705
706
707

                            op::ConvOp::SEXT => {
                                if from_ty_size != to_ty_size {
                                    if to_ty_size <= 64 {
                                        self.backend.emit_sbfx(
                                            &tmp_res,
                                            &cast_value(&tmp_op, &to_ty),
                                            0,
708
                                            from_ty_size as u8
709
710
711
712
713
714
715
716
717
                                        );
                                    } else if to_ty_size == 128 {
                                        let (res_l, res_h) = split_int128(&tmp_res, f_context, vm);

                                        // res_l = SEXT src
                                        self.backend.emit_sbfx(
                                            &res_l,
                                            &cast_value(&tmp_op, &UINT64_TYPE),
                                            0,
718
                                            from_ty_size as u8
719
                                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
720
721
                                        // res_h = ASR src, 63
                                        self.backend.emit_asr_imm(&res_h, &tmp_op, 63);
722
723
724
725
726
727
728
729
730
731
732
733

                                    } else {
                                        panic!("unexpected int length {}", to_ty_size);
                                    }

                                } else {
                                    // Trivial, just do a move
                                    emit_move_value_to_value(
                                        self.backend.as_mut(),
                                        &tmp_res,
                                        &tmp_op,
                                        f_context,
734
                                        vm
735
736
                                    );
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
737
                            }
738
739
740
                            op::ConvOp::REFCAST | op::ConvOp::PTRCAST => {
                                // just a mov (and hopefully reg alloc will coalesce it)
                                self.backend.emit_mov(&tmp_res, &tmp_op);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
741
                            }
742
743
744
745
746
747
748
749
750
751

                            op::ConvOp::UITOFP => {
                                if from_ty_size == 128 {
                                    if to_ty_size == 64 {
                                        self.emit_runtime_entry(
                                            &entrypoints::UITOFP_U128_DOUBLE,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
752
                                            vm
753
754
755
756
757
758
759
760
                                        );
                                    } else {
                                        self.emit_runtime_entry(
                                            &entrypoints::UITOFP_U128_FLOAT,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
761
                                            vm
762
763
764
765
766
767
                                        );
                                    }
                                } else {
                                    emit_zext(self.backend.as_mut(), &tmp_op);
                                    self.backend.emit_ucvtf(&tmp_res, &tmp_op);
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
768
                            }
769
770
771
772
773
774
775
776
777
778

                            op::ConvOp::SITOFP => {
                                if from_ty_size == 128 {
                                    if to_ty_size == 64 {
                                        self.emit_runtime_entry(
                                            &entrypoints::SITOFP_I128_DOUBLE,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
779
                                            vm
780
781
782
783
784
785
786
787
                                        );
                                    } else {
                                        self.emit_runtime_entry(
                                            &entrypoints::SITOFP_I128_FLOAT,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
788
                                            vm
789
790
791
792
793
794
                                        );
                                    }
                                } else {
                                    emit_sext(self.backend.as_mut(), &tmp_op);
                                    self.backend.emit_scvtf(&tmp_res, &tmp_op);
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
795
                            }
796
797
798
799
800
801
802
803
804
805

                            op::ConvOp::FPTOUI => {
                                if to_ty_size == 128 {
                                    if from_ty_size == 64 {
                                        self.emit_runtime_entry(
                                            &entrypoints::FPTOUI_DOUBLE_U128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
806
                                            vm
807
808
809
810
811
812
813
814
                                        );
                                    } else {
                                        self.emit_runtime_entry(
                                            &entrypoints::FPTOUI_FLOAT_U128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
815
                                            vm
816
817
818
819
820
821
                                        );
                                    }
                                } else {
                                    self.backend.emit_fcvtzu(&tmp_res, &tmp_op);
                                    // We have to emmit code to handle the case when the real result
                                    // overflows to_ty_size, but not to_ty_reg_size
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
822
823
824

                                    // The size of the aarch64 register
                                    let to_ty_reg_size = check_op_len(&tmp_res.ty);
825
826
827
828
829
                                    if to_ty_size != to_ty_reg_size {
                                        // Compare the bits of the result after the lower
                                        // to_ty_size bits
                                        self.backend.emit_tst_imm(
                                            &tmp_res,
830
                                            bits_ones(to_ty_reg_size - to_ty_size) << to_ty_size
831
832
833
834
835
836
837
838
                                        );

                                        // If the above condition is true, the an overflow occurred
                                        // So set tmp_res to !0 (i.e. all ones, the maximum value)
                                        self.backend.emit_csinv(
                                            &tmp_res,
                                            &tmp_res,
                                            &get_alias_for_length(XZR.id(), to_ty_size),
839
                                            "EQ"
840
841
842
                                        );
                                    }
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
843
                            }
844
845
846
847
848
849
850
851
852
853

                            op::ConvOp::FPTOSI => {
                                if to_ty_size == 128 {
                                    if from_ty_size == 64 {
                                        self.emit_runtime_entry(
                                            &entrypoints::FPTOSI_DOUBLE_I128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
854
                                            vm
855
856
857
858
859
860
861
862
                                        );
                                    } else {
                                        self.emit_runtime_entry(
                                            &entrypoints::FPTOSI_FLOAT_I128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
863
                                            vm
864
865
866
867
868
                                        );
                                    }
                                } else {
                                    self.backend.emit_fcvtzs(&tmp_res, &tmp_op);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
869
870
                                    // TODO This code is horrible and inefficient due to branches
                                    // and duplication, is there a better way?
871
872
873

                                    // We have to emmit code to handle the case when the real result
                                    // overflows to_ty_size, but not to_ty_reg_size
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
874
875
876

                                    // The size of the aarch64 register
                                    let to_ty_reg_size = check_op_len(&tmp_res.ty);
877
878
879
880
                                    if to_ty_size != to_ty_reg_size {
                                        let blk_positive = make_block_name(
                                            &self.current_fv_name,
                                            node.id(),
881
                                            "positive"
882
883
884
885
                                        );
                                        let blk_negative = make_block_name(
                                            &self.current_fv_name,
                                            node.id(),
886
                                            "negative"
887
888
889
890
                                        );
                                        let blk_end = make_block_name(
                                            &self.current_fv_name,
                                            node.id(),
891
                                            "end"
892
893
894
895
896
897
                                        );
                                        let tmp = make_temporary(f_context, to_ty.clone(), vm);

                                        self.backend.emit_tbnz(
                                            &tmp_res,
                                            (to_ty_size - 1) as u8,
898
                                            blk_negative.clone()
899
900
901
902
903
904
                                        );
                                        self.finish_block();

                                        self.start_block(blk_positive.clone());
                                        {
                                            // check to see if the higher bits are the same as the
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
905
906
                                            // sign bit (which is 0), if their not there's an
                                            // overflow
907
908
909
                                            self.backend.emit_tst_imm(
                                                &tmp_res,
                                                bits_ones(to_ty_reg_size - to_ty_size) <<
910
                                                    to_ty_size
911
912
913
914
915
916
917
918
919
920
921
922
923
924
                                            );
                                            self.backend
                                                .emit_mov_imm(&tmp, bits_ones(to_ty_size - 1));

                                            // if the above test fails (i.e. results in zero)
                                            // then set temp_res to tmp
                                            self.backend.emit_csel(&tmp_res, &tmp, &tmp_res, "EQ");

                                            self.backend.emit_b(blk_end.clone());
                                            self.finish_block();
                                        }
                                        self.start_block(blk_negative.clone());
                                        {
                                            self.backend.emit_mvn(&tmp, &tmp_res);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
925
926
927
                                            // check to see if the higher bits of temp are the same
                                            // as the sign bit (which is 1), if their not there's
                                            // an overflow
928
929
930
                                            self.backend.emit_tst_imm(
                                                &tmp_res,
                                                bits_ones(to_ty_reg_size - to_ty_size) <<
931
                                                    to_ty_size
932
933
                                            );

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
934
935
                                            // Set just the sign bit (this is smallest representable
                                            // signed number)
936
937
                                            self.backend.emit_mov_imm(&tmp, 1 << to_ty_size);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
938
939
                                            // if the above test fails (i.e. results in zero), then
                                            // set temp_res to tmp
940
941
942
943
944
945
                                            self.backend.emit_csel(&tmp_res, &tmp, &tmp_res, "EQ");
                                            self.finish_block();
                                        }
                                        self.start_block(blk_end.clone());
                                    }
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
946
                            }
947
948
949

                            op::ConvOp::BITCAST => {
                                self.backend.emit_fmov(&tmp_res, &tmp_op);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
950
                            }
951
952
                            op::ConvOp::FPTRUNC | op::ConvOp::FPEXT => {
                                self.backend.emit_fcvt(&tmp_res, &tmp_op);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
953
                            }
954
                        }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
955
                    }
956
957
958
959
960
961
962
963
964
965
966
967

                    Instruction_::Load { order, mem_loc, .. } => {
                        trace!("instsel on LOAD");
                        let ref ops = inst.ops;
                        let ref loc_op = ops[mem_loc];

                        let resolved_loc =
                            self.emit_node_addr_to_value(loc_op, f_content, f_context, vm);
                        let res = self.get_result_value(node, 0);

                        if self.match_ireg(node) || self.match_fpreg(node) {
                            // Whether to use a load acquire
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
968
969
                            let use_acquire = match order {
                                MemoryOrder::Relaxed | MemoryOrder::NotAtomic => false,
970
                                MemoryOrder::Consume | MemoryOrder::Acquire |
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
971
972
                                MemoryOrder::SeqCst => true,
                                _ => panic!("didnt expect order {:?} with load inst", order)
973
974
975
976
977
978
979
980
981
                            };


                            if use_acquire {
                                // Can only have a base for a LDAR
                                let temp_loc = emit_mem_base(
                                    self.backend.as_mut(),
                                    &resolved_loc,
                                    f_context,
982
                                    vm
983
                                );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
984
                                match res.ty.v {
985
986
987
988
989
990
                                    // Have to load a temporary GPR first
                                    MuType_::Float => {
                                        let temp =
                                            make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                        self.backend.emit_ldar(&temp, &temp_loc);
                                        self.backend.emit_fmov(&res, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
991
                                    }
992
993
994
995
996
                                    MuType_::Double => {
                                        let temp =
                                            make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                        self.backend.emit_ldar(&temp, &temp_loc);
                                        self.backend.emit_fmov(&res, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
997
                                    }
998
                                    // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
999
                                    _ => self.backend.emit_ldar(&res, &temp_loc)
1000
1001
1002
1003
1004
1005
1006
                                };
                            } else {
                                let temp_loc = emit_mem(
                                    self.backend.as_mut(),
                                    &resolved_loc,
                                    get_type_alignment(&res.ty, vm),
                                    f_context,
1007
                                    vm
1008
1009
1010
1011
1012
1013
                                );
                                self.backend.emit_ldr(&res, &temp_loc, false);
                            }
                        } else if self.match_ireg_ex(node) {
                            let (res_l, res_h) = split_int128(&res, f_context, vm);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1014
                            match order {
1015
1016
1017
1018
1019
1020
                                MemoryOrder::NotAtomic => {
                                    let temp_loc = emit_mem(
                                        self.backend.as_mut(),
                                        &resolved_loc,
                                        get_type_alignment(&res.ty, vm),
                                        f_context,
1021
                                        vm
1022
1023
                                    );
                                    self.backend.emit_ldp(&res_l, &res_h, &temp_loc);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1024
                                }
1025
1026

                                // Aarch64 dosn't have a load acquire pair instruction
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1027
1028
                                // So instead we have to write a loop using load/store exclusive
                                // pairs
1029
1030
                                _ => {
                                    // Whether to use a load exclusive acquire
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1031
1032
                                    let use_acquire = match order {
                                        MemoryOrder::Relaxed => false,
1033
1034
                                        MemoryOrder::Consume |
                                        MemoryOrder::Acquire |
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1035
                                        MemoryOrder::SeqCst => true,
1036
1037
1038
1039
1040
                                        _ => {
                                            panic!(
                                                "didnt expect order {:?} with atomic load inst",
                                                order
                                            )
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1041
                                        }
1042
1043
                                    };
                                    // Whether to use a store exclusive release
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1044
                                    let use_release = match order {
1045
1046
                                        MemoryOrder::Relaxed |
                                        MemoryOrder::Consume |
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1047
1048
                                        MemoryOrder::Acquire => false,
                                        MemoryOrder::SeqCst => true,
1049
1050
1051
1052
1053
                                        _ => {
                                            panic!(
                                                "didnt expect order {:?} with atomic load inst",
                                                order
                                            )
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1054
                                        }
1055
1056
1057
1058
1059
1060
1061
                                    };

                                    // Exclusive loads/stores, only supports a base address
                                    let temp_loc = emit_mem_base(
                                        self.backend.as_mut(),
                                        &resolved_loc,
                                        f_context,
1062
                                        vm
1063
1064
1065
1066
1067
1068
1069
                                    );

                                    self.finish_block();

                                    let blk_load_start = make_block_name(
                                        &self.current_fv_name,
                                        node.id(),
1070
                                        "load_start"
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
                                    );

                                    // load_start:
                                    self.start_block(blk_load_start.clone());


                                    // Load the value:
                                    if use_acquire {
                                        self.backend.emit_ldaxp(&res_l, &res_h, &temp_loc);
                                    } else {
                                        self.backend.emit_ldxp(&res_l, &res_h, &temp_loc);
                                    }

                                    let success = make_temporary(f_context, UINT1_TYPE.clone(), vm);

                                    // Store the value we just read back to memory
                                    if use_release {
                                        self.backend
                                            .emit_stlxp(&temp_loc, &success, &res_l, &res_h);
                                    } else {
                                        self.backend.emit_stxp(&temp_loc, &success, &res_l, &res_h);
                                    }

                                    // If the store failed, then branch back to 'load_start:'
                                    self.backend.emit_cbnz(&success, blk_load_start.clone())
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1096
                                }
1097
1098
1099
1100
                            }
                        } else {
                            unimplemented!();
                        }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1101
                    }
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118

                    Instruction_::Store {
                        order,
                        mem_loc,
                        value,
                        ..
                    } => {
                        trace!("instsel on STORE");
                        let ref ops = inst.ops;
                        let ref loc_op = ops[mem_loc];
                        let ref val_op = ops[value];

                        let resolved_loc =
                            self.emit_node_addr_to_value(loc_op, f_content, f_context, vm);

                        if self.match_ireg(val_op) || self.match_fpreg(val_op) {
                            // Whether to use a store release or not
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1119
1120
1121
1122
                            let use_release = match order {
                                MemoryOrder::Relaxed | MemoryOrder::NotAtomic => false,
                                MemoryOrder::Release | MemoryOrder::SeqCst => true,
                                _ => panic!("didnt expect order {:?} with load inst", order)
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
                            };

                            let val = self.emit_reg(val_op, f_content, f_context, vm);

                            if use_release {
                                // Can only have a base for a STLR
                                let temp_loc = emit_mem_base(
                                    self.backend.as_mut(),
                                    &resolved_loc,
                                    f_context,
1133
                                    vm
1134
1135
                                );

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1136
                                match val.ty.v {
1137
1138
1139
1140
1141
1142
                                    // Have to store a temporary GPR
                                    MuType_::Float => {
                                        let temp =
                                            make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                        self.backend.emit_fmov(&temp, &val);
                                        self.backend.emit_stlr(&temp_loc, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1143
                                    }
1144
1145
1146
1147
1148
                                    MuType_::Double => {
                                        let temp =
                                            make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                        self.backend.emit_fmov(&temp, &val);
                                        self.backend.emit_stlr(&temp_loc, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1149
                                    }
1150
                                    // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1151
                                    _ => self.backend.emit_stlr(&temp_loc, &val)
1152
1153
1154
1155
1156
1157
1158
                                };
                            } else {
                                let temp_loc = emit_mem(
                                    self.backend.as_mut(),
                                    &resolved_loc,
                                    get_type_alignment(&val.ty, vm),
                                    f_context,
1159
                                    vm
1160
1161
1162
1163
1164
1165
1166
                                );
                                self.backend.emit_str(&temp_loc, &val);
                            }
                        } else if self.match_ireg_ex(val_op) {
                            let (val_l, val_h) =
                                self.emit_ireg_ex(val_op, f_content, f_context, vm);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1167
                            match order {
1168
1169
1170
1171
1172
1173
                                MemoryOrder::NotAtomic => {
                                    let temp_loc = emit_mem(
                                        self.backend.as_mut(),
                                        &resolved_loc,
                                        16,
                                        f_context,
1174
                                        vm
1175
1176
                                    );
                                    self.backend.emit_stp(&temp_loc, &val_l, &val_h);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1177
                                }
1178
1179

                                // Aarch64 dosn't have a store release pair instruction
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1180
1181
                                // So instead we have to write a loop using load/store exclusive
                                // pairs
1182
1183
                                _ => {
                                    // Whether to use a load exclusive acquire
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1184
1185
1186
                                    let use_acquire = match order {
                                        MemoryOrder::Relaxed | MemoryOrder::Release => false,
                                        MemoryOrder::SeqCst => true,
1187
1188
1189
1190
1191
                                        _ => {
                                            panic!(
                                                "didnt expect order {:?} with atomic store inst",
                                                order
                                            )
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1192
                                        }
1193
1194
                                    };
                                    // Whether to use a store exclusive release
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1195
1196
1197
                                    let use_release = match order {
                                        MemoryOrder::Relaxed => false,
                                        MemoryOrder::Release | MemoryOrder::SeqCst => true,
1198
1199
1200
1201
1202
                                        _ => {
                                            panic!(
                                                "didnt expect order {:?} with atomic store inst",
                                                order
                                            )
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1203
                                        }
1204
1205
1206
1207
1208
1209
1210
                                    };

                                    // Exclusive loads/stores, only supports a base address
                                    let temp_loc = emit_mem_base(
                                        self.backend.as_mut(),
                                        &resolved_loc,
                                        f_context,
1211
                                        vm
1212
1213
1214
1215
1216
1217
1218
                                    );

                                    self.finish_block();

                                    let blk_store_start = make_block_name(
                                        &self.current_fv_name,
                                        node.id(),
1219
                                        "store_start"
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
                                    );

                                    // store_start:
                                    self.start_block(blk_store_start.clone());

                                    let success = make_temporary(f_context, UINT1_TYPE.clone(), vm);
                                    let discard_reg = cast_value(&success, &UINT64_TYPE);
                                    // Load a value (discard it)
                                    if use_acquire {
                                        self.backend.emit_ldaxp(&XZR, &discard_reg, &temp_loc);
                                    } else {
                                        self.backend.emit_ldxp(&XZR, &discard_reg, &temp_loc);
                                    }

                                    // Store the value
                                    if use_release {
                                        self.backend
                                            .emit_stlxp(&temp_loc, &success, &val_l, &val_h);
                                    } else {
                                        self.backend.emit_stxp(&temp_loc, &success, &val_l, &val_h);
                                    }

                                    // If the store failed, then branch back to 'store_start:'
                                    self.backend.emit_cbnz(&success, blk_store_start.clone())
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1244
                                }
1245
1246
1247
1248
1249
                            }
                        } else {
                            unimplemented!();
                        }

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1250
                    }
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262

                    Instruction_::CmpXchg {
                        is_weak,
                        success_order,
                        fail_order,
                        mem_loc,
                        expected_value,
                        desired_value,
                        ..
                    } => {
                        // Note: this uses the same operations as GCC (for the C++ atomic cmpxchg)
                        // Clang is slightly different and ignores the 'fail_order'
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1263
1264
                        let use_acquire = match fail_order {
                            MemoryOrder::Acquire | MemoryOrder::SeqCst => true,
1265
                            MemoryOrder::Relaxed => {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1266
                                match success_order {
1267
1268
                                    MemoryOrder::Acquire |
                                    MemoryOrder::AcqRel |
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1269
1270
                                    MemoryOrder::SeqCst => true,
                                    MemoryOrder::Relaxed | MemoryOrder::Release => false,
1271
1272
1273
1274
1275
                                    _ => {
                                        panic!(
                                            "didnt expect success order {:?} for cmpxchg",
                                            success_order
                                        )
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1276
                                    }
1277
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1278
1279
                            }
                            _ => panic!("didnt expect fail order {:?} for cmpxchg", fail_order)
1280
                        };
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1281
                        let use_release = match fail_order {
1282
                            MemoryOrder::Acquire => {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1283
                                match success_order {
1284
1285
1286
                                    MemoryOrder::Relaxed |
                                    MemoryOrder::Release |
                                    MemoryOrder::AcqRel |
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1287
1288
                                    MemoryOrder::SeqCst => true,
                                    MemoryOrder::Acquire => false,
1289
1290
1291
1292
1293
                                    _ => {
                                        panic!(
                                            "didnt expect success order {:?} for cmpxchg",
                                            success_order
                                        )
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1294
                                    }
1295
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1296
1297
                            }
                            MemoryOrder::SeqCst => true,
1298
                            MemoryOrder::Relaxed => {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1299
                                match success_order {
1300
1301
                                    MemoryOrder::Release |
                                    MemoryOrder::AcqRel |
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1302
1303
                                    MemoryOrder::SeqCst => true,
                                    MemoryOrder::Relaxed | MemoryOrder::Acquire => false,
1304
1305
1306
1307
1308
                                    _ => {
                                        panic!(
                                            "didnt expect success order {:?} for cmpxchg",
                                            success_order
                                        )
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1309
                                    }
1310
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1311
1312
                            }
                            _ => panic!("didnt expect fail order {:?} for cmpxchg", fail_order)
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
                        };


                        let ref ops = inst.ops;
                        let loc =
                            self.emit_node_addr_to_value(&ops[mem_loc], f_content, f_context, vm);
                        let expected =
                            self.emit_reg(&ops[expected_value], f_content, f_context, vm);
                        let desired = self.emit_reg(&ops[desired_value], f_content, f_context, vm);

                        let res_value = self.get_result_value(node, 0);
                        let res_success = self.get_result_value(node, 1);

                        let blk_cmpxchg_start =
                            make_block_name(&self.current_fv_name, node.id(), "cmpxchg_start");
                        let blk_cmpxchg_failed =
                            make_block_name(&self.current_fv_name, node.id(), "cmpxchg_failed");
                        let blk_cmpxchg_succeded =
                            make_block_name(&self.current_fv_name, node.id(), "cmpxchg_succeded");

                        self.finish_block();

                        // cmpxchg_start:
                        self.start_block(blk_cmpxchg_start.clone());

                        if use_acquire {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1339
                            match res_value.ty.v {
1340
1341
1342
1343
1344
                                // Have to load a temporary GPR first
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_ldaxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1345
                                }
1346
1347
1348
1349
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_ldaxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1350
                                }
1351
                                // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1352
                                _ => self.backend.emit_ldaxr(&res_value, &loc)
1353
1354
                            };
                        } else {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1355
                            match res_value.ty.v {
1356
1357
1358
1359
1360
                                // Have to load a temporary GPR first
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_ldxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1361
                                }
1362
1363
1364
1365
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_ldxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1366
                                }
1367
                                // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1368
                                _ => self.backend.emit_ldxr(&res_value, &loc)
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
                            };
                        }

                        if is_int_reg(&expected) {
                            self.backend.emit_cmp(&res_value, &expected);
                        } else {
                            self.backend.emit_fcmp(&res_value, &expected);
                        }
                        self.backend.emit_b_cond("NE", blk_cmpxchg_failed.clone());

                        if use_release {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1380
                            match desired.ty.v {
1381
1382
1383
1384
1385
                                // Have to store a temporary GPR
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stlxr(&loc, &res_success, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1386
                                }
1387
1388
1389
1390
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stlxr(&loc, &res_success, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1391
                                }
1392
                                // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1393
                                _ => self.backend.emit_stlxr(&loc, &res_success, &desired)
1394
1395
                            };
                        } else {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1396
                            match desired.ty.v {
1397
1398
1399
1400
1401
                                // Have to store a temporary GPR
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stxr(&loc, &res_success, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1402
                                }
1403
1404
1405
1406
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stxr(&loc, &res_success, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1407
                                }
1408
                                // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1409
                                _ => self.backend.emit_stxr(&loc, &res_success, &desired)