To protect your data, the CISO officer has suggested users to enable GitLab 2FA as soon as possible.

inst_sel.rs 253 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
// Copyright 2017 The Australian National University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![warn(unused_imports)]
#![warn(unreachable_code)]

use ast::ir::*;
use ast::ptr::*;
use ast::inst::*;
use ast::op;
use ast::op::*;
use ast::types::*;
use utils::math::align_up;
use vm::VM;
use runtime::mm;
use runtime::mm::OBJECT_HEADER_SIZE;

use runtime::ValueLocation;
use runtime::thread;
use runtime::entrypoints;
use runtime::entrypoints::RuntimeEntrypoint;

use compiler::CompilerPass;

36
use compiler::PROLOGUE_BLOCK_NAME;
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

use compiler::backend::aarch64::*;
use compiler::backend::make_block_name;
use compiler::machine_code::CompiledFunction;
use compiler::frame::Frame;

use std::collections::HashMap;
use std::collections::LinkedList;
use std::mem;
use std::any::Any;
use num::integer::lcm;

const INLINE_FASTPATH: bool = false;

pub struct InstructionSelection {
    name: &'static str,
    backend: Box<CodeGenerator>,

    // The key is the function signature id, the value is the combined return type
    combined_return_types: HashMap<MuID, P<MuType>>,
    current_return_type: Option<P<MuType>>,
    current_fv_id: MuID,
    current_fv_name: MuName,
    current_callsite_id: usize,
    current_frame: Option<Frame>,
    current_block: Option<MuName>,
    current_block_in_ir: Option<MuName>,
    current_func_start: Option<ValueLocation>,

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
66
67
68
    // A list of all callsites, with the corresponding exception block (if there is one)

    // Technically this is a map in that each Key is unique, but we will never try and add duplicate
69
70
71
72
73
74
75
    // keys, or look things up, so a list of pairs is faster than a Map.
    current_callsites: LinkedList<(MuName, MuID, usize)>,
    // key: block id, val: block location
    current_exn_blocks: HashMap<MuID, MuName>,
    current_stack_arg_size: usize,
    current_xr_value: Option<P<Value>>, // A temporary that holds to saved XR value (if needed)
    current_constants: HashMap<MuID, P<Value>>,
76
    current_constants_locs: HashMap<MuID, P<Value>>
77
78
}

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
79
80
// TODO: Move all functions that are in here that don't need access to 'self'
// (or only call functions that don't need access to self (even if called on self)) to Mod.rs
81
82
83
84
85
86
87
88
89
90
91
92
93
impl<'a> InstructionSelection {
    #[cfg(feature = "aot")]
    pub fn new() -> InstructionSelection {
        InstructionSelection {
            name: "Instruction Selection (aarch64)",
            backend: Box::new(ASMCodeGen::new()),
            combined_return_types: HashMap::new(),
            current_return_type: None,
            current_fv_id: 0,
            current_fv_name: String::new(),
            current_callsite_id: 0,
            current_frame: None,
            current_block: None,
94
95
            // it is possible the block is newly
            // created in instruction selection
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
96
97
            // but sometimes we want to know its control flow
            // so we need to track what block it is from the IR
98
99
100
            // FIXME: ideally we should not create new blocks
            // in instruction selection. See Issue #6
            current_block_in_ir: None,
101
102
103
104
105
106
            current_func_start: None,
            current_callsites: LinkedList::new(),
            current_exn_blocks: HashMap::new(),
            current_stack_arg_size: 0,
            current_xr_value: None,
            current_constants: HashMap::new(),
107
            current_constants_locs: HashMap::new()
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
        }

    }

    #[cfg(feature = "jit")]
    pub fn new() -> InstructionSelection {
        unimplemented!()
    }

    // in this pass, we assume that
    // * we do not need to backup/restore caller-saved registers
    // if any of these assumption breaks, we will need to re-emit the code
    fn instruction_select(
        &mut self,
        node: &'a TreeNode,
        f_content: &FunctionContent,
        f_context: &mut FunctionContext,
125
        vm: &VM
126
127
128
    ) {
        trace!("instsel on node#{} {}", node.id(), node);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
129
        match node.v {
130
            TreeNode_::Instruction(ref inst) => {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
131
                match inst.v {
132
133
134
135
136
137
138
139
                    // TODO: Optimise if cond is a flag from a binary operation?
                    Instruction_::Branch2 {
                        cond,
                        ref true_dest,
                        ref false_dest,
                        ..
                    } => {
                        trace!("instsel on BRANCH2");
140
                        let (fallthrough_dest, branch_dest) = (false_dest, true_dest);
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165

                        let ref ops = inst.ops;

                        self.process_dest(&ops, fallthrough_dest, f_content, f_context, vm);
                        self.process_dest(&ops, branch_dest, f_content, f_context, vm);

                        let branch_target = f_content.get_block(branch_dest.target).name();

                        let ref cond = ops[cond];

                        if self.match_cmp_res(cond) {
                            trace!("emit cmp_res-branch2");
                            // Emit a CBNZ for 128-bit comparisons that are not symmetric
                            let use_cbnz = self.is_int128_asym_cmp(cond);
                            let tmp_cond = if use_cbnz {
                                Some(make_temporary(f_context, UINT1_TYPE.clone(), vm))
                            } else {
                                None
                            };
                            let cond_box = if use_cbnz {
                                Some(Box::new(tmp_cond.as_ref().unwrap().clone()))
                            } else {
                                None
                            };

166
                            let cmpop = self.emit_cmp_res(cond, cond_box, f_content, f_context, vm);
167
168

                            if use_cbnz {
169
170
                                self.backend
                                    .emit_cbnz(tmp_cond.as_ref().unwrap(), branch_target);
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
                            } else {
                                let cond = get_condition_codes(cmpop);

                                if cmpop == op::CmpOp::FFALSE {
; // Do nothing
                                } else if cmpop == op::CmpOp::FTRUE {
                                    self.backend.emit_b(branch_target);
                                } else {
                                    self.backend.emit_b_cond(cond[0], branch_target.clone());

                                    if cond.len() == 2 {
                                        self.backend.emit_b_cond(cond[1], branch_target);
                                    }
                                }
                            }
                        } else {
                            let cond_reg = self.emit_ireg(cond, f_content, f_context, vm);
188
                            self.backend.emit_tbnz(&cond_reg, 0, branch_target.clone());
189
                        };
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
190
                    }
191
192
193
194

                    Instruction_::Select {
                        cond,
                        true_val,
195
                        false_val
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
                    } => {
                        use ast::op::CmpOp::*;

                        trace!("instsel on SELECT");
                        let ref ops = inst.ops;

                        let ref cond = ops[cond];
                        let ref true_val = ops[true_val];
                        let ref false_val = ops[false_val];

                        let tmp_res = self.get_result_value(node, 0);

                        // moving integers/pointers
                        // generate compare
                        let cmpop = if self.match_cmp_res(cond) {
                            self.emit_cmp_res(cond, None, f_content, f_context, vm)
                        } else if self.match_ireg(cond) {
                            let tmp_cond = self.emit_ireg(cond, f_content, f_context, vm);
                            self.backend.emit_cmp_imm(&tmp_cond, 0, false);
                            NE
                        } else {
                            panic!("expected ireg, found {}", cond)
                        };

                        let tmp_true = self.emit_reg(true_val, f_content, f_context, vm);
                        let tmp_false = self.emit_reg(false_val, f_content, f_context, vm);

                        let cond = get_condition_codes(cmpop);

                        if self.match_ireg(true_val) {
                            if cmpop == FFALSE {
                                self.backend.emit_mov(&tmp_res, &tmp_false);
                            } else if cmpop == FTRUE {
                                self.backend.emit_mov(&tmp_res, &tmp_true);
                            } else {
                                self.backend
                                    .emit_csel(&tmp_res, &tmp_true, &tmp_false, cond[0]);

                                if cond.len() == 2 {
                                    self.backend
                                        .emit_csel(&tmp_res, &tmp_true, &tmp_res, cond[1]);
                                }
                            }
                        } else if self.match_fpreg(true_val) {
                            if cmpop == FFALSE {
                                self.backend.emit_fmov(&tmp_res, &tmp_false);
                            } else if cmpop == FTRUE {
                                self.backend.emit_fmov(&tmp_res, &tmp_true);
                            } else {
                                self.backend
                                    .emit_fcsel(&tmp_res, &tmp_true, &tmp_false, cond[0]);

                                if cond.len() == 2 {
                                    self.backend
                                        .emit_fcsel(&tmp_res, &tmp_true, &tmp_res, cond[1]);
                                }
                            }
                        } else {
                            // moving vectors
                            unimplemented!()
                        }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
257
                    }
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278

                    Instruction_::CmpOp(op, op1, op2) => {
                        use ast::op::CmpOp::*;

                        trace!("instsel on CMPOP");
                        let ref ops = inst.ops;
                        let ref op1 = ops[op1];
                        let ref op2 = ops[op2];

                        let tmp_res = self.get_result_value(node, 0);

                        debug_assert!(tmp_res.ty.get_int_length().is_some());
                        debug_assert!(tmp_res.ty.get_int_length().unwrap() == 1);

                        let cmpop = self.emit_cmp_res_op(
                            op,
                            Some(Box::new(tmp_res.clone())),
                            &op1,
                            &op2,
                            f_content,
                            f_context,
279
                            vm
280
281
282
283
284
285
286
287
288
289
290
291
                        );
                        let cond = get_condition_codes(cmpop);

                        // emit_cmp_res_op will set tmp_res for 128-bit assymettric comparisons
                        if !self.is_int128_asym_cmp(node) {
                            if cmpop == FFALSE {
                                emit_mov_u64(self.backend.as_mut(), &tmp_res, 0);
                            } else if cmpop == FTRUE {
                                emit_mov_u64(self.backend.as_mut(), &tmp_res, 1);
                            } else {
                                self.backend.emit_cset(&tmp_res, cond[0]);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
292
                                // Note: some compariosns can't be computed based on a single
293
294
                                // aarch64 flag insted they are computed as
                                // a condition OR NOT another condition.
295
296
297
298
299
                                if cond.len() == 2 {
                                    self.backend.emit_csinc(
                                        &tmp_res,
                                        &tmp_res,
                                        &WZR,
300
                                        invert_condition_code(cond[1])
301
302
303
304
                                    );
                                }
                            }
                        }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
305
                    }
306
307
308
309
310
311
312
313
314
315
316
317

                    Instruction_::Branch1(ref dest) => {
                        trace!("instsel on BRANCH1");
                        let ref ops = inst.ops;

                        self.process_dest(&ops, dest, f_content, f_context, vm);

                        let target = f_content.get_block(dest.target).name();

                        trace!("emit branch1");
                        // jmp
                        self.backend.emit_b(target);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
318
                    }
319
320
321
322

                    Instruction_::Switch {
                        cond,
                        ref default,
323
                        ref branches
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
                    } => {
                        trace!("instsel on SWITCH");
                        let ref ops = inst.ops;

                        let ref cond = ops[cond];

                        if self.match_ireg(cond) {
                            let tmp_cond = self.emit_ireg(cond, f_content, f_context, vm);
                            emit_zext(self.backend.as_mut(), &tmp_cond);

                            // emit each branch
                            for &(case_op_index, ref case_dest) in branches {
                                let ref case_op = ops[case_op_index];

                                // process dest
                                self.process_dest(&ops, case_dest, f_content, f_context, vm);

                                let target = f_content.get_block(case_dest.target).name();

                                let mut imm_val = 0 as u64;
                                // Is one of the arguments a valid immediate?
                                let emit_imm = if match_node_int_imm(&case_op) {
                                    imm_val = node_imm_to_u64(&case_op);
                                    is_valid_arithmetic_imm(imm_val)
                                } else {
                                    false
                                };

                                if emit_imm {
                                    let imm_shift = imm_val > 4096;
                                    let imm_op2 = if imm_shift { imm_val >> 12 } else { imm_val };
                                    self.backend
                                        .emit_cmp_imm(&tmp_cond, imm_op2 as u16, imm_shift);
                                } else {
                                    let tmp_case_op =
                                        self.emit_ireg(case_op, f_content, f_context, vm);
                                    emit_zext(self.backend.as_mut(), &tmp_case_op);
                                    self.backend.emit_cmp(&tmp_cond, &tmp_case_op);
                                }

                                self.backend.emit_b_cond("EQ", target);

                                self.finish_block();
                                let block_name = make_block_name(
368
                                    &node.name(),
369
                                    format!("switch_not_met_case_{}", case_op_index).as_str()
370
371
372
373
374
375
376
377
378
379
380
381
                                );
                                self.start_block(block_name);
                            }

                            // emit default
                            self.process_dest(&ops, default, f_content, f_context, vm);

                            let default_target = f_content.get_block(default.target).name();
                            self.backend.emit_b(default_target);
                        } else {
                            panic!("expecting cond in switch to be ireg: {}", cond);
                        }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
382
                    }
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398

                    Instruction_::ExprCall { ref data, is_abort } => {
                        trace!("instsel on EXPRCALL");

                        if is_abort {
                            unimplemented!()
                        }

                        self.emit_mu_call(
                            false, // is tail
                            inst,  // inst: &Instruction,
                            data,  // calldata: &CallData,
                            None,  // resumption: Option<&ResumptionData>,
                            node,  // cur_node: &TreeNode,
                            f_content,
                            f_context,
399
                            vm
400
                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
401
                    }
402
403
404

                    Instruction_::Call {
                        ref data,
405
                        ref resume
406
407
408
409
410
411
412
413
414
415
416
                    } => {
                        trace!("instsel on CALL");

                        self.emit_mu_call(
                            false, // is tail
                            inst,
                            data,
                            Some(resume),
                            node,
                            f_content,
                            f_context,
417
                            vm
418
                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
419
                    }
420
421
422
423
424
425
426
427
428
429
430
431

                    Instruction_::TailCall(ref data) => {
                        trace!("instsel on TAILCALL");

                        self.emit_mu_call(
                            true, // is tail
                            inst,
                            data,
                            None,
                            node,
                            f_content,
                            f_context,
432
                            vm
433
                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
434
                    }
435
436
437
438
439
440
441
442
443

                    Instruction_::ExprCCall { ref data, is_abort } => {
                        trace!("instsel on EXPRCCALL");

                        if is_abort {
                            unimplemented!()
                        }

                        self.emit_c_call_ir(inst, data, None, node, f_content, f_context, vm);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
444
                    }
445
446
447

                    Instruction_::CCall {
                        ref data,
448
                        ref resume
449
450
451
452
453
454
455
456
457
458
                    } => {
                        trace!("instsel on CCALL");

                        self.emit_c_call_ir(
                            inst,
                            data,
                            Some(resume),
                            node,
                            f_content,
                            f_context,
459
                            vm
460
                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
461
                    }
462
463
464
465
466
467

                    Instruction_::Return(ref vals) => {
                        trace!("instsel on RETURN");

                        // prepare return regs
                        let ref ops = inst.ops;
468
469
                        // TODO: Are vals in the same order as the return types in the
                        // functions signature?
470
471

                        let ret_type = self.current_return_type.as_ref().unwrap().clone();
472
                        let n = vals.len(); // number of return values
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
                        let xr_value = self.current_xr_value.as_ref().unwrap().clone();

                        if n == 0 {
                            // Do nothing
                        } else if n == 1 {
                            let ret_loc = self.compute_return_locations(&ret_type, &xr_value, &vm);
                            let ret_val =
                                self.emit_node_value(&ops[vals[0]], f_content, f_context, vm);

                            if is_machine_reg(&ret_loc) && is_int_ex_reg(&ret_val) {
                                let (val_l, val_h) = split_int128(&ret_val, f_context, vm);
                                let ret_loc_h = get_register_from_id(ret_loc.id() + 2);
                                // nothing special needs to be done
                                emit_move_value_to_value(
                                    self.backend.as_mut(),
                                    &ret_loc,
                                    &val_l,
                                    f_context,
491
                                    vm
492
493
494
495
496
497
                                );
                                emit_move_value_to_value(
                                    self.backend.as_mut(),
                                    &ret_loc_h,
                                    &val_h,
                                    f_context,
498
                                    vm
499
500
501
502
503
504
505
                                );
                            } else {
                                emit_move_value_to_value(
                                    self.backend.as_mut(),
                                    &ret_loc,
                                    &ret_val,
                                    f_context,
506
                                    vm
507
508
509
510
511
512
513
514
515
516
517
                                );
                            }
                        } else {
                            let ret_loc = self.compute_return_locations(&ret_type, &xr_value, &vm);

                            let mut i = 0;
                            for ret_index in vals {
                                let ret_val = self.emit_node_value(
                                    &ops[*ret_index],
                                    f_content,
                                    f_context,
518
                                    vm
519
520
521
522
                                );
                                let ref ty = ret_val.ty;
                                let offset = self.get_field_offset(&ret_type, i, &vm);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
523
524
525
526
527
                                match ty.v {
                                    MuType_::Vector(_, _) => unimplemented!(),
                                    MuType_::Void => panic!("Unexpected void"),
                                    MuType_::Struct(_) | MuType_::Array(_, _) => unimplemented!(),
                                    MuType_::Hybrid(_) => panic!("Can't return a hybrid"),
528
                                    // Integral, pointer or floating point type
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
529
530
531
532
533
534
535
536
537
                                    _ => {
                                        self.insert_bytes(
                                            &ret_loc,
                                            &ret_val,
                                            offset as i64,
                                            f_context,
                                            vm
                                        )
                                    }
538
539
540
541
542
543
                                }

                                i += 1;
                            }
                        }

544
545
                        self.emit_epilogue(f_context, vm);
                        self.backend.emit_ret(&LR);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
546
                    }
547
548
549
550
551
552
553
554
555
556
557

                    Instruction_::BinOp(op, op1, op2) => {
                        trace!("instsel on BINOP");
                        self.emit_binop(
                            node,
                            inst,
                            op,
                            BinOpStatus {
                                flag_n: false,
                                flag_z: false,
                                flag_c: false,
558
                                flag_v: false
559
560
561
562
563
                            },
                            op1,
                            op2,
                            f_content,
                            f_context,
564
                            vm
565
                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
566
                    }
567
568
569
570

                    Instruction_::BinOpWithStatus(op, status, op1, op2) => {
                        trace!("instsel on BINOP_STATUS");
                        self.emit_binop(node, inst, op, status, op1, op2, f_content, f_context, vm);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
571
                    }
572
573
574
575
576

                    Instruction_::ConvOp {
                        operation,
                        ref from_ty,
                        ref to_ty,
577
                        operand
578
579
580
581
582
583
584
585
586
587
588
589
590
                    } => {
                        trace!("instsel on CONVOP");

                        let ref ops = inst.ops;

                        let ref op = ops[operand];

                        let tmp_res = self.get_result_value(node, 0);
                        let tmp_op = self.emit_reg(op, f_content, f_context, vm);

                        let from_ty_size = get_bit_size(&from_ty, vm);
                        let to_ty_size = get_bit_size(&to_ty, vm);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
591
                        match operation {
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
                            op::ConvOp::TRUNC => {
                                // src is in one register
                                if self.match_ireg(op) {
                                    self.backend
                                        .emit_mov(&tmp_res, &cast_value(&tmp_op, &to_ty));
                                } else if self.match_ireg_ex(op) {
                                    // Move the lower word
                                    if from_ty_size != to_ty_size {
                                        let (op_l, _) =
                                            self.emit_ireg_ex(op, f_content, f_context, vm);
                                        self.backend.emit_mov(&tmp_res, &cast_value(&op_l, &to_ty));
                                    } else {
                                        self.emit_move_node_to_value(
                                            &tmp_res,
                                            op,
                                            f_content,
                                            f_context,
609
                                            vm
610
611
612
613
614
615
                                        );
                                    }
                                } else {
                                    panic!("unexpected op (expect ireg): {}", op);
                                }

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
616
                            }
617
618
619
620
621
622
623
624

                            op::ConvOp::ZEXT => {
                                if from_ty_size != to_ty_size {
                                    if to_ty_size <= 64 {
                                        self.backend.emit_ubfx(
                                            &tmp_res,
                                            &cast_value(&tmp_op, &to_ty),
                                            0,
625
                                            from_ty_size as u8
626
627
628
629
630
631
632
633
634
                                        );
                                    } else if to_ty_size == 128 {
                                        let (res_l, res_h) = split_int128(&tmp_res, f_context, vm);

                                        // res_l = ZEXT src
                                        self.backend.emit_ubfx(
                                            &res_l,
                                            &cast_value(&tmp_op, &UINT64_TYPE),
                                            0,
635
                                            from_ty_size as u8
636
637
638
639
640
641
642
643
644
645
646
647
648
                                        );
                                        self.backend.emit_mov(&res_h, &XZR); // res_h = 0

                                    } else {
                                        panic!("unexpected int length {}", to_ty_size);
                                    }
                                } else {
                                    // Trivial, just do a move
                                    emit_move_value_to_value(
                                        self.backend.as_mut(),
                                        &tmp_res,
                                        &tmp_op,
                                        f_context,
649
                                        vm
650
651
                                    );
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
652
                            }
653
654
655
656
657
658
659
660

                            op::ConvOp::SEXT => {
                                if from_ty_size != to_ty_size {
                                    if to_ty_size <= 64 {
                                        self.backend.emit_sbfx(
                                            &tmp_res,
                                            &cast_value(&tmp_op, &to_ty),
                                            0,
661
                                            from_ty_size as u8
662
663
664
665
666
667
668
669
670
                                        );
                                    } else if to_ty_size == 128 {
                                        let (res_l, res_h) = split_int128(&tmp_res, f_context, vm);

                                        // res_l = SEXT src
                                        self.backend.emit_sbfx(
                                            &res_l,
                                            &cast_value(&tmp_op, &UINT64_TYPE),
                                            0,
671
                                            from_ty_size as u8
672
                                        );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
673
674
                                        // res_h = ASR src, 63
                                        self.backend.emit_asr_imm(&res_h, &tmp_op, 63);
675
676
677
678
679
680
681
682
683
684
685
686

                                    } else {
                                        panic!("unexpected int length {}", to_ty_size);
                                    }

                                } else {
                                    // Trivial, just do a move
                                    emit_move_value_to_value(
                                        self.backend.as_mut(),
                                        &tmp_res,
                                        &tmp_op,
                                        f_context,
687
                                        vm
688
689
                                    );
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
690
                            }
691
692
693
                            op::ConvOp::REFCAST | op::ConvOp::PTRCAST => {
                                // just a mov (and hopefully reg alloc will coalesce it)
                                self.backend.emit_mov(&tmp_res, &tmp_op);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
694
                            }
695

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
                            op::ConvOp::UITOFP => {
                                if from_ty_size == 128 {
                                    if to_ty_size == 64 {
                                        self.emit_runtime_entry(
                                            &entrypoints::UITOFP_U128_DOUBLE,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
                                            vm
                                        );
                                    } else {
                                        self.emit_runtime_entry(
                                            &entrypoints::UITOFP_U128_FLOAT,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
                                            vm
                                        );
                                    }
717
                                } else {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
718
719
                                    emit_zext(self.backend.as_mut(), &tmp_op);
                                    self.backend.emit_ucvtf(&tmp_res, &tmp_op);
720
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
721
                            }
722

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
                            op::ConvOp::SITOFP => {
                                if from_ty_size == 128 {
                                    if to_ty_size == 64 {
                                        self.emit_runtime_entry(
                                            &entrypoints::SITOFP_I128_DOUBLE,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
                                            vm
                                        );
                                    } else {
                                        self.emit_runtime_entry(
                                            &entrypoints::SITOFP_I128_FLOAT,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
                                            vm
                                        );
                                    }
744
                                } else {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
745
746
                                    emit_sext(self.backend.as_mut(), &tmp_op);
                                    self.backend.emit_scvtf(&tmp_res, &tmp_op);
747
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
748
                            }
749
750
751
752
753
754
755
756
757
758

                            op::ConvOp::FPTOUI => {
                                if to_ty_size == 128 {
                                    if from_ty_size == 64 {
                                        self.emit_runtime_entry(
                                            &entrypoints::FPTOUI_DOUBLE_U128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
759
                                            vm
760
761
762
763
764
765
766
767
                                        );
                                    } else {
                                        self.emit_runtime_entry(
                                            &entrypoints::FPTOUI_FLOAT_U128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
768
                                            vm
769
770
771
772
773
774
                                        );
                                    }
                                } else {
                                    self.backend.emit_fcvtzu(&tmp_res, &tmp_op);
                                    // We have to emmit code to handle the case when the real result
                                    // overflows to_ty_size, but not to_ty_reg_size
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
775
776
777

                                    // The size of the aarch64 register
                                    let to_ty_reg_size = check_op_len(&tmp_res.ty);
778
779
780
781
782
                                    if to_ty_size != to_ty_reg_size {
                                        // Compare the bits of the result after the lower
                                        // to_ty_size bits
                                        self.backend.emit_tst_imm(
                                            &tmp_res,
783
                                            bits_ones(to_ty_reg_size - to_ty_size) << to_ty_size
784
785
786
787
788
789
790
791
                                        );

                                        // If the above condition is true, the an overflow occurred
                                        // So set tmp_res to !0 (i.e. all ones, the maximum value)
                                        self.backend.emit_csinv(
                                            &tmp_res,
                                            &tmp_res,
                                            &get_alias_for_length(XZR.id(), to_ty_size),
792
                                            "EQ"
793
794
795
                                        );
                                    }
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
796
                            }
797
798
799
800
801
802
803
804
805
806

                            op::ConvOp::FPTOSI => {
                                if to_ty_size == 128 {
                                    if from_ty_size == 64 {
                                        self.emit_runtime_entry(
                                            &entrypoints::FPTOSI_DOUBLE_I128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
807
                                            vm
808
809
810
811
812
813
814
815
                                        );
                                    } else {
                                        self.emit_runtime_entry(
                                            &entrypoints::FPTOSI_FLOAT_I128,
                                            vec![tmp_op.clone()],
                                            Some(vec![tmp_res.clone()]),
                                            Some(node),
                                            f_context,
816
                                            vm
817
818
819
820
821
                                        );
                                    }
                                } else {
                                    self.backend.emit_fcvtzs(&tmp_res, &tmp_op);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
822
                                    // TODO This code is horrible and inefficient due to branches
823
824
                                    // and duplication
                                    // is there a better way?
825
826
827

                                    // We have to emmit code to handle the case when the real result
                                    // overflows to_ty_size, but not to_ty_reg_size
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
828
829
                                    // The size of the aarch64 register
                                    let to_ty_reg_size = check_op_len(&tmp_res.ty);
830
                                    if to_ty_size != to_ty_reg_size {
831
832
833
834
835
                                        let blk_positive =
                                            make_block_name(&node.name(), "positive");
                                        let blk_negative =
                                            make_block_name(&node.name(), "negative");
                                        let blk_end = make_block_name(&node.name(), "end");
836
837
838
839
840
                                        let tmp = make_temporary(f_context, to_ty.clone(), vm);

                                        self.backend.emit_tbnz(
                                            &tmp_res,
                                            (to_ty_size - 1) as u8,
841
                                            blk_negative.clone()
842
843
844
845
846
847
                                        );
                                        self.finish_block();

                                        self.start_block(blk_positive.clone());
                                        {
                                            // check to see if the higher bits are the same as the
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
848
849
                                            // sign bit (which is 0), if their not there's an
                                            // overflow
850
851
852
                                            self.backend.emit_tst_imm(
                                                &tmp_res,
                                                bits_ones(to_ty_reg_size - to_ty_size) <<
853
                                                    to_ty_size
854
855
856
857
858
859
860
861
862
863
864
865
866
867
                                            );
                                            self.backend
                                                .emit_mov_imm(&tmp, bits_ones(to_ty_size - 1));

                                            // if the above test fails (i.e. results in zero)
                                            // then set temp_res to tmp
                                            self.backend.emit_csel(&tmp_res, &tmp, &tmp_res, "EQ");

                                            self.backend.emit_b(blk_end.clone());
                                            self.finish_block();
                                        }
                                        self.start_block(blk_negative.clone());
                                        {
                                            self.backend.emit_mvn(&tmp, &tmp_res);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
868
869
870
                                            // check to see if the higher bits of temp are the same
                                            // as the sign bit (which is 1), if their not there's
                                            // an overflow
871
872
873
                                            self.backend.emit_tst_imm(
                                                &tmp_res,
                                                bits_ones(to_ty_reg_size - to_ty_size) <<
874
                                                    to_ty_size
875
876
                                            );

877
878
                                            // Set just the sign bit (this is smallest
                                            // representable signed number)
879
880
                                            self.backend.emit_mov_imm(&tmp, 1 << to_ty_size);

881
882
                                            // if the above test fails (i.e. results in zero),
                                            // then set temp_res to tmp
883
884
885
886
                                            self.backend.emit_csel(&tmp_res, &tmp, &tmp_res, "EQ");
                                            self.finish_block();
                                        }
                                        self.start_block(blk_end.clone());
887

888
889
                                    }
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
890
                            }
891
892
893

                            op::ConvOp::BITCAST => {
                                self.backend.emit_fmov(&tmp_res, &tmp_op);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
894
                            }
895
896
                            op::ConvOp::FPTRUNC | op::ConvOp::FPEXT => {
                                self.backend.emit_fcvt(&tmp_res, &tmp_op);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
897
                            }
898
                        }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
899
                    }
900
901
902
903
904
905
906
907
908
909
910
911

                    Instruction_::Load { order, mem_loc, .. } => {
                        trace!("instsel on LOAD");
                        let ref ops = inst.ops;
                        let ref loc_op = ops[mem_loc];

                        let resolved_loc =
                            self.emit_node_addr_to_value(loc_op, f_content, f_context, vm);
                        let res = self.get_result_value(node, 0);

                        if self.match_ireg(node) || self.match_fpreg(node) {
                            // Whether to use a load acquire
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
912
913
                            let use_acquire = match order {
                                MemoryOrder::Relaxed | MemoryOrder::NotAtomic => false,
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
914
915
                                MemoryOrder::Consume | MemoryOrder::Acquire |
                                MemoryOrder::SeqCst => true,
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
916
                                _ => panic!("didnt expect order {:?} with load inst", order)
917
918
919
920
921
922
923
924
925
                            };


                            if use_acquire {
                                // Can only have a base for a LDAR
                                let temp_loc = emit_mem_base(
                                    self.backend.as_mut(),
                                    &resolved_loc,
                                    f_context,
926
                                    vm
927
                                );
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
928
                                match res.ty.v {
929
930
931
932
933
934
                                    // Have to load a temporary GPR first
                                    MuType_::Float => {
                                        let temp =
                                            make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                        self.backend.emit_ldar(&temp, &temp_loc);
                                        self.backend.emit_fmov(&res, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
935
                                    }
936
937
938
939
940
                                    MuType_::Double => {
                                        let temp =
                                            make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                        self.backend.emit_ldar(&temp, &temp_loc);
                                        self.backend.emit_fmov(&res, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
941
                                    }
942
                                    // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
943
                                    _ => self.backend.emit_ldar(&res, &temp_loc)
944
945
946
947
948
949
950
                                };
                            } else {
                                let temp_loc = emit_mem(
                                    self.backend.as_mut(),
                                    &resolved_loc,
                                    get_type_alignment(&res.ty, vm),
                                    f_context,
951
                                    vm
952
953
954
955
956
957
                                );
                                self.backend.emit_ldr(&res, &temp_loc, false);
                            }
                        } else if self.match_ireg_ex(node) {
                            let (res_l, res_h) = split_int128(&res, f_context, vm);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
958
                            match order {
959
960
961
962
963
964
                                MemoryOrder::NotAtomic => {
                                    let temp_loc = emit_mem(
                                        self.backend.as_mut(),
                                        &resolved_loc,
                                        get_type_alignment(&res.ty, vm),
                                        f_context,
965
                                        vm
966
967
                                    );
                                    self.backend.emit_ldp(&res_l, &res_h, &temp_loc);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
968
                                }
969
970

                                // Aarch64 dosn't have a load acquire pair instruction
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
971
972
                                // So instead we have to write a loop using load/store exclusive
                                // pairs
973
974
                                _ => {
                                    // Whether to use a load exclusive acquire
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
975
976
                                    let use_acquire = match order {
                                        MemoryOrder::Relaxed => false,
977
978
                                        MemoryOrder::Consume |
                                        MemoryOrder::Acquire |
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
979
                                        MemoryOrder::SeqCst => true,
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
980
981
982
983
984
985
                                        _ => {
                                            panic!(
                                                "didnt expect order {:?} with atomic load inst",
                                                order
                                            )
                                        }
986
987
                                    };
                                    // Whether to use a store exclusive release
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
988
                                    let use_release = match order {
989
990
                                        MemoryOrder::Relaxed |
                                        MemoryOrder::Consume |
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
991
992
                                        MemoryOrder::Acquire => false,
                                        MemoryOrder::SeqCst => true,
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
993
994
995
996
997
998
                                        _ => {
                                            panic!(
                                                "didnt expect order {:?} with atomic load inst",
                                                order
                                            )
                                        }
999
1000
1001
1002
1003
1004
1005
                                    };

                                    // Exclusive loads/stores, only supports a base address
                                    let temp_loc = emit_mem_base(
                                        self.backend.as_mut(),
                                        &resolved_loc,
                                        f_context,
1006
                                        vm
1007
1008
1009
1010
                                    );

                                    self.finish_block();

1011
1012
                                    let blk_load_start =
                                        make_block_name(&node.name(), "load_start");
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036

                                    // load_start:
                                    self.start_block(blk_load_start.clone());


                                    // Load the value:
                                    if use_acquire {
                                        self.backend.emit_ldaxp(&res_l, &res_h, &temp_loc);
                                    } else {
                                        self.backend.emit_ldxp(&res_l, &res_h, &temp_loc);
                                    }

                                    let success = make_temporary(f_context, UINT1_TYPE.clone(), vm);

                                    // Store the value we just read back to memory
                                    if use_release {
                                        self.backend
                                            .emit_stlxp(&temp_loc, &success, &res_l, &res_h);
                                    } else {
                                        self.backend.emit_stxp(&temp_loc, &success, &res_l, &res_h);
                                    }

                                    // If the store failed, then branch back to 'load_start:'
                                    self.backend.emit_cbnz(&success, blk_load_start.clone())
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1037
                                }
1038
1039
1040
1041
                            }
                        } else {
                            unimplemented!();
                        }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1042
                    }
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059

                    Instruction_::Store {
                        order,
                        mem_loc,
                        value,
                        ..
                    } => {
                        trace!("instsel on STORE");
                        let ref ops = inst.ops;
                        let ref loc_op = ops[mem_loc];
                        let ref val_op = ops[value];

                        let resolved_loc =
                            self.emit_node_addr_to_value(loc_op, f_content, f_context, vm);

                        if self.match_ireg(val_op) || self.match_fpreg(val_op) {
                            // Whether to use a store release or not
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1060
1061
1062
1063
                            let use_release = match order {
                                MemoryOrder::Relaxed | MemoryOrder::NotAtomic => false,
                                MemoryOrder::Release | MemoryOrder::SeqCst => true,
                                _ => panic!("didnt expect order {:?} with load inst", order)
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
                            };

                            let val = self.emit_reg(val_op, f_content, f_context, vm);

                            if use_release {
                                // Can only have a base for a STLR
                                let temp_loc = emit_mem_base(
                                    self.backend.as_mut(),
                                    &resolved_loc,
                                    f_context,
1074
                                    vm
1075
1076
                                );

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1077
                                match val.ty.v {
1078
1079
1080
1081
1082
1083
                                    // Have to store a temporary GPR
                                    MuType_::Float => {
                                        let temp =
                                            make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                        self.backend.emit_fmov(&temp, &val);
                                        self.backend.emit_stlr(&temp_loc, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1084
                                    }
1085
1086
1087
1088
1089
                                    MuType_::Double => {
                                        let temp =
                                            make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                        self.backend.emit_fmov(&temp, &val);
                                        self.backend.emit_stlr(&temp_loc, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1090
                                    }
1091
                                    // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1092
                                    _ => self.backend.emit_stlr(&temp_loc, &val)
1093
1094
1095
1096
1097
1098
1099
                                };
                            } else {
                                let temp_loc = emit_mem(
                                    self.backend.as_mut(),
                                    &resolved_loc,
                                    get_type_alignment(&val.ty, vm),
                                    f_context,
1100
                                    vm
1101
1102
1103
1104
1105
1106
1107
                                );
                                self.backend.emit_str(&temp_loc, &val);
                            }
                        } else if self.match_ireg_ex(val_op) {
                            let (val_l, val_h) =
                                self.emit_ireg_ex(val_op, f_content, f_context, vm);

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1108
                            match order {
1109
1110
1111
1112
1113
1114
                                MemoryOrder::NotAtomic => {
                                    let temp_loc = emit_mem(
                                        self.backend.as_mut(),
                                        &resolved_loc,
                                        16,
                                        f_context,
1115
                                        vm
1116
1117
                                    );
                                    self.backend.emit_stp(&temp_loc, &val_l, &val_h);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1118
                                }
1119
1120

                                // Aarch64 dosn't have a store release pair instruction
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1121
1122
                                // So instead we have to write a loop using load/store exclusive
                                // pairs
1123
1124
                                _ => {
                                    // Whether to use a load exclusive acquire
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1125
1126
1127
                                    let use_acquire = match order {
                                        MemoryOrder::Relaxed | MemoryOrder::Release => false,
                                        MemoryOrder::SeqCst => true,
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1128
1129
1130
1131
1132
1133
                                        _ => {
                                            panic!(
                                                "didnt expect order {:?} with atomic store inst",
                                                order
                                            )
                                        }
1134
1135
                                    };
                                    // Whether to use a store exclusive release
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1136
1137
1138
                                    let use_release = match order {
                                        MemoryOrder::Relaxed => false,
                                        MemoryOrder::Release | MemoryOrder::SeqCst => true,
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1139
1140
1141
1142
1143
1144
                                        _ => {
                                            panic!(
                                                "didnt expect order {:?} with atomic store inst",
                                                order
                                            )
                                        }
1145
1146
1147
1148
1149
1150
1151
                                    };

                                    // Exclusive loads/stores, only supports a base address
                                    let temp_loc = emit_mem_base(
                                        self.backend.as_mut(),
                                        &resolved_loc,
                                        f_context,
1152
                                        vm
1153
1154
1155
1156
                                    );

                                    self.finish_block();

1157
1158
                                    let blk_store_start =
                                        make_block_name(&node.name(), "store_start");
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181

                                    // store_start:
                                    self.start_block(blk_store_start.clone());

                                    let success = make_temporary(f_context, UINT1_TYPE.clone(), vm);
                                    let discard_reg = cast_value(&success, &UINT64_TYPE);
                                    // Load a value (discard it)
                                    if use_acquire {
                                        self.backend.emit_ldaxp(&XZR, &discard_reg, &temp_loc);
                                    } else {
                                        self.backend.emit_ldxp(&XZR, &discard_reg, &temp_loc);
                                    }

                                    // Store the value
                                    if use_release {
                                        self.backend
                                            .emit_stlxp(&temp_loc, &success, &val_l, &val_h);
                                    } else {
                                        self.backend.emit_stxp(&temp_loc, &success, &val_l, &val_h);
                                    }

                                    // If the store failed, then branch back to 'store_start:'
                                    self.backend.emit_cbnz(&success, blk_store_start.clone())
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1182
                                }
1183
1184
1185
1186
1187
                            }
                        } else {
                            unimplemented!();
                        }

Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1188
                    }
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200

                    Instruction_::CmpXchg {
                        is_weak,
                        success_order,
                        fail_order,
                        mem_loc,
                        expected_value,
                        desired_value,
                        ..
                    } => {
                        // Note: this uses the same operations as GCC (for the C++ atomic cmpxchg)
                        // Clang is slightly different and ignores the 'fail_order'
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1201
1202
                        let use_acquire = match fail_order {
                            MemoryOrder::Acquire | MemoryOrder::SeqCst => true,
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
                            MemoryOrder::Relaxed => {
                                match success_order {
                                    MemoryOrder::Acquire |
                                    MemoryOrder::AcqRel |
                                    MemoryOrder::SeqCst => true,
                                    MemoryOrder::Relaxed | MemoryOrder::Release => false,
                                    _ => {
                                        panic!(
                                            "didnt expect success order {:?} for cmpxchg",
                                            success_order
                                        )
                                    }
1215
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1216
                            }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1217
                            _ => panic!("didnt expect fail order {:?} for cmpxchg", fail_order)
1218
                        };
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1219
                        let use_release = match fail_order {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
                            MemoryOrder::Acquire => {
                                match success_order {
                                    MemoryOrder::Relaxed |
                                    MemoryOrder::Release |
                                    MemoryOrder::AcqRel |
                                    MemoryOrder::SeqCst => true,
                                    MemoryOrder::Acquire => false,
                                    _ => {
                                        panic!(
                                            "didnt expect success order {:?} for cmpxchg",
                                            success_order
                                        )
                                    }
                                }
                            }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1235
                            MemoryOrder::SeqCst => true,
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
                            MemoryOrder::Relaxed => {
                                match success_order {
                                    MemoryOrder::Release |
                                    MemoryOrder::AcqRel |
                                    MemoryOrder::SeqCst => true,
                                    MemoryOrder::Relaxed | MemoryOrder::Acquire => false,
                                    _ => {
                                        panic!(
                                            "didnt expect success order {:?} for cmpxchg",
                                            success_order
                                        )
                                    }
1248
                                }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1249
                            }
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1250
                            _ => panic!("didnt expect fail order {:?} for cmpxchg", fail_order)
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
                        };


                        let ref ops = inst.ops;
                        let loc =
                            self.emit_node_addr_to_value(&ops[mem_loc], f_content, f_context, vm);
                        let expected =
                            self.emit_reg(&ops[expected_value], f_content, f_context, vm);
                        let desired = self.emit_reg(&ops[desired_value], f_content, f_context, vm);

                        let res_value = self.get_result_value(node, 0);
                        let res_success = self.get_result_value(node, 1);

1264
1265
                        let blk_cmpxchg_start = make_block_name(&node.name(), "cmpxchg_start");
                        let blk_cmpxchg_failed = make_block_name(&node.name(), "cmpxchg_failed");
1266
                        let blk_cmpxchg_succeded =
1267
                            make_block_name(&node.name(), "cmpxchg_succeded");
1268
1269
1270
1271
1272
1273
1274

                        self.finish_block();

                        // cmpxchg_start:
                        self.start_block(blk_cmpxchg_start.clone());

                        if use_acquire {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1275
                            match res_value.ty.v {
1276
1277
1278
1279
1280
                                // Have to load a temporary GPR first
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_ldaxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1281
                                }
1282
1283
1284
1285
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_ldaxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1286
                                }
1287
                                // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1288
                                _ => self.backend.emit_ldaxr(&res_value, &loc)
1289
1290
                            };
                        } else {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1291
                            match res_value.ty.v {
1292
1293
1294
1295
1296
                                // Have to load a temporary GPR first
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_ldxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1297
                                }
1298
1299
1300
1301
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_ldxr(&temp, &loc);
                                    self.backend.emit_fmov(&res_value, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1302
                                }
1303
                                // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1304
                                _ => self.backend.emit_ldxr(&res_value, &loc)
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
                            };
                        }

                        if is_int_reg(&expected) {
                            self.backend.emit_cmp(&res_value, &expected);
                        } else {
                            self.backend.emit_fcmp(&res_value, &expected);
                        }
                        self.backend.emit_b_cond("NE", blk_cmpxchg_failed.clone());

                        if use_release {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1316
                            match desired.ty.v {
1317
1318
1319
1320
1321
                                // Have to store a temporary GPR
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stlxr(&loc, &res_success, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1322
                                }
1323
1324
1325
1326
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stlxr(&loc, &res_success, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1327
                                }
1328
                                // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1329
                                _ => self.backend.emit_stlxr(&loc, &res_success, &desired)
1330
1331
                            };
                        } else {
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1332
                            match desired.ty.v {
1333
1334
1335
1336
1337
                                // Have to store a temporary GPR
                                MuType_::Float => {
                                    let temp = make_temporary(f_context, UINT32_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stxr(&loc, &res_success, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1338
                                }
1339
1340
1341
1342
                                MuType_::Double => {
                                    let temp = make_temporary(f_context, UINT64_TYPE.clone(), vm);
                                    self.backend.emit_fmov(&temp, &desired);
                                    self.backend.emit_stxr(&loc, &res_success, &temp);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1343
                                }
1344
                                // Can load the register directly
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1345
                                _ => self.backend.emit_stxr(&loc, &res_success, &desired)
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
                            };
                        }

                        if !is_weak {
                            // Store failed, try again
                            self.backend
                                .emit_cbnz(&res_success, blk_cmpxchg_start.clone());
                        }

                        self.backend.emit_b(blk_cmpxchg_succeded.clone());

                        self.finish_block();

                        // cmpxchg_failed:
                        self.start_block(blk_cmpxchg_failed.clone());

                        self.backend.emit_clrex();
                        // Set res_success to 1 (the same value STXR/STLXR uses to indicate failure)
                        self.backend.emit_mov_imm(&res_success, 1);

                        self.finish_block();

                        // cmpxchg_succeded:
                        self.start_block(blk_cmpxchg_succeded.clone());
1370
1371
                        // this NOT is needed as STXR/STLXR returns sucess as '0',
                        // wheras the Mu spec says it should be 1
1372
                        self.backend.emit_eor_imm(&res_success, &res_success, 1);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1373
                    }
1374
1375
1376
1377
1378
1379
1380
1381
1382
                    Instruction_::GetIRef(_) |
                    Instruction_::GetFieldIRef { .. } |
                    Instruction_::GetElementIRef { .. } |
                    Instruction_::GetVarPartIRef { .. } |
                    Instruction_::ShiftIRef { .. } => {
                        trace!("instsel on GET/FIELD/VARPARTIREF, SHIFTIREF");
                        let mem_addr = self.emit_get_mem_from_inst(node, f_content, f_context, vm);
                        let tmp_res = self.get_result_value(node, 0);
                        emit_calculate_address(self.backend.as_mut(), &tmp_res, &mem_addr, vm);
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1383
                    }
1384
1385
1386
1387