Faster code when there are unnecessary byte order conversions

I was writing an AES implementation for exercise, and I accidentally discovered that I can get the code to run faster if I do some unnecessary byte order conversions.

After some investigation, I managed to create a minimal reproduction: https://play.rust-lang.org/?version=stable&mode=release&edition=2021&gist=76e707c6ebd45bb95fea189c846731f1

pub fn foo(a: &mut [u32; 4], s: &[u32; 256]) -> u32 {
    let b = a.map(|x| x.to_le_bytes());
    let mut x = 0;
    for i in 0..4 {
        for j in 0..4 {
            x ^= s[b[i][j] as usize];
        }
    }
    x
}

pub fn bar(a: &mut [u32; 4], s: &[u32; 256]) -> u32 {
    let b = a.map(|x| x.to_be_bytes());
    let mut x = 0;
    for i in 0..4 {
        for j in 0..4 {
            x ^= s[b[i][j] as usize];
        }
    }
    x
}

You can see that foo and bar are doing the same thing, but in bar there is an unnecessary byte order conversion (on little-endian systems like x86). You can also generate ASM codes in the playground online:

playground::foo:
\tmovups\t(%rdi), %xmm0
\tmovaps\t%xmm0, -24(%rsp)
\tmovzbl\t-24(%rsp), %ecx
\tmovzbl\t-23(%rsp), %eax
\tmovl\t(%rsi,%rax,4), %eax
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-22(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-21(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-20(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-19(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-18(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-17(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-16(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-15(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-14(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-13(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-12(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-11(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-10(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovzbl\t-9(%rsp), %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tretq

playground::bar:
\tmovl\t(%rdi), %ecx
\tbswapl\t%ecx
\tmovzbl\t%cl, %edx
\tmovzbl\t%ch, %eax
\tmovl\t(%rsi,%rax,4), %eax
\txorl\t(%rsi,%rdx,4), %eax
\tmovl\t%ecx, %edx
\tshrl\t$16, %edx
\tmovzbl\t%dl, %edx
\txorl\t(%rsi,%rdx,4), %eax
\tshrl\t$24, %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovl\t4(%rdi), %ecx
\tbswapl\t%ecx
\tmovzbl\t%cl, %edx
\txorl\t(%rsi,%rdx,4), %eax
\tmovzbl\t%ch, %edx
\txorl\t(%rsi,%rdx,4), %eax
\tmovl\t%ecx, %edx
\tshrl\t$16, %edx
\tmovzbl\t%dl, %edx
\txorl\t(%rsi,%rdx,4), %eax
\tshrl\t$24, %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovl\t8(%rdi), %ecx
\tbswapl\t%ecx
\tmovzbl\t%cl, %edx
\txorl\t(%rsi,%rdx,4), %eax
\tmovzbl\t%ch, %edx
\txorl\t(%rsi,%rdx,4), %eax
\tmovl\t%ecx, %edx
\tshrl\t$16, %edx
\tmovzbl\t%dl, %edx
\txorl\t(%rsi,%rdx,4), %eax
\tshrl\t$24, %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tmovl\t12(%rdi), %ecx
\tbswapl\t%ecx
\tmovzbl\t%cl, %edx
\txorl\t(%rsi,%rdx,4), %eax
\tmovzbl\t%ch, %edx
\txorl\t(%rsi,%rdx,4), %eax
\tmovl\t%ecx, %edx
\tshrl\t$16, %edx
\tmovzbl\t%dl, %edx
\txorl\t(%rsi,%rdx,4), %eax
\tshrl\t$24, %ecx
\txorl\t(%rsi,%rcx,4), %eax
\tretq

There are 34 memory accesses in foo (17 of them are stack accesses) but only 20 in bar.

As a result of the additional memory accesses, despite the extra byte order conversion, Criterion shows that bar is 1.6x faster than foo on my machine.

Any idea why the byte order conversion has an impact on stack memory accesses? It seems really strange.