cleanup and slightly improve speed

This gets rid of a lot of the compiler internals that aren't needed. It also changes the representation of `FixedSizeArray` to a `Memory` rather than a `MemoryRef` because the `MemoryRef` was only needed to allow the backing to change.

src/FixedSizeArrays.jl

@@ -3,36 +3,28 @@ module FixedSizeArrays
 export FixedSizeArray, FixedSizeVector, FixedSizeMatrix
 
 mutable struct FixedSizeArray{T,N} <: DenseArray{T,N}
-    ref::MemoryRef{T}
+    mem::Memory{T}
     const size::NTuple{N,Int}
 end
 
 const FixedSizeVector{T} = FixedSizeArray{T,1}
 const FixedSizeMatrix{T} = FixedSizeArray{T,2}
 
-eval(:(function (self::Type{FixedSizeArray{T,N}})(::UndefInitializer, size::Vararg{Int,N}) where {T,N}
-    mem = fieldtype(fieldtype(self, :ref), :mem)(undef, prod(size))
-    return $(Expr(:new, :self, :(Core.memoryref(mem)), :(size)))
-end))
+function (self::Type{FixedSizeArray{T,N}})(::UndefInitializer, size::Vararg{Int,N}) where {T,N}
+    return FixedSizeArray(Memory{T}(undef, prod(size)), size)
+end
 
-function Base.setindex!(A::FixedSizeArray{T}, x, i::Int) where {T}
-    Base.@_noub_if_noinbounds_meta
-    @boundscheck (i - 1)%UInt < length(A)%UInt || throw_boundserror(A, (i,))
-    Core.memoryrefset!(Core.memoryref(A.ref, i, false), x isa T ? x : convert(T,x)::T, :not_atomic, false)
+Base.@propagate_inbounds function Base.setindex!(A::FixedSizeArray{T}, x, i::Int) where {T}
+    getfield(A, :mem)[i] = x
     return A
 end
-function Base.setindex!(A::FixedSizeArray{T}, x, i1::Int, i2::Int, I::Int...) where {T}
-    @inline
-    Base.@_noub_if_noinbounds_meta
+Base.@inline function Base.setindex!(A::FixedSizeArray{T}, x, i1::Int, i2::Int, I::Int...) where {T}
     @boundscheck checkbounds(A, i1, i2, I...) # generally _to_linear_index requires bounds checking
-    Core.memoryrefset!(Core.memoryref(A.ref, Base._to_linear_index(A, i1, i2, I...), false), x isa T ? x : convert(T,x)::T, :not_atomic, false)
+    getfield(A, :mem)[Base._to_linear_index(A, i1, i2, I...)] = x
     return A
 end
-
-function Base.getindex(A::FixedSizeArray, i::Int)
-    Base.@_noub_if_noinbounds_meta
-    @boundscheck Base.ult_int(Base.bitcast(UInt, Base.sub_int(i, 1)), Base.bitcast(UInt, length(A))) || throw_boundserror(A, (i,))
-    Core.memoryrefget(Core.memoryref(getfield(A, :ref), i, false), :not_atomic, false)
+Base.@propagate_inbounds function Base.getindex(A::FixedSizeArray, i::Int)
+    getfield(A, :mem)[i]
 end
 function Base.getindex(A::FixedSizeArray, i1::Int, i2::Int, I::Int...)
     @inline