precompile first interpretation #353

glou-nes · 2024-12-09T19:57:37Z

Partial fix #348
remove a assertion in XLA. Get some result.
Now:

using Reactant.XLA
cpu = XLA.CPUClient()
x = Reactant.ConcreteRArray(randn(Float64, 2, 2); client=cpu)
@time @code_hlo sum(x)
5.14s

On master:

 61.77 s

Use SnoopCompile to track invalidation, get several invalidation in the abstract interpreter. I already get similar invalidation because of REPL abstract interpreter in Julia 1.12. I don't know how to fix it. Some in the list are probably easier to handle.

lock_mi_inference(interp::GPUCompiler.GPUInterpreter, mi::Core.MethodInstance) @ GPUCompiler /depot/packages/GPUCompiler/GnbhK/src/jlgen.jl:405
unlock_mi_inference(interp::GPUCompiler.GPUInterpreter, mi::Core.MethodInstance) @ GPUCompiler /depot/packages/GPUCompiler/GnbhK/src/jlgen.jl:406
add_remark!(interp::GPUCompiler.GPUInterpreter, sv::Core.Compiler.InferenceState, msg) @ GPUCompiler /depot/packages/GPUCompiler/GnbhK/src/jlgen.jl:408
may_compress(interp::GPUCompiler.GPUInterpreter) @ GPUCompiler /depot/packages/GPUCompiler/GnbhK/src/jlgen.jl:413
may_discard_trees(interp::GPUCompiler.GPUInterpreter) @ GPUCompiler /depot/packages/GPUCompiler/GnbhK/src/jlgen.jl:414
verbose_stmt_info(interp::GPUCompiler.GPUInterpreter) @ GPUCompiler /depot/packages/GPUCompiler/GnbhK/src/jlgen.jl:415
abstract_call_known(interp::Enzyme.Compiler.Interpreter.EnzymeInterpreter{Handler}, f, arginfo::Core.Compiler.ArgInfo, si::Core.Compiler.StmtInfo, sv::Union{Core.Compiler.IRInterpretationState, Core.Compiler.InferenceState}, max_methods::Int64) where Handler @ Enzyme.Compiler.Interpreter /depot/packages/Enzyme/6C71q/src/compiler/interpreter.jl:906
inlining_policy(interp::Enzyme.Compiler.Interpreter.EnzymeInterpreter, src, info::Core.Compiler.CallInfo, stmt_flag::UInt32) @ Enzyme.Compiler.Interpreter /depot/packages/Enzyme/6C71q/src/compiler/interpreter.jl:427
get_inference_cache(interp::REPL.REPLCompletions.REPLInterpreter) @ REPL.REPLCompletions /code/usr/share/julia/stdlib/v1.11/REPL/src/REPLCompletions.jl:569
ifelse(pred::Reactant.TracedRNumber{Bool}, x::Reactant.TracedRNumber{T}, y::Reactant.TracedRNumber{T}) where T @ Reactant /depot/dev/Reactant/src/TracedRNumber.jl:198
active_reg_inner(::Type{Reactant.TracedRArray{T, N}}, seen::ST, world::Union{Nothing, UInt64}, ::Val{justActive}, ::Val{UnionSret}) where {ST, T, N, justActive, UnionSret} @ Reactant /depot/dev/Reactant/src/TracedRArray.jl:418
active_reg_inner(::Type{Reactant.TracedRArray{T, N}}, seen::ST, world::Union{Nothing, UInt64}, ::Val{justActive}) where {ST, T, N, justActive} @ Reactant /depot/dev/Reactant/src/TracedRArray.jl:418
active_reg_inner(::Type{Reactant.TracedRArray{T, N}}, seen::ST, world::Union{Nothing, UInt64}) where {ST, T, N} @ Reactant /depot/dev/Reactant/src/TracedRArray.jl:418
kwcall(::NamedTuple, ::typeof(printstyled), io::Base.AnnotatedIOBuffer, msg...) @ StyledStrings.Legacy /code/usr/share/julia/stdlib/v1.11/StyledStrings/src/legacy.jl:126
write(filename::String, jit::Reactant.MLIR.IR.ExecutionEngine) @ Reactant.MLIR.IR /depot/dev/Reactant/src/mlir/IR/ExecutionEngine.jl:59
peek(io::StringEncodings.StringDecoder) @ YAML /depot/packages/YAML/XZZ1Y/src/buffered_input.jl:78
displaysize(t::REPL.Terminals.UnixTerminal) @ REPL.Terminals /code/usr/share/julia/stdlib/v1.11/REPL/src/Terminals.jl:153
displaysize(::REPL.Terminals.TextTerminal) @ REPL.Terminals /code/usr/share/julia/stdlib/v1.11/REPL/src/Terminals.jl:48
convert(::Type{Reactant.TracedRNumber{T}}, x::Number) where T @ Reactant /depot/dev/Reactant/src/TracedRNumber.jl:56
*(a::ObjectFile.fixed_string, b::String) @ ObjectFile /depot/packages/ObjectFile/eLzIt/src/string_utils.jl:52
lookup(ip::UInt64) @ Profile /code/usr/share/julia/stdlib/v1.11/Profile/src/Profile.jl:45
convert(::Type{Ptr{T}}, val::LLVM.GenericValue) where T @ LLVM /depot/packages/LLVM/wMjUU/src/execution.jl:103
show(io::IO, c::Base.AnnotatedChar) @ StyledStrings /code/usr/share/julia/stdlib/v1.11/StyledStrings/src/io.jl:286
ci_cache_token(job::GPUCompiler.CompilerJob{<:Any, <:Enzyme.Compiler.AbstractEnzymeCompilerParams}) @ Enzyme.Compiler /depot/packages/Enzyme/6C71q/src/compiler.jl:1203
kwcall(::NamedTuple, ::typeof(GPUCompiler.codegen), output::Symbol, job::GPUCompiler.CompilerJob{<:Enzyme.Compiler.EnzymeTarget}) @ Enzyme.Compiler /depot/packages/Enzyme/6C71q/src/compiler.jl:3223
get_inference_world(interp::REPL.REPLCompletions.REPLInterpreter) @ REPL.REPLCompletions /code/usr/share/julia/stdlib/v1.11/REPL/src/REPLCompletions.jl:568
may_optimize(::REPL.REPLCompletions.REPLInterpreter) @ REPL.REPLCompletions /code/usr/share/julia/stdlib/v1.11/REPL/src/REPLCompletions.jl:573
get_interpreter(job::GPUCompiler.CompilerJob{<:Any, <:Enzyme.Compiler.AbstractEnzymeCompilerParams}) @ Enzyme.Compiler /depot/packages/Enzyme/6C71q/src/compiler.jl:1212
eltype(::Type{<:EnzymeCore.RArray{T}}) where T @ EnzymeCore /depot/packages/EnzymeCore/15Zff/src/EnzymeCore.jl:740
convert(::Type{Array{T, N}}, X::Enzyme.TupleArray{T, Shape, Length, N}) where {T, Shape, Length, N} @ Enzyme /depot/packages/Enzyme/6C71q/src/sugar.jl:455
runtime_module(::GPUCompiler.CompilerJob{<:Any, <:Enzyme.Compiler.AbstractEnzymeCompilerParams}) @ Enzyme.Compiler /depot/packages/Enzyme/6C71q/src/compiler.jl:1185
promote_rule(::Type{T}, ::Type{Reactant.TracedRNumber{S}}) where {T, S} @ Reactant /depot/dev/Reactant/src/TracedRNumber.jl:52
print(io::IO, s::Union{Base.AnnotatedString, SubString{<:Base.AnnotatedString}}) @ StyledStrings /code/usr/share/julia/stdlib/v1.11/StyledStrings/src/io.jl:261
==(lhs, rhs::Reactant.TracedRNumber{T}) where T @ Reactant /depot/dev/Reactant/src/TracedRNumber.jl:166
print(io::IO, c::Base.AnnotatedChar) @ StyledStrings /code/usr/share/julia/stdlib/v1.11/StyledStrings/src/io.jl:284
convert(::Type{T}, x::Union{ConcreteRArray{T, 0}, ConcreteRNumber{T}}) where T @ Reactant /depot/dev/Reactant/src/ConcreteRArray.jl:126
Core.Compiler.InferenceParams(interp::REPL.REPLCompletions.REPLInterpreter) @ REPL.REPLCompletions /code/usr/share/julia/stdlib/v1.11/REPL/src/REPLCompletions.jl:566
==(lhs::Reactant.TracedRNumber{T}, rhs) where T @ Reactant /depot/dev/Reactant/src/TracedRNumber.jl:157

mofeing

Thanks!

Would you mind providing comptimes of first interpretation before and after this PR? Times can vary between machines.

Use SnoopCompile to track invalidation, get several invalidation in the abstract interpreter.

Yeah, that's why there's still time expent on compilation on 2nd, 3rd, ... calls to @code_hlo. But that's an issue for another PR.

mofeing · 2024-12-09T20:02:15Z

Project.toml

@@ -47,6 +48,7 @@ GPUArraysCore = "0.1.6, 0.2"
 LinearAlgebra = "1.10"
 NNlib = "0.9.24"
 OrderedCollections = "1"
+PrecompileTools = "1.2.1"


Do we need to be so exact with the compat version? I prefer leaving it as open as we can to avoid compat conflicts with other libs.

will relax it

mofeing · 2024-12-09T20:07:55Z

src/precompile.jl

+    XLA.__init__()
+    @compile_workload begin
+        x = Reactant.ConcreteRArray(randn(Float64, 2, 2))
+        @jit sum(x)


Don't do @jit but @code_hlo optimize=false. Problem is first interpretation so just tracing, not compilation.

Also, would you mind checking if XLA.__init__() and Reactant.__init__() are still required with @code_hlo optimize=false? I feel a lil bit nervous with them due to relocation.

Furthermore, it might be more interesting to use GPUCompiler.precompile as @vchuravy suggested to me because the interpreter we used is based on GPUCompiler.

it actually is not based on gpucompiler (the enzyme interp is not a descendant of gpuinterpreter)

mofeing · 2024-12-09T20:08:10Z

src/XLA.jl

@@ -44,7 +44,7 @@ const cpuclientcount = Ref(0)
 # TODO synchronization when async is not working because `future` in `ConcreteRArray` is always `nothing`
 function CPUClient(asynchronous=false, node_id=0, num_nodes=1)
    global cpuclientcount
-    @assert cpuclientcount[] == 0
+    #@assert cpuclientcount[] == 0


mmm is this ok @wsmoses?

also, why is this assert failing?

@mofeing One create a client for precompilation. And one when we init.

wsmoses · 2024-12-09T22:37:20Z

src/precompile.jl

+
+@setup_workload begin
+    Reactant.__init__()
+    XLA.__init__()


can we do this without init and just making our own cpuclient?

and ideally registry

this would avoid some of the weird "globals" problems in precompilation

in particular if used we need to carefully init and uninit globals: https://github.com/JuliaGPU/GPUCompiler.jl/blob/a94e6e2b7102f18e4b9aa08c623357a79ac62486/src/precompile.jl#L40

yeah, like some kind of MockClient right?

I mean we could literally make a real Cpuclient which isn’t the global client, and make sure to free it

I tried using ScopedValue to proper deal with XLA.Client. But it's quite instable.

ConcreteRArray and related constructors should take an optional client argument

Using the client argument works extremely well. It goes down to 3s for @code_hlo. I just need now to deal with Client finalizer.

You don't need to use the client if you just do the Base.code_ircode I explained below.

mofeing · 2024-12-10T07:57:21Z

so i just tried and check that just calling Base.code_ircode is sufficient, so we don't need any Client to initialize and deinitialize.

julia> interp = Reactant.ReactantInterpreter()

julia> @time Base.code_ircode(sum, (Reactant.TracedRArray{Float64,2},); interp)
 52.089402 seconds (101.97 M allocations: 5.132 GiB, 2.34% gc time, 100.00% compilation time)
...

julia> @time Base.code_ircode(sum, (Reactant.TracedRArray{Float64,2},); interp)
  0.000488 seconds (1.53 k allocations: 67.750 KiB, 92.79% compilation time)
...

julia> @time Base.code_ircode(*, (Reactant.TracedRArray{Int64,2},Reactant.TracedRArray{Int64,2}); interp)
  0.829383 seconds (659.11 k allocations: 33.190 MiB, 99.99% compilation time)
...

wsmoses · 2024-12-10T08:09:14Z

Bear in mind as well though that the absint will (hopefully) cache between calls

glou-nes · 2024-12-10T12:26:13Z

@setup_workload begin
    @compile_workload begin
        Reactant.__init__()
        cpu = XLA.CPUClient()
        x = Reactant.ConcreteRArray(randn(Float64, 2, 2); client=cpu)
        @code_hlo optimize = false sum(x)
    end

    @compile_workload begin
        interp = Reactant.ReactantInterpreter()
        Base.code_ircode(sum, (Reactant.TracedRArray{Float64,2},); interp)
    end
end

I test the precompile with the following:

using Reactant.XLA
cpu = XLA.CPUClient()
x = Reactant.ConcreteRArray(randn(Float64, 2, 2); client=cpu)
t = @elapsed @code_hlo sum(x)

I suppose we can use both of them.

Compile Workload	Time
code_hlo 1	5.28
Abstract Interpreter 2	6.99
1 + 2	5.14

src/precompile.jl

mofeing

Everything's fine from my side then. I'm approving but you need to fix the tests first. There is a problem with the file Enzyme.jl which is quite weird because you haven't touch it? dialects? (Nightly and ubuntu x86-32 bits are ok to fail)

wsmoses · 2024-12-10T20:06:16Z

So the direct call to ircode is about to get removed, so this should ideally do the codehlo call or equivalent

mofeing · 2024-12-10T21:52:48Z

So the direct call to ircode is about to get removed, so this should ideally do the codehlo call or equivalent

ah really? do we have another mechanism now?

glou-nes · 2024-12-11T19:45:06Z

@mofeing I got errors for macOS 'ERROR: LoadError: StackOverflowError' and the strange Enzyme one with 1.10. Should we use precompilation only with julia 1.11 and excluding MacOS?

mofeing · 2024-12-11T20:34:02Z

@mofeing I got errors for macOS 'ERROR: LoadError: StackOverflowError' and the strange Enzyme one with 1.10. Should we use precompilation only with julia 1.11 and excluding MacOS?

mmm in what arch? I just tried in aarch64-macos and got another error

also CI says that there are also problems in linux

julia> @jit sum(a)
Internal error: encountered unexpected error in runtime:
MethodError(f=Core.Compiler.:(⊑), args=(Core.Compiler.JLTypeLattice(), (:stmt, :line), Core.TypeofBottom()), world=0x00000000000015ac)
jl_method_error_bare at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:2208
jl_method_error at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:2226
jl_lookup_generic_ at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:3058 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:3073
⊑ at ./compiler/typelattice.jl:530
⊑ at ./compiler/typelattice.jl:508
⊑ at ./compiler/typelattice.jl:397 [inlined]
reprocess_instruction! at ./compiler/ssair/irinterp.jl:169
#_ir_abstract_constant_propagation#511 at ./compiler/ssair/irinterp.jl:251
_ir_abstract_constant_propagation at ./compiler/ssair/irinterp.jl:206
ir_abstract_constant_propagation at ./compiler/ssair/irinterp.jl:384
semi_concrete_eval_call at ./compiler/abstractinterpretation.jl:1168
abstract_call_method_with_const_args at ./compiler/abstractinterpretation.jl:812
abstract_call_method_with_const_args at ./compiler/abstractinterpretation.jl:788
abstract_call_gf_by_type at ./compiler/abstractinterpretation.jl:103
abstract_call_known at ./compiler/abstractinterpretation.jl:2087
abstract_call at ./compiler/abstractinterpretation.jl:2169
abstract_call at ./compiler/abstractinterpretation.jl:2162
abstract_call at ./compiler/abstractinterpretation.jl:2354
abstract_eval_call at ./compiler/abstractinterpretation.jl:2370
abstract_eval_statement_expr at ./compiler/abstractinterpretation.jl:2380
abstract_eval_statement at ./compiler/abstractinterpretation.jl:2624
abstract_eval_basic_statement at ./compiler/abstractinterpretation.jl:2913
typeinf_local at ./compiler/abstractinterpretation.jl:3098
typeinf_nocycle at ./compiler/abstractinterpretation.jl:3186
_typeinf at ./compiler/typeinfer.jl:247
typeinf at ./compiler/typeinfer.jl:216
typeinf_edge at ./compiler/typeinfer.jl:930
abstract_call_method at ./compiler/abstractinterpretation.jl:629
abstract_call_gf_by_type at ./compiler/abstractinterpretation.jl:95
abstract_call_unknown at ./compiler/abstractinterpretation.jl:2156
abstract_call at ./compiler/abstractinterpretation.jl:2166
abstract_call at ./compiler/abstractinterpretation.jl:2162
abstract_call at ./compiler/abstractinterpretation.jl:2354
abstract_eval_call at ./compiler/abstractinterpretation.jl:2370
abstract_eval_statement_expr at ./compiler/abstractinterpretation.jl:2380
abstract_eval_statement at ./compiler/abstractinterpretation.jl:2624
abstract_eval_basic_statement at ./compiler/abstractinterpretation.jl:2889
typeinf_local at ./compiler/abstractinterpretation.jl:3098
typeinf_nocycle at ./compiler/abstractinterpretation.jl:3186
_typeinf at ./compiler/typeinfer.jl:247
typeinf at ./compiler/typeinfer.jl:216
typeinf_ext at ./compiler/typeinfer.jl:1051
typeinf_ext_toplevel at ./compiler/typeinfer.jl:1082
typeinf_ext_toplevel at ./compiler/typeinfer.jl:1078
jfptr_typeinf_ext_toplevel_35906.1 at /Users/mofeing/.julia/juliaup/julia-1.10.7+0.aarch64.apple.darwin14/lib/julia/sys.dylib (unknown line)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:3077
jl_apply at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/./julia.h:1982 [inlined]
jl_type_infer at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:394
jl_generate_fptr_impl at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/jitlayers.cpp:504
jl_compile_method_internal at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:2481
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:2887 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:3077
#make_mlir_fn#25 at /Users/mofeing/Developer/Reactant.jl/src/utils.jl:120
make_mlir_fn at /Users/mofeing/Developer/Reactant.jl/src/utils.jl:40 [inlined]
#10 at /Users/mofeing/Developer/Reactant.jl/src/Compiler.jl:295 [inlined]
block! at /Users/mofeing/Developer/Reactant.jl/src/mlir/IR/Block.jl:201
#9 at /Users/mofeing/Developer/Reactant.jl/src/Compiler.jl:294 [inlined]
mmodule! at /Users/mofeing/Developer/Reactant.jl/src/mlir/IR/Module.jl:92
jfptr_mmoduleNOT._18679 at /Users/mofeing/.julia/compiled/v1.10/Reactant/p9PzF_pjiEo.dylib (unknown line)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:3077
#compile_mlir!#8 at /Users/mofeing/Developer/Reactant.jl/src/Compiler.jl:291
compile_mlir! at /Users/mofeing/Developer/Reactant.jl/src/Compiler.jl:290 [inlined]
#34 at /Users/mofeing/Developer/Reactant.jl/src/Compiler.jl:698
context! at /Users/mofeing/Developer/Reactant.jl/src/mlir/IR/Context.jl:76
unknown function (ip: 0x13c96018b)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:3077
#compile_xla#33 at /Users/mofeing/Developer/Reactant.jl/src/Compiler.jl:695
compile_xla at /Users/mofeing/Developer/Reactant.jl/src/Compiler.jl:690 [inlined]
#compile#38 at /Users/mofeing/Developer/Reactant.jl/src/Compiler.jl:722
compile at /Users/mofeing/Developer/Reactant.jl/src/Compiler.jl:721
unknown function (ip: 0x12effc077)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:3077
jl_apply at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/./julia.h:1982 [inlined]
do_call at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/interpreter.c:126
eval_body at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/interpreter.c:0
jl_interpret_toplevel_thunk at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/interpreter.c:775
top-level scope at /Users/mofeing/Developer/Reactant.jl/src/Compiler.jl:475
jl_toplevel_eval_flex at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/toplevel.c:934
jl_toplevel_eval_flex at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/toplevel.c:877
jl_toplevel_eval_flex at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/toplevel.c:877
ijl_toplevel_eval at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/toplevel.c:943 [inlined]
ijl_toplevel_eval_in at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/toplevel.c:985
eval at ./boot.jl:385 [inlined]
eval_user_input at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/usr/share/julia/stdlib/v1.10/REPL/src/REPL.jl:150
repl_backend_loop at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/usr/share/julia/stdlib/v1.10/REPL/src/REPL.jl:246
#start_repl_backend#46 at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/usr/share/julia/stdlib/v1.10/REPL/src/REPL.jl:231
start_repl_backend at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/usr/share/julia/stdlib/v1.10/REPL/src/REPL.jl:228
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:3077
#run_repl#59 at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/usr/share/julia/stdlib/v1.10/REPL/src/REPL.jl:389
run_repl at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/usr/share/julia/stdlib/v1.10/REPL/src/REPL.jl:375
jfptr_run_repl_92227.1 at /Users/mofeing/.julia/juliaup/julia-1.10.7+0.aarch64.apple.darwin14/lib/julia/sys.dylib (unknown line)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:3077
#1014 at ./client.jl:437
jfptr_YY.1014_83184.1 at /Users/mofeing/.julia/juliaup/julia-1.10.7+0.aarch64.apple.darwin14/lib/julia/sys.dylib (unknown line)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:3077
jl_apply at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/./julia.h:1982 [inlined]
jl_f__call_latest at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/builtins.c:812
#invokelatest#2 at ./essentials.jl:892 [inlined]
invokelatest at ./essentials.jl:889 [inlined]
run_main_repl at ./client.jl:421
exec_options at ./client.jl:338
_start at ./client.jl:557
jfptr__start_83210.1 at /Users/mofeing/.julia/juliaup/julia-1.10.7+0.aarch64.apple.darwin14/lib/julia/sys.dylib (unknown line)
_jl_invoke at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:0 [inlined]
ijl_apply_generic at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/gf.c:3077
jl_apply at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/./julia.h:1982 [inlined]
true_main at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/jlapi.c:582
jl_repl_entrypoint at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-honeycrisp-XC9YQX9HH2.0/build/default-honeycrisp-XC9YQX9HH2-0/julialang/julia-release-1-dot-10/src/jlapi.c:731
ERROR: TypeError: in new, expected DataType, got a value of type Tuple{Symbol, Symbol}
Stacktrace:
  [1] (NamedTuple{(:interp,)})(args::Tuple{Enzyme.Compiler.Interpreter.EnzymeInterpreter{typeof(Reactant.set_reactant_abi)}})
    @ Core ./boot.jl:622
  [2] (::Reactant.var"#32#42")()
    @ Reactant ~/Developer/Reactant.jl/src/utils.jl:135
  [3] block!(f::Any, blk::Reactant.MLIR.IR.Block)
    @ Reactant.MLIR.IR ~/Developer/Reactant.jl/src/mlir/IR/Block.jl:201
  [4] make_mlir_fn(f::Function, args::Tuple{ConcreteRArray{Float64, 2}}, kwargs::Tuple{}, name::String, concretein::Bool; toscalar::Bool, return_dialect::Symbol, no_args_in_result::Bool, construct_function_without_args::Bool, do_transpose::Bool)
    @ Reactant ~/Developer/Reactant.jl/src/utils.jl:120
  [5] make_mlir_fn
    @ ~/Developer/Reactant.jl/src/utils.jl:40 [inlined]
  [6] #10
    @ ~/Developer/Reactant.jl/src/Compiler.jl:295 [inlined]
  [7] block!(f::Reactant.Compiler.var"#10#15"{typeof(sum), Tuple{ConcreteRArray{Float64, 2}}}, blk::Reactant.MLIR.IR.Block)
    @ Reactant.MLIR.IR ~/Developer/Reactant.jl/src/mlir/IR/Block.jl:201
  [8] #9
    @ ~/Developer/Reactant.jl/src/Compiler.jl:294 [inlined]
  [9] mmodule!(f::Reactant.Compiler.var"#9#14"{Reactant.MLIR.IR.Module, typeof(sum), Tuple{ConcreteRArray{Float64, 2}}}, blk::Reactant.MLIR.IR.Module)
    @ Reactant.MLIR.IR ~/Developer/Reactant.jl/src/mlir/IR/Module.jl:92
 [10] compile_mlir!(mod::Reactant.MLIR.IR.Module, f::Function, args::Tuple{ConcreteRArray{Float64, 2}}; optimize::Bool)
    @ Reactant.Compiler ~/Developer/Reactant.jl/src/Compiler.jl:291
 [11] compile_mlir!
    @ ~/Developer/Reactant.jl/src/Compiler.jl:290 [inlined]
 [12] (::Reactant.Compiler.var"#34#36"{Bool, typeof(sum), Tuple{ConcreteRArray{Float64, 2}}})()
    @ Reactant.Compiler ~/Developer/Reactant.jl/src/Compiler.jl:698
 [13] context!(f::Reactant.Compiler.var"#34#36"{Bool, typeof(sum), Tuple{ConcreteRArray{Float64, 2}}}, ctx::Reactant.MLIR.IR.Context)
    @ Reactant.MLIR.IR ~/Developer/Reactant.jl/src/mlir/IR/Context.jl:76
 [14] compile_xla(f::Function, args::Tuple{ConcreteRArray{Float64, 2}}; client::Nothing, optimize::Bool)
    @ Reactant.Compiler ~/Developer/Reactant.jl/src/Compiler.jl:695
 [15] compile_xla
    @ ~/Developer/Reactant.jl/src/Compiler.jl:690 [inlined]
 [16] compile(f::Function, args::Tuple{ConcreteRArray{Float64, 2}}; client::Nothing, optimize::Bool, sync::Bool)
    @ Reactant.Compiler ~/Developer/Reactant.jl/src/Compiler.jl:722
 [17] top-level scope
    @ ~/Developer/Reactant.jl/src/Compiler.jl:475

mofeing · 2024-12-11T20:48:09Z

but yeah, it's ok to disable for macOS on Julia 1.10 if you can't make it work
just open an issue so we remember that this is a known issue

also, does this happen for all OS in Julia 1.10? if so, then it's probably sth that we can't solve. but if it just happens, then it might a bug and we should report

glou-nes · 2024-12-11T21:02:58Z

I was just looking to [Julia 1.11 - integration - ubuntu-20.04 - x64 - packaged libReactant - assertions=false - pull_request]. Right there is several kinds of error. For instance, one gets 'ParseError' with [Julia 1.10 - neural_networks - macOS-latest - x64]. I manage to reproduce a error locally. It occurs when I use 'Pkg.test()', direct usage work correctly.
So I using a static now.

mofeing · 2024-12-12T13:47:22Z

CI is still not working... and the issue seems to be coverage? It's very weird that you're having a ParseError, but the erroring file is /Users/runner/work/Reactant.jl/Reactant.jl/src/mlir/Dialects/Enzyme.jl.2696.cov:4, which seems to be produced by coverage.

I have tried it out locally and works so... very weird. @glou-nes do you mind asking in Slack's #github-actions channel maybe?

mofeing · 2024-12-12T16:46:17Z

lol, so it seems like there is a bug in PrecompileTools with coverage?

glou-nes · 2024-12-12T17:03:05Z

For which one do you find that? I don't understand why it's failing in so many different ways. @mofeing I think I don't understand the problem enough to provide a useful reduced reproducer for #github-actions. Do you know which checks must success?

mofeing · 2024-12-12T18:50:31Z

so you should check all the combinations that fulfill "ubuntu or macos, x64, Julia 1.10 or 1.11"

For which one do you find that?

so in this commit from you 8c5a1cf
you disable any code from PrecompileTools when testing, and it worked

do you mind trying to run an empty @setup_workload on CI?

glou-nes · 2024-12-12T18:58:21Z

Sure! I switch to use a CI defined ENV to enable PrecompileTools. It's working! I don't know if that make much sense to run this in CI anyways.

mofeing

I think #377 fixes the problem between coverage and PrecompileTools.

I also reviewed some code to clean.

src/Reactant.jl

src/precompile.jl

glou-nes · 2024-12-14T15:08:10Z

@mofeing only the covering one seems to be corrected. We still get StackOverflow for instance (https://github.com/EnzymeAD/Reactant.jl/actions/runs/12329253047). Is that acceptable to skip the CI with PrecompilationTools using the ENV workaround?

src/XLA.jl

src/Precompile.jl

src/XLA.jl

This reverts commit a688556.

wsmoses · 2025-01-03T23:34:53Z

I dived into this on a 1.10 debug build. It looks like there's some invalid state during deserialization (that at first glance appears to be a julia bug not one of ours).

wsmoses · 2025-01-03T23:38:05Z

@gbaraldi @vchuravy any ideas what's happening here?

wsmoses · 2025-01-03T23:49:21Z

This does trigger on different methods at different times, but I’m really confused how/why since presumably the serialization won’t change ?

gbaraldi · 2025-01-03T23:53:40Z

Memory corruption?

wsmoses · 2025-01-04T03:47:25Z

the (seemingly clearly busted) data is not written between loading the precompiled object and when it breaks. So precompilation is apparently creating a busted serialization?

wsmoses · 2025-01-04T03:47:55Z

Also the fact that this is busted on 1.10 and not 1.11 [combined with the serialization itself being awry] makes me really think this is a Julia bug.

mofeing reviewed Dec 9, 2024

View reviewed changes

wsmoses reviewed Dec 9, 2024

View reviewed changes

mofeing requested changes Dec 10, 2024

View reviewed changes

src/precompile.jl Outdated Show resolved Hide resolved

mofeing approved these changes Dec 10, 2024

View reviewed changes

glou-nes force-pushed the branch-3 branch from 9f3b38f to 565efd8 Compare December 11, 2024 19:28

glou-nes marked this pull request as ready for review December 12, 2024 11:06

glou-nes force-pushed the branch-3 branch from 65830b1 to 75d894e Compare December 12, 2024 19:46

mofeing mentioned this pull request Dec 14, 2024

fix: include files if they end with .jl #377

Merged

mofeing requested changes Dec 14, 2024

View reviewed changes

src/Reactant.jl Outdated Show resolved Hide resolved

mofeing reviewed Dec 14, 2024

View reviewed changes

src/precompile.jl Outdated Show resolved Hide resolved

wsmoses reviewed Dec 14, 2024

View reviewed changes

src/XLA.jl Show resolved Hide resolved

wsmoses reviewed Dec 14, 2024

View reviewed changes

src/Precompile.jl Show resolved Hide resolved

wsmoses reviewed Dec 14, 2024

View reviewed changes

src/Precompile.jl Outdated Show resolved Hide resolved

glou-nes force-pushed the branch-3 branch from 3671ddb to e301a71 Compare December 14, 2024 18:20

mofeing reviewed Dec 15, 2024

View reviewed changes

src/XLA.jl Outdated Show resolved Hide resolved

glou-nes and others added 13 commits January 3, 2025 15:08

add __init__ & assert remove

702c3b1

remove counter & fix precompilation

6a0dc9b

keep only one workload

9ebf3da

try to fix CI

9d2c0d2

forgotten init

67ca304

compact

6ade5e7

apply CI check inside setup_workload

7952694

reviews

7239533

typo

36c03cc

remove CI ENV hack

8c883bd

Revert "remove CI ENV hack"

6c4d00c

This reverts commit a688556.

test CI

9d82b9f

test CI 2

d4c71bb

wsmoses force-pushed the branch-3 branch from 5cab19e to d4c71bb Compare January 3, 2025 20:08

wsmoses added 4 commits January 3, 2025 16:05

Merge branch 'main' into branch-3

63b0b03

de initialize opaque closure cache

4031e9c

ongoing

77fe082

more attempts

598860a

wsmoses mentioned this pull request Jan 4, 2025

1.10: Apparent segfault in precompilation serialization JuliaLang/julia#56947

Closed

wsmoses added 4 commits January 3, 2025 22:57

fix

26d0c0a

fix

adc2622

fix

03ff85e

Disable rules for now as unused

7bdcaab

wsmoses merged commit b32441e into EnzymeAD:main Jan 4, 2025
28 of 38 checks passed

precompile first interpretation #353

precompile first interpretation #353

Conversation

glou-nes commented Dec 9, 2024 • edited Loading

mofeing left a comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

glou-nes Dec 9, 2024 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

mofeing commented Dec 10, 2024

wsmoses commented Dec 10, 2024

glou-nes commented Dec 10, 2024

mofeing left a comment

Choose a reason for hiding this comment

wsmoses commented Dec 10, 2024

mofeing commented Dec 10, 2024

glou-nes commented Dec 11, 2024

mofeing commented Dec 11, 2024

mofeing commented Dec 11, 2024

glou-nes commented Dec 11, 2024 • edited Loading

mofeing commented Dec 12, 2024

mofeing commented Dec 12, 2024

glou-nes commented Dec 12, 2024

mofeing commented Dec 12, 2024

glou-nes commented Dec 12, 2024 • edited Loading

mofeing left a comment

Choose a reason for hiding this comment

glou-nes commented Dec 14, 2024

wsmoses commented Jan 3, 2025

wsmoses commented Jan 3, 2025

wsmoses commented Jan 3, 2025

gbaraldi commented Jan 3, 2025

wsmoses commented Jan 4, 2025

wsmoses commented Jan 4, 2025

glou-nes commented Dec 9, 2024 •

edited

Loading

glou-nes Dec 9, 2024 •

edited

Loading

glou-nes commented Dec 11, 2024 •

edited

Loading

glou-nes commented Dec 12, 2024 •

edited

Loading