Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

v0.4.53 #587

Merged
merged 9 commits into from
Sep 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ updates:
directory: "/" # Location of package manifests
schedule:
interval: "weekly"
target-branch: "dev"
21 changes: 21 additions & 0 deletions .github/workflows/Docs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
name: Documenter
on:
push:
branches: [main, master, dev]
tags: [v*]
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
Documenter:
permissions:
contents: write
statuses: write
name: Documentation
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: julia-actions/julia-docdeploy@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
15 changes: 8 additions & 7 deletions .github/workflows/Downgrade.yml
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
name: Downgrade
on:
pull_request:
branches:
- master
branches: [main, master, dev]
paths-ignore:
- 'docs/**'
push:
branches:
- master
branches: [main, master, dev]
paths-ignore:
- 'docs/**'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
runs-on: ubuntu-latest
Expand All @@ -18,11 +19,11 @@ jobs:
version: ['1']
steps:
- uses: actions/checkout@v4
- uses: julia-actions/setup-julia@v1
- uses: julia-actions/setup-julia@v2
with:
version: ${{ matrix.version }}
- uses: cjdoris/julia-downgrade-compat-action@v1
- uses: julia-actions/julia-downgrade-compat@v1
with:
skip: Pkg,TOML,Mmap
skip: Pkg, TOML, Mmap
- uses: julia-actions/julia-buildpkg@v1
- uses: julia-actions/julia-runtest@v1
4 changes: 1 addition & 3 deletions .github/workflows/Invalidations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name: Invalidations

on:
pull_request:
branches: [main, master, dev]

concurrency:
# Skip intermediate builds: always.
Expand All @@ -11,9 +12,6 @@ concurrency:

jobs:
evaluate:
# Only run on PRs to the default branch.
# In the PR trigger above branches can be specified only explicitly whereas this check should work for master, main, or any other default branch
if: github.base_ref == github.event.repository.default_branch
runs-on: ubuntu-latest
steps:
- uses: julia-actions/setup-julia@v2
Expand Down
56 changes: 8 additions & 48 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,40 +1,28 @@
name: CI
on:
pull_request:
branches:
- master
- dev
branches: [main, master, dev]
push:
branches:
- master
- dev
branches: [main, master, dev]
tags: '*'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
name: Tests, Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
version:
- '1.6'
- '1'
- 'nightly'
os: [ubuntu-latest, windows-latest, macos-latest] # adjust according to need, e.g. os: [ubuntu-latest] if testing only on linux
arch:
- x64
version: ['min', '1', 'nightly']
os: [ubuntu-latest, windows-latest, macos-latest]
arch: [x64]
include:
- os: ubuntu-latest
version: '1'
arch: x86
steps:
# Cancel ongoing CI test runs if pushing to branch again before the previous tests
# have finished
- name: Cancel ongoing test runs for previous commits
uses: styfle/[email protected]
with:
access_token: ${{ github.token }}

# Do tests
- uses: actions/checkout@v4
- uses: julia-actions/setup-julia@v2
Expand All @@ -49,31 +37,3 @@ jobs:
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: lcov.info
docs:
name: Documentation
runs-on: ubuntu-latest
steps:
# Cancel ongoing documentation build if pushing to branch again before the previous
# build is finished.
- name: Cancel ongoing documentation builds for previous commits
uses: styfle/[email protected]
with:
access_token: ${{ github.token }}

# Build docs
- uses: actions/checkout@v4
- uses: julia-actions/setup-julia@v2
with:
version: '1'
- name: Instantiate and install dependencies
run: |
julia --project=docs -e '
using Pkg
Pkg.develop(PackageSpec(path=pwd()))
Pkg.instantiate()'
- name: Generate documentation and deploy
env: # needed for pushing to gh-pages branch
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} # For authentication with SSH deploy key
run:
julia --project=docs docs/make.jl
71 changes: 71 additions & 0 deletions .github/workflows/stale_preview_removal.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
name: Doc Preview Cleanup

on:
schedule:
- cron: "0 0 * * *"

jobs:
doc-preview-cleanup:
runs-on: ubuntu-latest
steps:
- name: Checkout gh-pages branch
uses: actions/checkout@v4
with:
ref: gh-pages
- uses: julia-actions/setup-julia@v2
- name: Check for stale PR previews
shell: julia {0}
run: |
using Pkg
pkg"activate --temp"
pkg"add HTTP JSON3"

using HTTP
using JSON3
using Dates

repo = ENV["GITHUB_REPOSITORY"]
retention_days = 14

pr_previews = map(filter(startswith("PR"), readdir("previews"))) do dir
parse(Int, match(r"PR(\d*)", dir)[1])
end

function all_prs()
query_prs(page) = JSON3.read(HTTP.get("https://api.github.com/repos/$repo/pulls?per_page=100;page=$(page)").body)
prs = []
page = 1
while true
page_prs = query_prs(page)
isempty(page_prs) && break
append!(prs, page_prs)
page += 1
end
return prs
end
prs = all_prs()
open_within_threshold = map(x -> x.number, filter(prs) do pr
time = DateTime(pr.updated_at[1:19], ISODateTimeFormat)
return pr.state == "open" && Dates.days(now() - time) <= retention_days
end)

stale_previews = setdiff(pr_previews, open_within_threshold)
@info "Found $(length(stale_previews)) stale previews"

if isempty(stale_previews)
@info "No stale previews"
exit(1)
end

for pr in stale_previews
path = joinpath("previews", "PR$pr")
@info "Removing $path"
run(`git rm -rf $path`)
end
- name: Push changes
run: |
git config user.name "Documenter.jl"
git config user.email "[email protected]"
git commit -m "delete preview"
git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree})
git push --force origin gh-pages-new:gh-pages
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
## 0.4.53
- Experimental: Slicing and inplace updating of array datasets
- updated CI workflows
- improve pretty printing of attribute header message
- fix storing of datatype info for h5 compat

## 0.4.52
- fix attribute loading
- new features: `readmmap` `ismmappable` and `allocate_early` (api experimental)
Expand Down
2 changes: 1 addition & 1 deletion Project.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name = "JLD2"
uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
version = "0.4.52"
version = "0.4.53"

[deps]
FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
Expand Down
12 changes: 6 additions & 6 deletions src/data/number_types.jl
Original file line number Diff line number Diff line change
Expand Up @@ -67,18 +67,18 @@ h5fieldtype(::JLDFile, ::Type{Bool}, ::Type{Bool}, ::Initialized) =BitFieldDatat
jltype(::JLDFile, ::BitFieldDatatype) = ReadRepresentation{Bool, Bool}()

h5fieldtype(::JLDFile, ::Type{Float16}, ::Type{Float16}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x20, 0x0f, 0x00, 2, 0, 16, 10, 5, 0, 10, 0x0000000f)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x20, 0x0f, 0x00, 2, 0, 16, 10, 5, 0, 10, 0x0000000f)
h5fieldtype(::JLDFile, ::Type{Float32}, ::Type{Float32}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x20, 0x1f, 0x00, 4, 0, 32, 23, 8, 0, 23, 0x0000007f)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x20, 0x1f, 0x00, 4, 0, 32, 23, 8, 0, 23, 0x0000007f)
h5fieldtype(::JLDFile, ::Type{Float64}, ::Type{Float64}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x20, 0x3f, 0x00, 8, 0, 64, 52, 11, 0, 52, 0x000003ff)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x20, 0x3f, 0x00, 8, 0, 64, 52, 11, 0, 52, 0x000003ff)

h5fieldtype(::JLDFile, ::Type{BENumber{Float16}}, ::Type{Float16}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x21, 0x0f, 0x00, 2, 0, 16, 10, 5, 0, 10, 0x0000000f)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x21, 0x0f, 0x00, 2, 0, 16, 10, 5, 0, 10, 0x0000000f)
h5fieldtype(::JLDFile, ::Type{BENumber{Float32}}, ::Type{Float32}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x21, 0x1f, 0x00, 4, 0, 32, 23, 8, 0, 23, 0x0000007f)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x21, 0x1f, 0x00, 4, 0, 32, 23, 8, 0, 23, 0x0000007f)
h5fieldtype(::JLDFile, ::Type{BENumber{Float64}}, ::Type{Float64}, ::Initialized) =
FloatingPointDatatype(DT_FLOATING_POINT, 0x21, 0x3f, 0x00, 8, 0, 64, 52, 11, 0, 52, 0x000003ff)
FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x21, 0x3f, 0x00, 8, 0, 64, 52, 11, 0, 52, 0x000003ff)

function jltype(f::JLDFile, dt::FloatingPointDatatype)
if dt == h5fieldtype(f, Float64, Float64, Val{true})
Expand Down
55 changes: 18 additions & 37 deletions src/data/reconstructing_datatypes.jl
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,10 @@

# jltype is the inverse of h5type, providing a ReadRepresentation for an
# H5Datatype. We handle committed datatypes here, and other datatypes below.
function jltype(f::JLDFile, cdt::CommittedDatatype)
function jltype(f::JLDFile, sdt::Union{SharedDatatype,CommittedDatatype})
cdt = get(f.datatype_locations, sdt.header_offset, sdt)
haskey(f.h5jltype, cdt) && return f.h5jltype[cdt]::ReadRepresentation

dt, attrs = read_shared_datatype(f, cdt)

julia_type_attr = nothing
Expand All @@ -75,51 +77,40 @@
written_type_attr = attr
end
end
isnothing(julia_type_attr) && return f.h5jltype[cdt] = jltype(f, dt)

if isa(julia_type_attr, Nothing)
throw(InvalidDataException())
end
julia_type_attr = julia_type_attr::ReadAttribute

# If type of datatype is this datatype, then this is the committed
# datatype that describes a datatype
if julia_type_attr.datatype isa SharedDatatype &&
julia_type_attr.datatype.header_offset == cdt.header_offset
# Verify that the datatype matches our expectations
# Bootstrap: the datatype of datatype is a datatype
if julia_type_attr.datatype == SharedDatatype(cdt.header_offset)
if dt != H5TYPE_DATATYPE
error("""The HDF5 datatype representing a Julia datatype does not match
the expectations of this version of JLD.

You may need to update JLD to read this file.""")
throw(InternalError("""The HDF5 datatype representing a Julia datatype does not match

Check warning on line 85 in src/data/reconstructing_datatypes.jl

View check run for this annotation

Codecov / codecov/patch

src/data/reconstructing_datatypes.jl#L85

Added line #L85 was not covered by tests
the expectations of this version of JLD2.
You may need to update JLD2 to read this file."""))
end
f.jlh5type[DataType] = cdt
f.datatypes[cdt.index] = dt
return (f.h5jltype[cdt] = ReadRepresentation{DataType, DataTypeODR()}())
end

f.plain && return f.h5jltype[cdt] = jltype(f, dt)

datatype = read_attr_data(f, julia_type_attr)
if f.plain && !(datatype isa Upgrade) && !(datatype <: Tuple)
rr = jltype(f, dt)
return f.h5jltype[cdt] = rr
end

if written_type_attr !== nothing
if !isnothing(written_type_attr)
# Custom serialization
custom_datatype = read_attr_data(f, written_type_attr)
read_as = _readas(custom_datatype, datatype)
if read_as <: UnknownType
@warn("custom serialization of $(typestring(read_as))" *
" encountered, but the type does not exist in the workspace; the data will be read unconverted")
rr = (constructrr(f, custom_datatype, dt, attrs)::Tuple{ReadRepresentation,Bool})[1]
rr, _ = constructrr(f, custom_datatype, dt, attrs)
canonical = false
else
rr, canonical = constructrr(f, custom_datatype, dt, attrs)::Tuple{ReadRepresentation,Bool}
rrty = typeof(rr)
rr = ReadRepresentation{read_as, CustomSerialization{rrty.parameters[1], rrty.parameters[2]}}()
canonical = canonical && writeas(read_as) === custom_datatype
rr, canonical = constructrr(f, custom_datatype, dt, attrs)
rr = ReadRepresentation{read_as, CustomSerialization{typeof(rr).parameters...}}()
canonical &= writeas(read_as) === custom_datatype
end
else
rr, canonical = constructrr(f, datatype, dt, attrs)::Tuple{ReadRepresentation,Bool}
rr, canonical = constructrr(f, datatype, dt, attrs)
end

canonical && (f.jlh5type[datatype] = cdt)
Expand All @@ -128,16 +119,6 @@
end


# jltype is the inverse of h5type, providing a ReadRepresentation for an
# H5Datatype. We handle shared datatypes here: ones that were not "committed" by JLD2.
function jltype(f::JLDFile, sdt::SharedDatatype)
haskey(f.h5jltype, sdt) && return f.h5jltype[sdt]::ReadRepresentation
dt, attrs = read_shared_datatype(f, sdt)
rr = jltype(f, dt)
f.h5jltype[sdt] = rr
end



# Constructs a ReadRepresentation for a given opaque (bitstype) type
function constructrr(::JLDFile, T::DataType, dt::BasicDatatype, attrs::Vector{ReadAttribute})
Expand Down Expand Up @@ -381,7 +362,7 @@
# If the reference is to a committed datatype, read the datatype
nulldt = CommittedDatatype(UNDEFINED_ADDRESS, 0)
cdt = get(f.datatype_locations, ref, nulldt)
res = cdt !== nulldt ? (typeof(jltype(f, cdt)::ReadRepresentation)::DataType).parameters[1] : load_dataset(f, ref)
res = cdt !== nulldt ? eltype(jltype(f, cdt)) : load_dataset(f, ref)
unknown_params = unknown_params || isunknowntype(res) || isreconstructed(res)
res
end for ref in refs]
Expand Down
Loading
Loading