Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

On ongoing branch of making everything async #565

Draft
wants to merge 33 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
3a820c5
async Deflate. Start of writer
adamhathcock Jan 24, 2021
bcdfd99
async dispose and fix tests?
adamhathcock Jan 24, 2021
2b4f029
more async await
adamhathcock Feb 1, 2021
9cf8a3d
more awaits
adamhathcock Feb 6, 2021
cdba5ec
AsyncEnumerable usage in entries
adamhathcock Feb 7, 2021
d234f2d
First pass of trying tar
adamhathcock Feb 7, 2021
7d2dc58
More API fixes
adamhathcock Feb 7, 2021
c6a011d
Fixed reader issue
adamhathcock Feb 7, 2021
9738b81
Fix rewindable stream and encoding tests
adamhathcock Feb 7, 2021
f40d334
Tar and Xz mostly work
adamhathcock Feb 7, 2021
813bd5a
Async open entry
adamhathcock Feb 8, 2021
ef3d4da
Fix test and some zip writing
adamhathcock Feb 8, 2021
d6fe729
create async
adamhathcock Feb 8, 2021
db02e8b
Minor fixes
adamhathcock Feb 8, 2021
949e903
More LZMA conversion going, BZip2 not for now
adamhathcock Feb 13, 2021
1f37ced
AsyncStream everything
adamhathcock Feb 13, 2021
fe4cc8e
Zip LZMA write will roundtrip
adamhathcock Feb 13, 2021
ea688e1
Writer problems still :(
adamhathcock Feb 13, 2021
1520917
AsyncStream for BZip2
adamhathcock Feb 13, 2021
7e9fb64
Minor changes
adamhathcock Feb 14, 2021
be34fe2
Merge branch 'master' into async
adamhathcock Feb 14, 2021
5c11075
Updates for merge
adamhathcock Feb 14, 2021
8cdc49c
ReadByteAsync
adamhathcock Feb 14, 2021
14e6d95
More clean up doesn’t help
adamhathcock Feb 14, 2021
d9c53e1
ZLIbStreamfile fixes?
adamhathcock Feb 20, 2021
30da0b9
Fixed Gzip by reverting EmitHeaderAsync
adamhathcock Feb 20, 2021
0c35abd
Explicit exception for read shortcut
adamhathcock Feb 20, 2021
eeb6761
Reuse gzip header reading
adamhathcock Feb 20, 2021
249f11f
Rework some zip writing
adamhathcock Feb 21, 2021
997c11e
Bug fix on counting
adamhathcock Feb 21, 2021
5cfc608
More fixes?
adamhathcock Feb 21, 2021
dd710ec
fixed read only sub stream
adamhathcock Feb 21, 2021
e08e4e5
Enabling Bzip2 but something else is broken
adamhathcock Feb 21, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 46 additions & 74 deletions src/SharpCompress/Archives/AbstractArchive.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,29 +2,25 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using SharpCompress.Common;
using SharpCompress.Readers;

namespace SharpCompress.Archives
{
public abstract class AbstractArchive<TEntry, TVolume> : IArchive, IArchiveExtractionListener
public abstract class AbstractArchive<TEntry, TVolume> : IArchive
where TEntry : IArchiveEntry
where TVolume : IVolume
{
private readonly LazyReadOnlyCollection<TVolume> lazyVolumes;
private readonly LazyReadOnlyCollection<TEntry> lazyEntries;

public event EventHandler<ArchiveExtractionEventArgs<IArchiveEntry>>? EntryExtractionBegin;
public event EventHandler<ArchiveExtractionEventArgs<IArchiveEntry>>? EntryExtractionEnd;

public event EventHandler<CompressedBytesReadEventArgs>? CompressedBytesRead;
public event EventHandler<FilePartExtractionBeginEventArgs>? FilePartExtractionBegin;

protected ReaderOptions ReaderOptions { get; }
protected ReaderOptions ReaderOptions { get; } = new ();

private bool disposed;

internal AbstractArchive(ArchiveType type, FileInfo fileInfo, ReaderOptions readerOptions)
internal AbstractArchive(ArchiveType type, FileInfo fileInfo, ReaderOptions readerOptions, CancellationToken cancellationToken)
{
Type = type;
if (!fileInfo.Exists)
Expand All @@ -33,42 +29,30 @@ internal AbstractArchive(ArchiveType type, FileInfo fileInfo, ReaderOptions read
}
ReaderOptions = readerOptions;
readerOptions.LeaveStreamOpen = false;
lazyVolumes = new LazyReadOnlyCollection<TVolume>(LoadVolumes(fileInfo));
lazyEntries = new LazyReadOnlyCollection<TEntry>(LoadEntries(Volumes));
lazyVolumes = new LazyReadOnlyCollection<TVolume>(LoadVolumes(fileInfo, cancellationToken));
lazyEntries = new LazyReadOnlyCollection<TEntry>(LoadEntries(Volumes, cancellationToken));
}


protected abstract IEnumerable<TVolume> LoadVolumes(FileInfo file);
protected abstract IAsyncEnumerable<TVolume> LoadVolumes(FileInfo file, CancellationToken cancellationToken);

internal AbstractArchive(ArchiveType type, IEnumerable<Stream> streams, ReaderOptions readerOptions)
internal AbstractArchive(ArchiveType type, IAsyncEnumerable<Stream> streams, ReaderOptions readerOptions, CancellationToken cancellationToken)
{
Type = type;
ReaderOptions = readerOptions;
lazyVolumes = new LazyReadOnlyCollection<TVolume>(LoadVolumes(streams.Select(CheckStreams)));
lazyEntries = new LazyReadOnlyCollection<TEntry>(LoadEntries(Volumes));
lazyVolumes = new LazyReadOnlyCollection<TVolume>(LoadVolumes(streams.Select(CheckStreams), cancellationToken));
lazyEntries = new LazyReadOnlyCollection<TEntry>(LoadEntries(Volumes, cancellationToken));
}

#nullable disable
internal AbstractArchive(ArchiveType type)
{
Type = type;
lazyVolumes = new LazyReadOnlyCollection<TVolume>(Enumerable.Empty<TVolume>());
lazyEntries = new LazyReadOnlyCollection<TEntry>(Enumerable.Empty<TEntry>());
lazyVolumes = new LazyReadOnlyCollection<TVolume>( AsyncEnumerable.Empty<TVolume>());
lazyEntries = new LazyReadOnlyCollection<TEntry>(AsyncEnumerable.Empty<TEntry>());
}
#nullable enable

public ArchiveType Type { get; }

void IArchiveExtractionListener.FireEntryExtractionBegin(IArchiveEntry entry)
{
EntryExtractionBegin?.Invoke(this, new ArchiveExtractionEventArgs<IArchiveEntry>(entry));
}

void IArchiveExtractionListener.FireEntryExtractionEnd(IArchiveEntry entry)
{
EntryExtractionEnd?.Invoke(this, new ArchiveExtractionEventArgs<IArchiveEntry>(entry));
}

private static Stream CheckStreams(Stream stream)
{
if (!stream.CanSeek || !stream.CanRead)
Expand All @@ -81,63 +65,48 @@ private static Stream CheckStreams(Stream stream)
/// <summary>
/// Returns an ReadOnlyCollection of all the RarArchiveEntries across the one or many parts of the RarArchive.
/// </summary>
public virtual ICollection<TEntry> Entries => lazyEntries;
public virtual IAsyncEnumerable<TEntry> Entries => lazyEntries;

/// <summary>
/// Returns an ReadOnlyCollection of all the RarArchiveVolumes across the one or many parts of the RarArchive.
/// </summary>
public ICollection<TVolume> Volumes => lazyVolumes;
public IAsyncEnumerable<TVolume> Volumes => lazyVolumes;

/// <summary>
/// The total size of the files compressed in the archive.
/// </summary>
public virtual long TotalSize => Entries.Aggregate(0L, (total, cf) => total + cf.CompressedSize);
public virtual async ValueTask<long> TotalSizeAsync()
{
await EnsureEntriesLoaded();
return await Entries.AggregateAsync(0L, (total, cf) => total + cf.CompressedSize);
}

/// <summary>
/// The total size of the files as uncompressed in the archive.
/// </summary>
public virtual long TotalUncompressSize => Entries.Aggregate(0L, (total, cf) => total + cf.Size);
public virtual async ValueTask<long> TotalUncompressedSizeAsync()
{
await EnsureEntriesLoaded();
return await Entries.AggregateAsync(0L, (total, cf) => total + cf.Size);
}

protected abstract IEnumerable<TVolume> LoadVolumes(IEnumerable<Stream> streams);
protected abstract IEnumerable<TEntry> LoadEntries(IEnumerable<TVolume> volumes);
protected abstract IAsyncEnumerable<TVolume> LoadVolumes(IAsyncEnumerable<Stream> streams, CancellationToken cancellationToken);
protected abstract IAsyncEnumerable<TEntry> LoadEntries(IAsyncEnumerable<TVolume> volumes, CancellationToken cancellationToken);

IEnumerable<IArchiveEntry> IArchive.Entries => Entries.Cast<IArchiveEntry>();
IAsyncEnumerable<IArchiveEntry> IArchive.Entries => Entries.Select(x => (IArchiveEntry)x);

IEnumerable<IVolume> IArchive.Volumes => lazyVolumes.Cast<IVolume>();
IAsyncEnumerable<IVolume> IArchive.Volumes => lazyVolumes.Select(x => (IVolume)x);

public virtual void Dispose()
public virtual async ValueTask DisposeAsync()
{
if (!disposed)
{
lazyVolumes.ForEach(v => v.Dispose());
lazyEntries.GetLoaded().Cast<Entry>().ForEach(x => x.Close());
await lazyVolumes.ForEachAsync(async v => await v.DisposeAsync());
await lazyEntries.GetLoaded().Cast<Entry>().ForEachAsync(async x => await x.CloseAsync());
disposed = true;
}
}

void IArchiveExtractionListener.EnsureEntriesLoaded()
{
lazyEntries.EnsureFullyLoaded();
lazyVolumes.EnsureFullyLoaded();
}

void IExtractionListener.FireCompressedBytesRead(long currentPartCompressedBytes, long compressedReadBytes)
{
CompressedBytesRead?.Invoke(this, new CompressedBytesReadEventArgs(
currentFilePartCompressedBytesRead: currentPartCompressedBytes,
compressedBytesRead: compressedReadBytes
));
}

void IExtractionListener.FireFilePartExtractionBegin(string name, long size, long compressedSize)
{
FilePartExtractionBegin?.Invoke(this, new FilePartExtractionBeginEventArgs(
compressedSize: compressedSize,
size: size,
name: name
));
}

/// <summary>
/// Use this method to extract all entries in an archive in order.
/// This is primarily for SOLID Rar Archives or 7Zip Archives as they need to be
Expand All @@ -149,29 +118,32 @@ void IExtractionListener.FireFilePartExtractionBegin(string name, long size, lon
/// occur if this is used at the same time as other extraction methods on this instance.
/// </summary>
/// <returns></returns>
public IReader ExtractAllEntries()
public async ValueTask<IReader> ExtractAllEntries()
{
await EnsureEntriesLoaded();
return await CreateReaderForSolidExtraction();
}

public async ValueTask EnsureEntriesLoaded()
{
((IArchiveExtractionListener)this).EnsureEntriesLoaded();
return CreateReaderForSolidExtraction();
await lazyEntries.EnsureFullyLoaded();
await lazyVolumes.EnsureFullyLoaded();
}

protected abstract IReader CreateReaderForSolidExtraction();
protected abstract ValueTask<IReader> CreateReaderForSolidExtraction();

/// <summary>
/// Archive is SOLID (this means the Archive saved bytes by reusing information which helps for archives containing many small files).
/// </summary>
public virtual bool IsSolid => false;
public virtual ValueTask<bool> IsSolidAsync() => new(false);

/// <summary>
/// The archive can find all the parts of the archive needed to fully extract the archive. This forces the parsing of the entire archive.
/// </summary>
public bool IsComplete
public async ValueTask<bool> IsCompleteAsync()
{
get
{
((IArchiveExtractionListener)this).EnsureEntriesLoaded();
return Entries.All(x => x.IsComplete);
}
await EnsureEntriesLoaded();
return await Entries.AllAsync(x => x.IsComplete);
}
}
}
Loading