diff --git a/src/SharpCompress/Compressors/Xz/XZBlock.cs b/src/SharpCompress/Compressors/Xz/XZBlock.cs
index ddd7eaf3..aec35470 100644
--- a/src/SharpCompress/Compressors/Xz/XZBlock.cs
+++ b/src/SharpCompress/Compressors/Xz/XZBlock.cs
@@ -25,13 +25,14 @@ public sealed class XZBlock : XZReadOnlyStream
     private bool _endOfStream;
     private bool _paddingSkipped;
     private bool _crcChecked;
-    private ulong _bytesRead;
+    private readonly long _startPosition;
 
     public XZBlock(Stream stream, CheckType checkType, int checkSize)
         : base(stream)
     {
         _checkType = checkType;
         _checkSize = checkSize;
+        _startPosition = stream.Position;
     }
 
     public override int Read(byte[] buffer, int offset, int count)
@@ -67,13 +68,12 @@ public override int Read(byte[] buffer, int offset, int count)
             CheckCrc();
         }
 
-        _bytesRead += (ulong)bytesRead;
         return bytesRead;
     }
 
     private void SkipPadding()
     {
-        var bytes = (int)(BaseStream.Position % 4);
+        var bytes = (BaseStream.Position - _startPosition) % 4;
         if (bytes > 0)
         {
             var paddingBytes = new byte[4 - bytes];
diff --git a/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs b/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs
index 180b8416..ae91533c 100644
--- a/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs
+++ b/tests/SharpCompress.Test/Mocks/ForwardOnlyStream.cs
@@ -31,7 +31,7 @@ public override void Flush() { }
 
     public override long Position
     {
-        get => throw new NotSupportedException();
+        get => stream.Position;
         set => throw new NotSupportedException();
     }
 
diff --git a/tests/SharpCompress.Test/Xz/XZBlockTests.cs b/tests/SharpCompress.Test/Xz/XZBlockTests.cs
index 78873590..e129f4ec 100644
--- a/tests/SharpCompress.Test/Xz/XZBlockTests.cs
+++ b/tests/SharpCompress.Test/Xz/XZBlockTests.cs
@@ -93,4 +93,19 @@ public void SkipsPaddingWhenPresent()
         sr.ReadToEnd();
         Assert.Equal(0L, CompressedIndexedStream.Position % 4L);
     }
+
+    [Fact]
+    public void HandlesPaddingInUnalignedBlock()
+    {
+        var compressedUnaligned = new byte[Compressed.Length + 1];
+        Compressed.CopyTo(compressedUnaligned, 1);
+        var compressedUnalignedStream = new MemoryStream(compressedUnaligned);
+        compressedUnalignedStream.Position = 13;
+
+        // Compressed's only block has no padding.
+        var xzBlock = new XZBlock(compressedUnalignedStream, CheckType.CRC64, 8);
+        var sr = new StreamReader(xzBlock);
+        sr.ReadToEnd();
+        Assert.Equal(1L, compressedUnalignedStream.Position % 4L);
+    }
 }