Skip to content

Commit

Permalink
Merge pull request #4181 from melissalinkert/ndpi-precompressed
Browse files Browse the repository at this point in the history 
Add support for reading compressed NDPI tiles
  • Loading branch information
@dgault
dgault authored Jul 22, 2024
2 parents 16a7bf5 + 4f2a96e commit 678fd01
Show file tree
Hide file tree
Showing 7 changed files with 521 additions and 170 deletions.
8 changes: 8 additions & 0 deletions components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1263,6 +1263,14 @@ private boolean doTileConversion(IFormatWriter writer, String outputFile)
if (writer instanceof DicomWriter ||
(writer instanceof ImageWriter && ((ImageWriter) writer).getWriter(outputFile) instanceof DicomWriter))
{
// if we asked to try a precompressed conversion,
// then the writer's tile sizes will have been set automatically
// according to the input data
// the conversion must then be performed tile-wise to match the tile sizes,
// even if precompression doesn't end up being possible
if (precompressed) {
return true;
}
MetadataStore r = reader.getMetadataStore();
return !(r instanceof IPyramidStore) || ((IPyramidStore) r).getResolutionCount(reader.getSeries()) > 1;
}
Expand Down
69 changes: 69 additions & 0 deletions components/formats-bsd/src/loci/formats/in/MinimalTiffReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -705,4 +705,73 @@ protected void initTiffParser() {
tiffParser.setUse64BitOffsets(use64Bit);
}

/**
* Get the index of the tile corresponding to given IFD (plane)
* and tile XY indexes.
*
* @param ifd IFD for the requested tile's plane
* @param x tile X index
* @param y tile Y index
* @return corresponding tile index
*/
protected int getTileIndex(IFD ifd, int x, int y) throws FormatException {
int rows = (int) ifd.getTilesPerColumn();
int cols = (int) ifd.getTilesPerRow();

if (x < 0 || x >= cols) {
throw new IllegalArgumentException("X index " + x + " not in range [0, " + cols + ")");
}
if (y < 0 || y >= rows) {
throw new IllegalArgumentException("Y index " + y + " not in range [0, " + rows + ")");
}

return (cols * y) + x;
}

protected long getCompressedByteCount(IFD ifd, int x, int y) throws FormatException, IOException {
long[] byteCounts = ifd.getStripByteCounts();
int tileIndex = getTileIndex(ifd, x, y);
byte[] jpegTable = (byte[]) ifd.getIFDValue(IFD.JPEG_TABLES);
int jpegTableBytes = jpegTable == null ? 0 : jpegTable.length - 2;
long expectedBytes = byteCounts[tileIndex];
if (expectedBytes > 0) {
expectedBytes += jpegTableBytes;
}
if (expectedBytes < 0 || expectedBytes > Integer.MAX_VALUE) {
throw new IOException("Invalid compressed tile size: " + expectedBytes);
}
return expectedBytes;
}

protected byte[] copyTile(IFD ifd, byte[] buf, int x, int y) throws FormatException, IOException {
long[] offsets = ifd.getStripOffsets();
long[] byteCounts = ifd.getStripByteCounts();

int tileIndex = getTileIndex(ifd, x, y);

byte[] jpegTable = (byte[]) ifd.getIFDValue(IFD.JPEG_TABLES);
int jpegTableBytes = jpegTable == null ? 0 : jpegTable.length - 2;
long expectedBytes = getCompressedByteCount(ifd, x, y);

if (buf.length < expectedBytes) {
throw new IllegalArgumentException("Tile buffer too small: expected >=" +
expectedBytes + ", got " + buf.length);
}
else if (expectedBytes < 0 || expectedBytes > Integer.MAX_VALUE) {
throw new IOException("Invalid compressed tile size: " + expectedBytes);
}

if (jpegTable != null && expectedBytes > 0) {
System.arraycopy(jpegTable, 0, buf, 0, jpegTable.length - 2);
// skip over the duplicate SOI marker
tiffParser.getStream().seek(offsets[tileIndex] + 2);
tiffParser.getStream().readFully(buf, jpegTable.length - 2, (int) byteCounts[tileIndex]);
}
else if (byteCounts[tileIndex] > 0) {
tiffParser.getStream().seek(offsets[tileIndex]);
tiffParser.getStream().readFully(buf, 0, (int) byteCounts[tileIndex]);
}
return buf;
}

}
138 changes: 111 additions & 27 deletions components/formats-bsd/src/loci/formats/out/DicomWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,9 @@ public class DicomWriter extends FormatWriter implements IExtraMetadataWriter {
private int baseTileHeight = 256;
private int[] tileWidth;
private int[] tileHeight;
private long[] tileWidthPointer;
private long[] tileHeightPointer;
private long[] tileCountPointer;
private PlaneOffset[][] planeOffsets;
private Integer currentPlane = null;
private UIDCreator uids;
Expand Down Expand Up @@ -232,14 +235,7 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h)
LOGGER.debug("savePrecompressedBytes(series={}, resolution={}, no={}, x={}, y={})",
series, resolution, no, x, y);

// TODO: may want better handling of non-tiled "extra" images (e.g. label, macro)
MetadataRetrieve r = getMetadataRetrieve();
if ((!(r instanceof IPyramidStore) ||
((IPyramidStore) r).getResolutionCount(series) == 1) &&
!isFullPlane(x, y, w, h))
{
throw new FormatException("DicomWriter does not allow tiles for non-pyramid images");
}

int bytesPerPixel = FormatTools.getBytesPerPixel(
FormatTools.pixelTypeFromString(
Expand Down Expand Up @@ -279,6 +275,13 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h)
boolean first = x == 0 && y == 0;
boolean last = x + w == getSizeX() && y + h == getSizeY();

int width = getSizeX();
int height = getSizeY();
int sizeZ = r.getPixelsSizeZ(series).getValue().intValue();

int tileCountX = (int) Math.ceil((double) width / tileWidth[resolutionIndex]);
int tileCountY = (int) Math.ceil((double) height / tileHeight[resolutionIndex]);

// the compression type isn't supplied to the writer until
// after setId is called, so metadata that indicates or
// depends on the compression type needs to be set in
Expand All @@ -296,6 +299,15 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h)
if (getTIFFCompression() == TiffCompression.JPEG) {
ifds[resolutionIndex][no].put(IFD.PHOTOMETRIC_INTERPRETATION, PhotoInterp.Y_CB_CR.getCode());
}

out.seek(tileWidthPointer[resolutionIndex]);
out.writeShort((short) getTileSizeX());
out.seek(tileHeightPointer[resolutionIndex]);
out.writeShort((short) getTileSizeY());
out.seek(tileCountPointer[resolutionIndex]);

out.writeBytes(padString(String.valueOf(
tileCountX * tileCountY * sizeZ * r.getChannelCount(series))));
}

out.seek(out.length());
Expand Down Expand Up @@ -334,6 +346,17 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h)
if (ifds[resolutionIndex][no] != null) {
tileByteCounts = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_BYTE_COUNTS);
tileOffsets = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_OFFSETS);

if (tileByteCounts.length < tileCountX * tileCountY) {
long[] newTileByteCounts = new long[tileCountX * tileCountY];
long[] newTileOffsets = new long[tileCountX * tileCountY];
System.arraycopy(tileByteCounts, 0, newTileByteCounts, 0, tileByteCounts.length);
System.arraycopy(tileOffsets, 0, newTileOffsets, 0, tileOffsets.length);
tileByteCounts = newTileByteCounts;
tileOffsets = newTileOffsets;
ifds[resolutionIndex][no].put(IFD.TILE_BYTE_COUNTS, tileByteCounts);
ifds[resolutionIndex][no].put(IFD.TILE_OFFSETS, tileOffsets);
}
}

if (tileByteCounts != null) {
Expand Down Expand Up @@ -367,13 +390,7 @@ public void saveBytes(int no, byte[] buf, int x, int y, int w, int h)
int thisTileHeight = tileHeight[resolutionIndex];

MetadataRetrieve r = getMetadataRetrieve();
if ((!(r instanceof IPyramidStore) ||
((IPyramidStore) r).getResolutionCount(series) == 1) &&
!isFullPlane(x, y, w, h))
{
throw new FormatException("DicomWriter does not allow tiles for non-pyramid images");
}
else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
(w != thisTileWidth && x + w != getSizeX()) ||
(h != thisTileHeight && y + h != getSizeY()))
{
Expand All @@ -385,6 +402,10 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
boolean first = x == 0 && y == 0;
boolean last = x + w == getSizeX() && y + h == getSizeY();

int xTiles = (int) Math.ceil((double) getSizeX() / thisTileWidth);
int yTiles = (int) Math.ceil((double) getSizeY() / thisTileHeight);
int sizeZ = r.getPixelsSizeZ(series).getValue().intValue();

// the compression type isn't supplied to the writer until
// after setId is called, so metadata that indicates or
// depends on the compression type needs to be set in
Expand All @@ -406,6 +427,15 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
ifds[resolutionIndex][no].put(IFD.PHOTOMETRIC_INTERPRETATION, PhotoInterp.Y_CB_CR.getCode());
}
}

out.seek(tileWidthPointer[resolutionIndex]);
out.writeShort((short) getTileSizeX());
out.seek(tileHeightPointer[resolutionIndex]);
out.writeShort((short) getTileSizeY());
out.seek(tileCountPointer[resolutionIndex]);

out.writeBytes(padString(String.valueOf(
xTiles * yTiles * sizeZ * r.getChannelCount(series))));
}

// TILED_SPARSE, so the tile coordinates must be written
Expand Down Expand Up @@ -498,7 +528,6 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
// in the IFD
// this tries to calculate the index without assuming sequential tile
// writing, but maybe there is a better way to calculate this?
int xTiles = (int) Math.ceil((double) getSizeX() / tileWidth[resolutionIndex]);
int xTile = x / tileWidth[resolutionIndex];
int yTile = y / tileHeight[resolutionIndex];
int tileIndex = (yTile * xTiles) + xTile;
Expand All @@ -508,6 +537,17 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
if (ifds[resolutionIndex][no] != null) {
tileByteCounts = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_BYTE_COUNTS);
tileOffsets = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_OFFSETS);

if (tileByteCounts.length < xTiles * yTiles) {
long[] newTileByteCounts = new long[xTiles * yTiles];
long[] newTileOffsets = new long[xTiles * yTiles];
System.arraycopy(tileByteCounts, 0, newTileByteCounts, 0, tileByteCounts.length);
System.arraycopy(tileOffsets, 0, newTileOffsets, 0, tileOffsets.length);
tileByteCounts = newTileByteCounts;
tileOffsets = newTileOffsets;
ifds[resolutionIndex][no].put(IFD.TILE_BYTE_COUNTS, tileByteCounts);
ifds[resolutionIndex][no].put(IFD.TILE_OFFSETS, tileOffsets);
}
}

if (compression == null || compression.equals(CompressionType.UNCOMPRESSED.getCompression())) {
Expand Down Expand Up @@ -640,6 +680,9 @@ public void setId(String id) throws FormatException, IOException {
planeOffsets = new PlaneOffset[totalFiles][];
tileWidth = new int[totalFiles];
tileHeight = new int[totalFiles];
tileWidthPointer = new long[totalFiles];
tileHeightPointer = new long[totalFiles];
tileCountPointer = new long[totalFiles];

// create UIDs that must be consistent across all files in the dataset
String specimenUIDValue = uids.getUID();
Expand Down Expand Up @@ -739,8 +782,9 @@ public void setId(String id) throws FormatException, IOException {
int tileCountX = (int) Math.ceil((double) width / tileWidth[resolutionIndex]);
int tileCountY = (int) Math.ceil((double) height / tileHeight[resolutionIndex]);
DicomTag numberOfFrames = new DicomTag(NUMBER_OF_FRAMES, IS);
// save space for up to 10 digits
numberOfFrames.value = padString(String.valueOf(
tileCountX * tileCountY * sizeZ * r.getChannelCount(pyramid)));
tileCountX * tileCountY * sizeZ * r.getChannelCount(pyramid)), " ", 10);
tags.add(numberOfFrames);

DicomTag matrixFrames = new DicomTag(TOTAL_PIXEL_MATRIX_FOCAL_PLANES, UL);
Expand Down Expand Up @@ -1374,6 +1418,9 @@ public void close() throws IOException {
ifds = null;
tiffSaver = null;
validPixelCount = null;
tileWidthPointer = null;
tileHeightPointer = null;
tileCountPointer = null;

tagProviders.clear();

Expand All @@ -1382,33 +1429,46 @@ public void close() throws IOException {

@Override
public int setTileSizeX(int tileSize) throws FormatException {
// TODO: this currently enforces the same tile size across all resolutions
// since the tile size is written during setId
// the tile size should probably be configurable per resolution,
// for better pre-compressed tile support
if (currentId == null) {
baseTileWidth = tileSize;
return baseTileWidth;
}
return baseTileWidth;

int resolutionIndex = getIndex(series, resolution);
tileWidth[resolutionIndex] = tileSize;
return tileWidth[resolutionIndex];
}

@Override
public int getTileSizeX() {
return baseTileWidth;
if (currentId == null) {
return baseTileWidth;
}

int resolutionIndex = getIndex(series, resolution);
return tileWidth[resolutionIndex];
}

@Override
public int setTileSizeY(int tileSize) throws FormatException {
// TODO: see note in setTileSizeX above
if (currentId == null) {
baseTileHeight = tileSize;
return baseTileHeight;
}
return baseTileHeight;

int resolutionIndex = getIndex(series, resolution);
tileHeight[resolutionIndex] = tileSize;
return tileHeight[resolutionIndex];
}

@Override
public int getTileSizeY() {
return baseTileHeight;
if (currentId == null) {
return baseTileHeight;
}

int resolutionIndex = getIndex(series, resolution);
return tileHeight[resolutionIndex];
}

// -- DicomWriter-specific methods --
Expand Down Expand Up @@ -1468,15 +1528,25 @@ private void writeTag(DicomTag tag) throws IOException {
out.writeShort((short) getStoredLength(tag));
}

int resolutionIndex = getIndex(series, resolution);
if (tag.attribute == TRANSFER_SYNTAX_UID) {
transferSyntaxPointer[getIndex(series, resolution)] = out.getFilePointer();
transferSyntaxPointer[resolutionIndex] = out.getFilePointer();
}
else if (tag.attribute == LOSSY_IMAGE_COMPRESSION_METHOD) {
compressionMethodPointer[getIndex(series, resolution)] = out.getFilePointer();
compressionMethodPointer[resolutionIndex] = out.getFilePointer();
}
else if (tag.attribute == FILE_META_INFO_GROUP_LENGTH) {
fileMetaLengthPointer = out.getFilePointer();
}
else if (tag.attribute == ROWS) {
tileHeightPointer[resolutionIndex] = out.getFilePointer();
}
else if (tag.attribute == COLUMNS) {
tileWidthPointer[resolutionIndex] = out.getFilePointer();
}
else if (tag.attribute == NUMBER_OF_FRAMES) {
tileCountPointer[resolutionIndex] = out.getFilePointer();
}

// sequences with no items still need to write a SequenceDelimitationItem below
if (tag.children.size() == 0 && tag.value == null && tag.vr != SQ) {
Expand Down Expand Up @@ -1665,6 +1735,17 @@ private String padString(String value, String append) {
return value + append;
}

private String padString(String value, String append, int length) {
String rtn = "";
if (value != null) {
rtn += value;
}
while (rtn.length() < length) {
rtn += append;
}
return rtn;
}

/**
* @return transfer syntax UID corresponding to the current compression type
*/
Expand Down Expand Up @@ -1919,6 +2000,9 @@ private void writeIFDs(int resIndex) throws IOException {
out.seek(ifdStart);

for (int no=0; no<ifds[resIndex].length; no++) {
ifds[resIndex][no].put(IFD.TILE_WIDTH, tileWidth[resIndex]);
ifds[resIndex][no].put(IFD.TILE_LENGTH, tileHeight[resIndex]);

try {
tiffSaver.writeIFD(ifds[resIndex][no], 0, no < ifds[resIndex].length - 1);
}
Expand Down
Loading

0 comments on commit 678fd01

Please sign in to comment.