From aca40bafd0137014544fe7ff2a004590dc6a0889 Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Thu, 17 Aug 2023 15:00:37 +0200 Subject: [PATCH 1/3] Rewrite to add support for random-acces for Crypt4GHReader if supported by the underlying (encrypted) data stream. --- streaming/in.go | 643 +++++++++++++++++++++++------------- streaming/streaming_test.go | 443 +++++++++++++++++++++++++ 2 files changed, 848 insertions(+), 238 deletions(-) diff --git a/streaming/in.go b/streaming/in.go index d0e820d..c643df7 100644 --- a/streaming/in.go +++ b/streaming/in.go @@ -1,28 +1,68 @@ -// Package streaming contains writer and reader implementing Crypt4GH encryption and decryption correspondingly. +// Package streaming contains Writer and Reader implementing +// Crypt4GH encryption and decryption correspondingly. package streaming import ( "bytes" - "container/list" "errors" "io" + "sync" "github.com/neicnordic/crypt4gh/model/body" "github.com/neicnordic/crypt4gh/model/headers" "golang.org/x/crypto/chacha20poly1305" ) +// crypt4GHInternalReader is the internal structure for managing +// the stream reader type crypt4GHInternalReader struct { + // reader is the Reader providing the encrypted stream data + // is consumed from. reader io.Reader - header []byte + // header is a binary copy of the C4GH file header. + header []byte + + // dataEncryptionParametersHeaderPackets may be one or more + // DataEncryptionParametersHeaderPacket:s. These provide e.g. symmetric + // keys for decrypting segments. dataEncryptionParametersHeaderPackets []headers.DataEncryptionParametersHeaderPacket - dataEditList *headers.DataEditListHeaderPacket - encryptedSegmentSize int - lastDecryptedSegment int - buffer bytes.Buffer + + // dataEditList possibly contains a pointer to a list of edits to apply + // (skip) when consuming the stream. + dataEditList *headers.DataEditListHeaderPacket + + // encryptedSegmentSize is the size of a segment in the encrypted stream, + // i.e. 65536 (data)+any extras added such as MAC or nonce. + encryptedSegmentSize int + + // lastDecryptedSegment is the number of the segment that was last decrypted + // (is available in buffer, if any). + lastDecryptedSegment int64 + + // buffer is where decrypted data is stored temporarily for consumption. It + // contains at most one segments worth of data. + buffer bytes.Buffer + + // bufferUse is the size of the last segment put into buffer at the time of + // writing. bufferUse-buffer.Len() give the number of bytes consumed from + // the buffer already. (Go 1.21 introduces buffer.Available() which allows + // for getting rid of this together with Len() and Cap()). + bufferUse int + + // streamPos is the current offset in the logical consumer stream, i.e. + // where Read or ReadByte should return data from. + streamPos int64 + + // sourcePos is the current offset in reader providing the encrypted stream. + sourcePos int64 + + // sourceStart is the offset for the start of the first encrypted segment. + sourceStart int64 } +// newCrypt4GHInternalReader returns a crypt4GHInternalReader initialised from +// the passed parameters. Returns a pointer or nil and any error encountered. func newCrypt4GHInternalReader(reader io.Reader, readerPrivateKey [chacha20poly1305.KeySize]byte) (*crypt4GHInternalReader, error) { binaryHeader, err := headers.ReadHeader(reader) if err != nil { @@ -47,315 +87,442 @@ func newCrypt4GHInternalReader(reader io.Reader, readerPrivateKey [chacha20poly1 return nil, errors.New("different data encryption methods are not supported") } } + // data encryption methods are the same (standard requirement and validated above), so + // we can just pick the size from the first. crypt4GHInternalReader.encryptedSegmentSize = firstDataEncryptionParametersHeader.EncryptedSegmentSize crypt4GHInternalReader.lastDecryptedSegment = -1 crypt4GHInternalReader.reader = reader - return &crypt4GHInternalReader, nil -} + crypt4GHInternalReader.streamPos = 0 + crypt4GHInternalReader.sourcePos = 0 + + if s, seekable := reader.(io.ReadSeeker); seekable { + // Figure out the offset in the file data starts at (end of headers) + // (move 0 bytes from current position - whence 1) + crypt4GHInternalReader.sourceStart, err = s.Seek(0, 1) -func (c *crypt4GHInternalReader) Read(p []byte) (n int, err error) { - if c.buffer.Len() == 0 { - err := c.fillBuffer() if err != nil { - return 0, err + return nil, err } } - return c.buffer.Read(p) + return &crypt4GHInternalReader, nil } -func (c *crypt4GHInternalReader) ReadByte() (byte, error) { +// ensureBuffer ensure the decrypted buffer is not empty +// unless we have reached the end of the stream +// it also makes sure any remaining data in the buffer +// matches what should be observed in the consumer stream +// at c.streamPos, specifically a Read or ReadByte will +// return the data that should be seen at c.streamPos and +// forward +// +// Returns any error encountered. +func (c *crypt4GHInternalReader) ensureBuffer() (err error) { + + neededSegment, err := c.consumerOffsetToSegment(c.streamPos) + if err != nil { + // Outside of file? Forward error (EOF) + return err + } + + neededPos, err := c.consumerOffsetToEncryptedStreamOffset(c.streamPos) + if err != nil { + return err + } + + // Figure out the needed offset within the segment + segmentOffset := int(neededPos - neededSegment*int64(headers.UnencryptedDataSegmentSize)) + bufferOffset := c.bufferUse - c.buffer.Len() + + if c.lastDecryptedSegment != neededSegment || segmentOffset < bufferOffset { + // If we want to read another segment than the current or if we've + // already read past the desired offset, we need to fetch data, signal + // this by throwing away whatever we currently have + c.buffer.Reset() + } + + // If we don't have any data on hand, fetch more if c.buffer.Len() == 0 { - err := c.fillBuffer() - if err != nil { - return 0, err + if err := c.fillBuffer(); err != nil { + return err } + bufferOffset = 0 } - return c.buffer.ReadByte() + // Find the correct place in the buffert + if bufferOffset < segmentOffset { + toSkip := int(segmentOffset) - bufferOffset + _ = c.buffer.Next(toSkip) + } + + return nil +} + +// consumerOffsetToSegment returns the segment in the underlying stream +// for the passed consumer offset. +// +// It can possibly return error EOF if the passed offset is outside +// of the exposed stream, but this is not guaranteed +func (c *crypt4GHInternalReader) consumerOffsetToSegment(n int64) (int64, error) { + // Figure out the segment + + eo, err := c.consumerOffsetToEncryptedStreamOffset(n) + segment := eo / int64(headers.UnencryptedDataSegmentSize) + + return segment, err } -func (c *crypt4GHInternalReader) Discard(n int) (discarded int, err error) { - if n < 0 { - return +// consumerOffsetToSEncryptedStreamOffset returns the offset in the underlying +// stream for the passed consumer offset. This is excluding the extra bytes +// added by the crypt4gh file format (e.g. headers or segment nonce and mac). +// +// It can possibly return error EOF if the passed offset is outside of the +// exposed stream, but this is not guaranteed. +func (c *crypt4GHInternalReader) consumerOffsetToEncryptedStreamOffset(n int64) (int64, error) { + // Calculate the offset in the encrypted stream from the consumer visible + // offset + // + // The returned offset does *not* include the header size or the additional + // bytes added for each segment (e.g. nonce, MAC) + + if c.dataEditList == nil || c.dataEditList.NumberLengths == 0 { + // No data edit list - offset is unchanged + return n, nil } - if c.buffer.Len() == 0 { - err = c.fillBuffer() - if err != nil { - return + + toCheck := int(c.dataEditList.NumberLengths) + keepSkipList := c.dataEditList.Lengths + skip := true + + var i int + var underlyingPos, exposedPos int64 = 0, 0 + + // Walk through list but stop if we've past the offset + for ; i < toCheck && exposedPos <= n; i++ { + if !skip { + // Stream presented to consumer only advances if not skipped + nextExposedPos := exposedPos + int64(keepSkipList[i]) + + if exposedPos <= n && nextExposedPos > n { + // Not skipping and within this window, + // calculate and return the offset + + return underlyingPos + n - exposedPos, nil + } + + exposedPos = nextExposedPos } + + // Underlying stream moves forward no matter if it's skipped or not. + underlyingPos += int64(keepSkipList[i]) + + skip = !skip } - bytesRead := c.buffer.Cap() - c.buffer.Len() - currentDecryptedPosition := c.lastDecryptedSegment*headers.UnencryptedDataSegmentSize + bytesRead - newDecryptedPosition := currentDecryptedPosition + n - newSegmentNumber := newDecryptedPosition / headers.UnencryptedDataSegmentSize - if newSegmentNumber != c.lastDecryptedSegment { - segmentsToDiscard := newSegmentNumber - c.lastDecryptedSegment - 1 - discarded, err = c.discardSegments(segmentsToDiscard) - if err != nil { - return discarded, err - } - err = c.fillBuffer() - if err != nil { - c.buffer.Reset() - return discarded, err - } - discarded += headers.UnencryptedDataSegmentSize - currentDecryptedPosition = c.lastDecryptedSegment * headers.UnencryptedDataSegmentSize + if i == toCheck && skip { + // Last entry seen was for keeping, skip rest of stream + return underlyingPos, io.EOF } - delta := newDecryptedPosition - currentDecryptedPosition - c.buffer.Next(delta) - return discarded + delta, err + // If last entry seen was skipping, include rest of stream + return n + underlyingPos - exposedPos, nil } -func (c *crypt4GHInternalReader) discardSegments(segments int) (bytesDiscarded int, err error) { - if segments <= 0 { - return +// readByte implements the work of the ReadByte function, +// reading just one byte at maximum. +// +// Returns the byte read or any error encountered +func (c *crypt4GHInternalReader) readByte() (byte, error) { + if err := c.ensureBuffer(); err != nil { + return 0, err } - for i := 0; i < segments; i++ { - discarded := 0 - discarded, err = c.discardSegment() - bytesDiscarded += discarded - if err != nil { - return - } + + b, err := c.buffer.ReadByte() + + if err == nil { + c.streamPos++ } - return + return b, err } -func (c *crypt4GHInternalReader) discardSegment() (bytesDiscarded int, err error) { - bytesToSkip := make([]byte, c.encryptedSegmentSize) - bytesDiscarded, err = c.reader.Read(bytesToSkip) - if err != nil { - return +// read implements the underpinnings of the Read function, serving +// up the unencrypted stream. +// +// Accepts the slice to read into, returns the number of bytes +// read and any error encountered +func (c *crypt4GHInternalReader) read(p []byte) (n int, err error) { + haveRead := 0 + + for haveRead < len(p) { + // We have space to read more? + + // Make sure we have a valid buffer for our position + if err := c.ensureBuffer(); err != nil { + return haveRead, err + } + + canRead := len(p[haveRead:]) + remainingInBuffer := c.bufferUse - c.buffer.Len() + + if remainingInBuffer < canRead { + canRead = remainingInBuffer + } + + start, err := c.consumerOffsetToEncryptedStreamOffset(c.streamPos) + if err != nil { + return haveRead, err + } + + end, _ := c.consumerOffsetToEncryptedStreamOffset(c.streamPos + int64(canRead)) + // Ignore if the end is outside of the file, will trigger EOF "normally" + // anyway + + if (end - start) != int64(canRead) { + // There's a gap somewhere close, read byte by byte. + + startedAt := haveRead + + // Do not try to read the entire desired amount byte for byte + // but rather try at most the rest of our buffer + // after that fall out to the outer loop again + for (startedAt-haveRead) < canRead && haveRead < len(p) { + + p[haveRead], err = c.readByte() + if err != nil { + // Error? Fall out + return haveRead, err + } + haveRead++ + } + } else { + // We can just read the rest of the buffer + + r, err := c.buffer.Read(p[haveRead:]) + haveRead += r + c.streamPos += int64(r) + + // Not sure why we'd get an error here, but forward + // if we see that + if err != nil { + return haveRead, err + } + } } - c.lastDecryptedSegment++ - return + return haveRead, nil } +// fillBuffer makes sure there is data available for reading unless +// we are at EOF. +// Returns any error encountered (e.g. EOF or read error that can +// possibly be due to data corruption). func (c *crypt4GHInternalReader) fillBuffer() error { encryptedSegmentBytes := make([]byte, c.encryptedSegmentSize) - read, err := io.ReadFull(c.reader, encryptedSegmentBytes) + neededSegment, err := c.consumerOffsetToSegment(c.streamPos) + if err != nil { - if err == io.EOF { - return err - } - if err != io.ErrUnexpectedEOF { - return err + return err + } + + segmentPos := neededSegment * int64(c.encryptedSegmentSize) + + // nolint:nestif + if segmentPos != c.sourcePos { + if r, seekable := c.reader.(io.ReadSeeker); seekable { + // If we can seek, do so, this may allow skipping fetching + // large amounts of data + o := segmentPos + c.sourceStart + offset, err := r.Seek(o, 0) + + if err != nil { + return err + } + c.sourcePos = offset + } else { + // Not seekable, figure out how much we need to skip + skip := segmentPos - c.sourcePos + + for skip > int64(0) { + canRead := int64(len(encryptedSegmentBytes)) + if canRead > skip { + canRead = skip + } + read, err := c.reader.Read(encryptedSegmentBytes[:canRead]) + + if err != nil { + // Since we're trying to skip forward to our desired block + // any error goes out + return err + } + skip -= int64(read) + } } } + + // reader should be positioned before the needed segment now + + read, err := io.ReadFull(c.reader, encryptedSegmentBytes) + if err != nil && err != io.ErrUnexpectedEOF { + return err + } + + c.bufferUse = 0 + c.buffer.Reset() + if read == 0 { - c.buffer.Reset() - } else { - c.buffer.Reset() - segment := body.Segment{DataEncryptionParametersHeaderPackets: c.dataEncryptionParametersHeaderPackets} - err := segment.UnmarshalBinary(encryptedSegmentBytes[:read]) - if err != nil { - return err - } - _, err = c.buffer.Write(segment.UnencryptedData) - if err != nil { - return err - } - c.lastDecryptedSegment++ + // Should we fail here? We'll reasonably eventually get + // an EOF anyway + return nil + } + + segment := body.Segment{DataEncryptionParametersHeaderPackets: c.dataEncryptionParametersHeaderPackets} + if err = segment.UnmarshalBinary(encryptedSegmentBytes[:read]); err != nil { + return err + } + c.lastDecryptedSegment = neededSegment + c.sourcePos += int64(read) + + // Keep track of how much data is directly available + + if c.bufferUse, err = c.buffer.Write(segment.UnencryptedData); err != nil { + return err } return nil } -// Crypt4GHReader structure implements io.Reader and io.ByteReader. +// seek implements the actual support for Seek, moves the stream +// (if possible) to the position derived from whence and offset +// returns the new position and/or any error encountered. +func (c *crypt4GHInternalReader) seek(offset int64, whence int) (pos int64, err error) { + if whence == 2 { + return -1, errors.New("Seeking from end not supported") + } + + if whence < 0 || whence > 2 { + return -1, errors.New("Bad whence") + } + + _, seekable := c.reader.(io.Seeker) + + if !seekable && ((whence == 0 && offset < c.streamPos) || (whence == 1 && offset < 0)) { + return -1, errors.New("Seeking backwards only supported when offered by underlying resource") + } + + if whence == 1 { + c.streamPos += offset + } else { + c.streamPos = offset + } + + return c.streamPos, nil +} + +// close method closes the reader, invalidating it. Any error +// encountered is returned. +func (c *crypt4GHInternalReader) close() (err error) { + r, closable := c.reader.(io.Closer) + + c.reader = nil + + if !closable { + // Assume we don't need to do anything, should we fail instead? + return nil + } + + err = r.Close() + + return err +} + +// Crypt4GHReader structure keeps the structure for the internal +// implementation, providing methods for io.Reader, +// io.ByteReader, io.Seeker, io.Closer. type Crypt4GHReader struct { + // reader is the internal crypt4GHInternalReader used for managing state + // and providing relevant methods. reader crypt4GHInternalReader - - useDataEditList bool - lengths list.List - bytesRead uint64 + // mut is a Mutex that provides thread safety. + mut sync.Mutex } -// NewCrypt4GHReader method constructs streaming.Crypt4GHReader instance from io.Reader and corresponding key. +// NewCrypt4GHReader method constructs streaming.Crypt4GHReader instance from +// io.Reader and corresponding key. Allows for overriding data edit list +// from stream, returns the struct pointer or nil and any error encountered. func NewCrypt4GHReader(reader io.Reader, readerPrivateKey [chacha20poly1305.KeySize]byte, dataEditList *headers.DataEditListHeaderPacket) (*Crypt4GHReader, error) { internalReader, err := newCrypt4GHInternalReader(reader, readerPrivateKey) if err != nil { return nil, err } crypt4GHReader := Crypt4GHReader{ - reader: *internalReader, - useDataEditList: dataEditList != nil || internalReader.dataEditList != nil, - lengths: list.List{}, - bytesRead: 0, + reader: *internalReader, } if dataEditList != nil { - skip := true - for i := uint32(0); i < dataEditList.NumberLengths; i++ { - crypt4GHReader.lengths.PushBack(dataEditListEntry{ - length: dataEditList.Lengths[i], - skip: skip, - }) - skip = !skip - } - } else if internalReader.dataEditList != nil { - skip := true - for i := uint32(0); i < internalReader.dataEditList.NumberLengths; i++ { - crypt4GHReader.lengths.PushBack(dataEditListEntry{ - length: internalReader.dataEditList.Lengths[i], - skip: skip, - }) - skip = !skip - } + crypt4GHReader.reader.dataEditList = dataEditList } return &crypt4GHReader, nil } -// Read method implements io.Reader.Read. -func (c *Crypt4GHReader) Read(p []byte) (n int, err error) { - readByte, err := c.ReadByte() - if err != nil { - return - } - p[0] = readByte - n = 1 - for ; n < len(p); n++ { - readByte, err = c.ReadByte() - if err != nil { - return - } - p[n] = readByte - } +// GetHeader method returns the bytes for the Crypt4GH header for the current +// stream. +func (c *Crypt4GHReader) GetHeader() []byte { + // No locking here, reader.header is not used - return + return c.reader.header } -// ReadByte method implements io.ByteReader.ReadByte. -func (c *Crypt4GHReader) ReadByte() (byte, error) { - if c.useDataEditList { - return c.readByteWithDataEditList() - } +// Discard advances the stream without returning the data, returns +// the skipped amount and possible error encountered. +func (c *Crypt4GHReader) Discard(skip int) (n int, err error) { + c.mut.Lock() + defer c.mut.Unlock() - return c.reader.ReadByte() -} + discarded := 0 -func (c *Crypt4GHReader) readByteWithDataEditList() (byte, error) { - if c.lengths.Len() != 0 { - element := c.lengths.Front() - dataEditListEntry := element.Value.(dataEditListEntry) - if dataEditListEntry.skip { - _, err := c.reader.Discard(int(dataEditListEntry.length)) - c.lengths.Remove(element) - if err != nil { - return 0, err - } - } - } - if c.lengths.Len() != 0 { - element := c.lengths.Front() - dataEditListEntry := element.Value.(dataEditListEntry) - length := dataEditListEntry.length - if c.bytesRead == length { - c.lengths.Remove(element) - c.bytesRead = 0 - - return c.readByteWithDataEditList() - } - c.bytesRead++ + for discarded < skip { - return c.reader.ReadByte() + _, err = c.reader.readByte() + + if err != nil { + return discarded, err + } + discarded++ } - return 0, io.EOF + return discarded, nil } -// Discard method skips the next n bytes, returning the number of bytes discarded. -func (c *Crypt4GHReader) Discard(n int) (discarded int, err error) { - if n <= 0 { - return - } - if c.useDataEditList { - return c.discardWithDataEditList(n) - } +// Read method implements io.Reader.Read for the Crypt4GHReader. +func (c *Crypt4GHReader) Read(p []byte) (n int, err error) { + c.mut.Lock() + defer c.mut.Unlock() - return c.reader.Discard(n) + return c.reader.read(p) } -func (c *Crypt4GHReader) discardWithDataEditList(n int) (int, error) { - bytesDiscarded := 0 - if c.lengths.Len() != 0 { //nolint - element := c.lengths.Front() - dataEditListEntry := element.Value.(dataEditListEntry) - if dataEditListEntry.skip { - discarded, err := c.reader.Discard(int(dataEditListEntry.length)) - c.lengths.Remove(element) - if err != nil { - return bytesDiscarded + discarded, err - } - } else { - length := dataEditListEntry.length - if c.bytesRead == length { - c.lengths.Remove(element) - c.bytesRead = 0 - } else { - bytesLeftToRead := length - c.bytesRead - if uint64(n) <= bytesLeftToRead { - c.bytesRead += uint64(n) - - return c.reader.Discard(n) - } - discarded, err := c.reader.Discard(int(bytesLeftToRead)) - bytesDiscarded += discarded - n -= int(bytesLeftToRead) - c.lengths.Remove(element) - c.bytesRead = 0 - if err != nil { - return bytesDiscarded, err - } - } - } - } - for c.lengths.Len() != 0 && n != 0 { - element := c.lengths.Front() - dataEditListEntry := element.Value.(dataEditListEntry) - if dataEditListEntry.skip { //nolint - discarded, err := c.reader.Discard(int(dataEditListEntry.length)) - c.lengths.Remove(element) - if err != nil { - return bytesDiscarded + discarded, err - } - } else { - length := dataEditListEntry.length - if uint64(n) <= length { - discarded, err := c.reader.Discard(n) - if err != nil { - return bytesDiscarded + discarded, err - } - c.bytesRead += uint64(discarded) - bytesDiscarded += discarded - - return bytesDiscarded, nil - } - discarded, err := c.reader.Discard(int(length)) - bytesDiscarded += discarded - n -= int(length) - c.lengths.Remove(element) - if err != nil { - return bytesDiscarded, err - } - } - } +// ReadByte method implements io.ByteReader.ReadByte for the Crypt4GHReader. +func (c *Crypt4GHReader) ReadByte() (byte, error) { + c.mut.Lock() + defer c.mut.Unlock() - return bytesDiscarded, nil + return c.reader.readByte() } -// GetHeader method returns Crypt4GH header structure. -func (c Crypt4GHReader) GetHeader() []byte { - return c.reader.header +// Seek method implements io.Seeker.Seek for the Crypt4GHReader. +func (c *Crypt4GHReader) Seek(offset int64, whence int) (pos int64, err error) { + c.mut.Lock() + defer c.mut.Unlock() + + return c.reader.seek(offset, whence) } -type dataEditListEntry struct { - length uint64 - skip bool +// Close method implements io.Closer.Close for the Crypt4GHReader. +func (c *Crypt4GHReader) Close() (err error) { + c.mut.Lock() + defer c.mut.Unlock() + + return c.reader.close() } diff --git a/streaming/streaming_test.go b/streaming/streaming_test.go index e28c890..ebc07c9 100644 --- a/streaming/streaming_test.go +++ b/streaming/streaming_test.go @@ -107,11 +107,14 @@ func TestReencryption(t *testing.T) { if err != nil { t.Error(err) } + reader, err := NewCrypt4GHReader(&buffer, readerSecretKey, nil) if err != nil { t.Error(err) } + discarded, err := reader.Discard(test.discard) + if err != nil { if test.discard != headers.UnencryptedDataSegmentSize*2 { t.Error(err) @@ -122,6 +125,7 @@ func TestReencryption(t *testing.T) { t.Fail() } } + all, err := io.ReadAll(reader) if err != nil { t.Error(err) @@ -130,6 +134,7 @@ func TestReencryption(t *testing.T) { if err != nil { t.Error(err) } + inBytes, err := io.ReadAll(inFile) if err != nil { t.Error(err) @@ -138,6 +143,7 @@ func TestReencryption(t *testing.T) { if test.discard > len(inBytes) { toDiscard = len(inBytes) } + if !bytes.Equal(all, inBytes[toDiscard:]) { t.Fail() } @@ -202,6 +208,7 @@ func TestReencryptionWithDataEditListInCrypt4GHWriterNoDiscard(t *testing.T) { if err != nil { t.Error(err) } + if !bytes.Equal(all[:837], inBytes[950:950+837]) { t.Fail() } @@ -324,6 +331,7 @@ func TestReencryptionWithDataEditListAndDiscard(t *testing.T) { if discarded != toDiscard { t.Fail() } + all, err := io.ReadAll(reader) if err != nil { t.Error(err) @@ -359,6 +367,7 @@ func TestReencryptionWithDataEditListAndDiscard(t *testing.T) { } expectedText := strings.TrimSpace(string(firstLine) + "\n" + string(secondLine)) actualText := strings.TrimSpace(string(all)) + if !strings.EqualFold(expectedText, actualText) { t.Fail() } @@ -432,14 +441,17 @@ func TestNewCrypt4GHWriterWithoutPrivateKey(t *testing.T) { if err != nil { t.Error(err) } + reader, err := NewCrypt4GHReader(&buffer, readerSecretKey, nil) if err != nil { t.Error(err) } + all, err := io.ReadAll(reader) if err != nil { t.Error(err) } + inFile, err = os.Open("../test/sample.txt") if err != nil { t.Error(err) @@ -540,3 +552,434 @@ func TestFileReEncryption(t *testing.T) { t.Fail() } } + +// TestConsumerToUnderlying verifies functionality of +// consumerOffsetToEncryptedStreamOffset. +func TestConsumerToUnderlying(t *testing.T) { + del := &headers.DataEditListHeaderPacket{NumberLengths: 0} + c := crypt4GHInternalReader{} + + r, err := c.consumerOffsetToEncryptedStreamOffset(10) + if r != 10 || err != nil { + t.Errorf("Conversion of consumer to underlying offset without DEL failed") + } + + c.dataEditList = del + r, err = c.consumerOffsetToEncryptedStreamOffset(10) + if r != 10 || err != nil { + t.Errorf("Conversion of consumer to underlying offset with 0-DEL failed") + } + + del.NumberLengths = 4 + del.Lengths = []uint64{10, 20, 30, 40} + + r, err = c.consumerOffsetToEncryptedStreamOffset(10) + if r != 20 || err != nil { + t.Errorf("Conversion of consumer to underlying failed, first hole") + } + + r, err = c.consumerOffsetToEncryptedStreamOffset(20) + if r != 60 || err != nil { + t.Errorf("Conversion of consumer to underlying failed, two holes") + } + + r, err = c.consumerOffsetToEncryptedStreamOffset(200) + if r != 100 || err == nil { + t.Errorf("Conversion of consumer to underlying EOF failed, got %d, %v", r, err) + } + + del.NumberLengths = 3 + r, err = c.consumerOffsetToEncryptedStreamOffset(200) + if r != 240 || err != nil { + t.Errorf("Conversion of consumer to underlying last infinite failed") + } + +} + +// TestBrokenFileRead verifies proper errors on reading broken files +func TestBrokenFileRead(t *testing.T) { + _, err := NewCrypt4GHReader(bytes.NewBuffer([]byte{}), [32]byte{}, nil) + if err == nil { + t.Errorf("Didn't get error for a reader for an empty file") + } + + _, err = NewCrypt4GHReader(bytes.NewBuffer([]byte{'c', 'r'}), [32]byte{}, nil) + if err == nil { + t.Errorf("Didn't get error for a reader for an empty file") + } + +} + +// TestFillBuffer verifies fillBuffer functionality +func TestFillBuffer(t *testing.T) { + + c := crypt4GHInternalReader{encryptedSegmentSize: 1024} + c.reader = bytes.NewBuffer([]byte{}) + + err := c.fillBuffer() + if err == nil { + t.Errorf("Didn't get error for a reader for an empty file") + } + + _, err = NewCrypt4GHReader(bytes.NewBuffer([]byte{'c', 'r'}), [32]byte{}, nil) + if err == nil { + t.Errorf("Didn't get error for a reader for an empty file") + } + + del := &headers.DataEditListHeaderPacket{} + del.NumberLengths = 4 + del.Lengths = []uint64{10, 20, 30, 40} + c.dataEditList = del + + c.streamPos = 4000 + err = c.fillBuffer() + if err == nil { + t.Errorf("Didn't get error for beyond file according to skiplist") + } +} + +func TestClose(t *testing.T) { + inFile, err := os.Open("../test/sample.txt") + if err != nil { + t.Error(err) + } + readerPublicKey, err := keys.ReadPublicKey(strings.NewReader(crypt4ghX25519Pub)) + if err != nil { + t.Error(err) + } + dataEditListHeaderPacket := headers.DataEditListHeaderPacket{ + PacketType: headers.PacketType{PacketType: headers.DataEditList}, + NumberLengths: 4, + Lengths: []uint64{950, 837, 510, 847}, + } + buffer := bytes.Buffer{} + readerPublicKeyList := [][chacha20poly1305.KeySize]byte{} + readerPublicKeyList = append(readerPublicKeyList, readerPublicKey) + readerPublicKeyList = append(readerPublicKeyList, readerPublicKey) + readerPublicKeyList = append(readerPublicKeyList, readerPublicKey) + if len(readerPublicKeyList) != 3 { + t.Errorf("expected %d public keys in list but got %d", 3, len(readerPublicKeyList)) + } + writer, err := NewCrypt4GHWriterWithoutPrivateKey(&buffer, readerPublicKeyList, &dataEditListHeaderPacket) + if err != nil { + t.Error(err) + } + _, err = io.Copy(writer, inFile) + if err != nil { + t.Error(err) + } + err = inFile.Close() + if err != nil { + t.Error(err) + } + err = writer.Close() + if err != nil { + t.Error(err) + } + readerSecretKey, err := keys.ReadPrivateKey(strings.NewReader(crypt4ghX25519Sec), []byte("password")) + if err != nil { + t.Error(err) + } + + buf1 := buffer.Bytes() + buf2 := bytes.Clone(buf1) + bufferReader := bytes.NewReader(buf1) + + reader, err := NewCrypt4GHReader(bufferReader, readerSecretKey, nil) + if err != nil { + t.Error(err) + } + + err = reader.Close() + if err != nil { + t.Error("Closing bytes.Reader failed") + } + closerReader := io.NopCloser(bytes.NewReader(buf2)) + + reader, err = NewCrypt4GHReader(closerReader, readerSecretKey, nil) + if err != nil { + t.Error(err) + } + + err = reader.Close() + if err != nil { + t.Error("Closing NopCloser failed") + } + +} + +func TestSeek(t *testing.T) { + inFile, err := os.Open("../test/sample.txt") + if err != nil { + t.Error(err) + } + + inBytes, err := io.ReadAll(inFile) + if err != nil { + t.Error(err) + } + + if err = inFile.Close(); err != nil { + t.Error(err) + } + + readerPublicKey, err := keys.ReadPublicKey(strings.NewReader(crypt4ghX25519Pub)) + if err != nil { + t.Error(err) + } + dataEditListHeaderPacket := headers.DataEditListHeaderPacket{ + PacketType: headers.PacketType{PacketType: headers.DataEditList}, + NumberLengths: 4, + Lengths: []uint64{950, 837, 510, 847}, + } + buffer := bytes.Buffer{} + + readerPublicKeyList := [][chacha20poly1305.KeySize]byte{} + readerPublicKeyList = append(readerPublicKeyList, readerPublicKey) + readerPublicKeyList = append(readerPublicKeyList, readerPublicKey) + readerPublicKeyList = append(readerPublicKeyList, readerPublicKey) + if len(readerPublicKeyList) != 3 { + t.Errorf("expected %d public keys in list but got %d", 3, len(readerPublicKeyList)) + } + writer, err := NewCrypt4GHWriterWithoutPrivateKey(&buffer, readerPublicKeyList, &dataEditListHeaderPacket) + if err != nil { + t.Error(err) + } + + if r, err := writer.Write(inBytes); err != nil || r != len(inBytes) { + t.Errorf("Problem when writing to cryptgh writer, r=%d, err=%v", r, err) + } + + err = writer.Close() + if err != nil { + t.Error(err) + } + readerSecretKey, err := keys.ReadPrivateKey(strings.NewReader(crypt4ghX25519Sec), []byte("password")) + if err != nil { + t.Error(err) + } + + bufferReader := bytes.NewReader(buffer.Bytes()) + + reader, err := NewCrypt4GHReader(bufferReader, readerSecretKey, nil) + if err != nil { + t.Error(err) + } + + _, err = reader.Seek(0, 2) + if err == nil { + t.Error("Seeking from end should not be allowed") + } + + _, err = reader.Seek(100, 10) + if err == nil { + t.Error("Bad whence should not be allowed") + } + + r, err := reader.Seek(60, 0) + if err != nil || r != 60 { + t.Error("Seeking from start failed") + } + + r, err = reader.Seek(50, 1) + if err != nil || r != 110 { + t.Error("Seeking forward failed") + } + + all, err := io.ReadAll(reader) + if err != nil { + t.Error(err) + } + + if !bytes.Equal(all[:727], inBytes[1060:1060+727]) { + t.Error("Mismatch after seek") + } + + r, err = reader.Seek(10, 0) + if err != nil || r != 10 { + t.Error("Seeking backward failed") + } + + all, err = io.ReadAll(reader) + + if err != nil { + t.Errorf("Failed when reading after seek %v", err) + } + + if !bytes.Equal(all[:827], inBytes[960:960+827]) || !bytes.Equal(all[827:827+847], inBytes[950+837+510:950+837+510+847]) { + t.Error("Mismatch after seek backwards") + } + + // Refill buffer + buffer.Reset() + writer, err = NewCrypt4GHWriterWithoutPrivateKey(&buffer, readerPublicKeyList, &dataEditListHeaderPacket) + if err != nil { + t.Error(err) + } + + if r, err := writer.Write(inBytes); err != nil || r != len(inBytes) { + t.Errorf("Problem when writing to cryptgh writer, r=%d, err=%v", r, err) + } + + err = writer.Close() + if err != nil { + t.Error(err) + } + + dataEditListHeaderPacket.NumberLengths = 0 + reader, err = NewCrypt4GHReader(&buffer, readerSecretKey, &dataEditListHeaderPacket) + if err != nil { + t.Errorf("Error while making reader from buffer %v", err) + } + + if r, err = reader.Seek(70000, 0); err != nil || r != 70000 { + t.Error("Seeking forward failed") + } + + if r, err = reader.Seek(10, 0); err == nil || r == 10 { + t.Error("Seeking back worked when it wasn't expected") + } + + buf := make([]byte, 10) + if s, err := reader.Read(buf); err != nil || s != 10 { + t.Error("Read after seek failed") + } + + if !bytes.Equal(buf, inBytes[70000:70010]) { + t.Error("Mismatch after seek") + } + + // Refill buffer + buffer.Reset() + writer, err = NewCrypt4GHWriterWithoutPrivateKey(&buffer, readerPublicKeyList, &dataEditListHeaderPacket) + if err != nil { + t.Error(err) + } + + if r, err := writer.Write(inBytes[:70225]); err != nil || r != len(inBytes) { + t.Errorf("Problem when writing to cryptgh writer, r=%d, err=%v", r, err) + } + + err = writer.Close() + if err != nil { + t.Error(err) + } + + reader, err = NewCrypt4GHReader(&buffer, readerSecretKey, &dataEditListHeaderPacket) + if err != nil { + t.Errorf("Error while making reader from buffer again %v", err) + } + + if r, err = reader.Seek(70000, 0); err != nil || r != 70000 { + t.Errorf("Seeking a long bit failed (r=%d, err=%v)", r, err) + } + + buf = make([]byte, 50000) + buf[225] = 42 + buf[226] = 137 + + if s, err := reader.Read(buf); err != io.EOF || s != 225 { + t.Errorf("Read after seek failed (got s=%d, err=%v)", s, err) + } + + if !bytes.Equal(buf[:225], inBytes[70000:70000+225]) { + t.Error("Read didn't return the expected data") + } + + if buf[225] != 42 || buf[226] != 137 { + t.Error("Read touched data unexpectedly") + } + +} + +func TestSmallBuffer(t *testing.T) { + inFile, err := os.Open("../test/sample.txt") + if err != nil { + t.Error(err) + } + inBytes, err := io.ReadAll(inFile) + if err != nil { + t.Error(err) + } + + if err = inFile.Close(); err != nil { + t.Error(err) + } + + inFile, err = os.Open("../test/sample.txt") + if err != nil { + t.Error(err) + } + readerPublicKey, err := keys.ReadPublicKey(strings.NewReader(crypt4ghX25519Pub)) + if err != nil { + t.Error(err) + } + dataEditListHeaderPacket := headers.DataEditListHeaderPacket{ + PacketType: headers.PacketType{PacketType: headers.DataEditList}, + NumberLengths: 8, + Lengths: []uint64{10, 20, 30, 40, 950, 837, 510, 847}, + } + buffer := bytes.Buffer{} + readerPublicKeyList := [][chacha20poly1305.KeySize]byte{} + readerPublicKeyList = append(readerPublicKeyList, readerPublicKey) + readerPublicKeyList = append(readerPublicKeyList, readerPublicKey) + readerPublicKeyList = append(readerPublicKeyList, readerPublicKey) + + if len(readerPublicKeyList) != 3 { + t.Errorf("expected %d public keys in list but got %d", 3, len(readerPublicKeyList)) + } + writer, err := NewCrypt4GHWriterWithoutPrivateKey(&buffer, readerPublicKeyList, &dataEditListHeaderPacket) + if err != nil { + t.Error(err) + } + + if _, err = io.Copy(writer, inFile); err != nil { + t.Error(err) + } + + if err = inFile.Close(); err != nil { + t.Error(err) + } + + if err = writer.Close(); err != nil { + t.Error(err) + } + + readerSecretKey, err := keys.ReadPrivateKey(strings.NewReader(crypt4ghX25519Sec), []byte("password")) + if err != nil { + t.Error(err) + } + + bufferReader := bytes.NewReader(buffer.Bytes()) + + reader, err := NewCrypt4GHReader(bufferReader, readerSecretKey, nil) + if err != nil { + t.Error(err) + } + + buf := make([]byte, 5) + + r, err := reader.Read(buf) + if err != nil || r != 5 { + t.Error("Seeking from end should not be allowed") + } + + if !bytes.Equal(buf, inBytes[10:15]) { + t.Error("Mismatch after first read") + } + + s, err := reader.Seek(18, 0) + if err != nil || s != 18 { + t.Error("Seeking failed") + } + + r, err = reader.Read(buf) + if err != nil || r != 5 { + t.Errorf("Reading failed r=%d err=%v", r, err) + } + + if !bytes.Equal(buf[:2], inBytes[28:30]) && !bytes.Equal(buf[2:], inBytes[60:63]) { + t.Error("Mismatch after second read") + } + +} From 8ed26e4ae1f6cdaf1f594ecef8a83e53fa670549 Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Fri, 18 Aug 2023 07:37:33 +0200 Subject: [PATCH 2/3] Bump version --- internal/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/version/version.go b/internal/version/version.go index d1db3c6..9db53ef 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -7,7 +7,7 @@ import ( ) // The version in the current branch -var Version = "1.7.6" +var Version = "1.8.0" // If this is "" (empty string) then it means that it is a final release. // Otherwise, this is a pre-release e.g. "dev", "beta", "rc1", etc. From 5c3091310992fa77ee1127d09a1262a8843131de Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Fri, 18 Aug 2023 08:06:04 +0200 Subject: [PATCH 3/3] Change type reference in comment Co-authored-by: Stefan Negru --- streaming/in.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streaming/in.go b/streaming/in.go index c643df7..c1da4a3 100644 --- a/streaming/in.go +++ b/streaming/in.go @@ -24,7 +24,7 @@ type crypt4GHInternalReader struct { header []byte // dataEncryptionParametersHeaderPackets may be one or more - // DataEncryptionParametersHeaderPacket:s. These provide e.g. symmetric + // DataEncryptionParametersHeaderPackets. These provide e.g. symmetric // keys for decrypting segments. dataEncryptionParametersHeaderPackets []headers.DataEncryptionParametersHeaderPacket