Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix frame header CC tests #1550

Merged
merged 5 commits into from
Aug 22, 2024
Merged

Fix frame header CC tests #1550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
88beb56
modules/zstd: fix formatting errors in C++ files
lpawelcz Aug 16, 2024
94ebb25
modules/zstd: fix formatting errors in BUILD file
lpawelcz Aug 16, 2024
99af284
modules/zstd/frame_header_test: Fix absl::Span handling
lpawelcz Aug 19, 2024
1671da5
modules/zstd: remove 'manual' tag from C++ test rules
lpawelcz Aug 21, 2024
27aea1c
modules/zstd: Remove not used absl/status dependency
lpawelcz Aug 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 12 additions & 15 deletions xls/modules/zstd/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,16 +153,15 @@ cc_library(
"@zstd//:decodecorpus",
],
deps = [
"//xls/common:subprocess",
"//xls/common/file:filesystem",
"//xls/common/file:get_runfile_path",
"//xls/common/status:status_macros",
"@com_google_absl//absl/algorithm:container",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/time",
"@com_google_absl//absl/types:span",
"//xls/common:subprocess",
"//xls/common/file:filesystem",
"//xls/common/file:get_runfile_path",
"//xls/common/status:status_macros",
],
)

Expand Down Expand Up @@ -210,11 +209,8 @@ cc_test(
":frame_header_test_dslx",
],
shard_count = 50,
tags = ["manual"],
deps = [
":data_generator",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/types:span",
"//xls/common:xls_gunit_main",
"//xls/common/file:filesystem",
"//xls/common/file:get_runfile_path",
Expand All @@ -230,8 +226,10 @@ cc_test(
"//xls/ir:bits",
"//xls/ir:ir_test_base",
"//xls/ir:value",
"@zstd",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/types:span",
"@com_google_googletest//:gtest",
"@zstd",
],
)

Expand Down Expand Up @@ -999,13 +997,8 @@ cc_test(
":zstd_dec_test.ir",
],
shard_count = 50,
tags = ["manual"],
deps = [
":data_generator",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/log",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/types:span",
"//xls/common:xls_gunit_main",
"//xls/common/file:filesystem",
"//xls/common/file:get_runfile_path",
Expand All @@ -1020,8 +1013,12 @@ cc_test(
"//xls/ir:ir_parser",
"//xls/ir:value",
"//xls/jit:jit_proc_runtime",
"@zstd",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/log",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/types:span",
"@com_google_googletest//:gtest",
"@zstd",
],
)

Expand Down
5 changes: 2 additions & 3 deletions xls/modules/zstd/data_generator.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,8 @@ static absl::StatusOr<SubprocessResult> CallDecodecorpus(
absl::Span<const std::string> args,
const std::optional<std::filesystem::path>& cwd = std::nullopt,
std::optional<absl::Duration> timeout = std::nullopt) {
XLS_ASSIGN_OR_RETURN(
std::filesystem::path path,
xls::GetXlsRunfilePath("external/zstd/decodecorpus"));
XLS_ASSIGN_OR_RETURN(std::filesystem::path path,
xls::GetXlsRunfilePath("external/zstd/decodecorpus"));

std::vector<std::string> cmd = {path};
cmd.insert(cmd.end(), args.begin(), args.end());
Expand Down
41 changes: 29 additions & 12 deletions xls/modules/zstd/frame_header_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,14 @@
#include <utility>
#include <vector>

#include "gtest/gtest.h"
#include "xls/common/fuzzing/fuzztest.h"
#include "absl/container/flat_hash_map.h"
#include "absl/types/span.h"
#include "external/zstd/lib/zstd.h"
#include "external/zstd/lib/zstd_errors.h"
#include "gtest/gtest.h"
#include "xls/common/file/filesystem.h"
#include "xls/common/file/get_runfile_path.h"
#include "xls/common/fuzzing/fuzztest.h"
#include "xls/common/status/matchers.h"
#include "xls/common/status/ret_check.h"
#include "xls/dslx/create_import_data.h"
Expand All @@ -42,8 +44,6 @@
#include "xls/ir/ir_test_base.h"
#include "xls/ir/value.h"
#include "xls/modules/zstd/data_generator.h"
#include "external/zstd/lib/zstd.h"
#include "external/zstd/lib/zstd_errors.h"

namespace xls {
namespace {
Expand All @@ -57,10 +57,22 @@ enum FrameHeaderStatus : uint8_t {
UNSUPPORTED_WINDOW_SIZE
};

struct ZstdFrameHeader {
absl::Span<const uint8_t> kBuffer;
ZSTD_frameHeader kHeader;
size_t kResult;
class ZstdFrameHeader {
public:
absl::Span<const uint8_t> buffer() const {
return absl::MakeConstSpan(buffer_);
}

ZSTD_frameHeader header() const { return header_; }

size_t result() const { return result_; }

ZstdFrameHeader(absl::Span<const uint8_t> buffer, ZSTD_frameHeader h,
size_t r)
: header_(std::move(h)), result_(r) {
std::vector<uint8_t> v(buffer.begin(), buffer.end());
buffer_ = v;
}
// Parse a frame header from an arbitrary buffer with the ZSTD library.
static absl::StatusOr<ZstdFrameHeader> Parse(
absl::Span<const uint8_t> buffer) {
Expand All @@ -69,8 +81,13 @@ struct ZstdFrameHeader {
ZSTD_frameHeader zstd_fh;
size_t result = ZSTD_getFrameHeader_advanced(
&zstd_fh, buffer.data(), buffer.size(), ZSTD_f_zstd1_magicless);
return ZstdFrameHeader(buffer, zstd_fh, result);
return ZstdFrameHeader(buffer, std::move(zstd_fh), result);
}

private:
std::vector<uint8_t> buffer_;
ZSTD_frameHeader header_;
size_t result_;
};

class FrameHeaderTest : public xls::IrTestBase {
Expand Down Expand Up @@ -113,7 +130,7 @@ class FrameHeaderTest : public xls::IrTestBase {
// expected values.
void RunAndExpectFrameHeader(const ZstdFrameHeader& zstd_frame_header) {
// Extend buffer contents to 128 bits if necessary.
const absl::Span<const uint8_t>& buffer = zstd_frame_header.kBuffer;
const absl::Span<const uint8_t> buffer = zstd_frame_header.buffer();
std::vector<uint8_t> buffer_extended(kDslxBufferSizeBytes, 0);
absl::Span<const uint8_t> input_buffer;
if (buffer.size() < kDslxBufferSizeBytes) {
Expand All @@ -124,8 +141,8 @@ class FrameHeaderTest : public xls::IrTestBase {
}

// Decide on the expected status
ZSTD_frameHeader zstd_fh = zstd_frame_header.kHeader;
size_t result = zstd_frame_header.kResult;
ZSTD_frameHeader zstd_fh = zstd_frame_header.header();
size_t result = zstd_frame_header.result();
FrameHeaderStatus expected_status = FrameHeaderStatus::OK;
if (result != 0) {
if (ZSTD_isError(result)) {
Expand Down
4 changes: 2 additions & 2 deletions xls/modules/zstd/zstd_dec_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,12 @@
#include <utility>
#include <vector>

#include "gtest/gtest.h"
#include "absl/container/flat_hash_map.h"
#include "absl/log/log.h"
#include "absl/status/statusor.h"
#include "absl/types/span.h"
#include "external/zstd/lib/zstd.h"
#include "gtest/gtest.h"
#include "xls/common/file/filesystem.h"
#include "xls/common/file/get_runfile_path.h"
#include "xls/common/status/matchers.h"
Expand All @@ -53,7 +54,6 @@
#include "xls/ir/value.h"
#include "xls/jit/jit_proc_runtime.h"
#include "xls/modules/zstd/data_generator.h"
#include "external/zstd/lib/zstd.h"

namespace xls {
namespace {
Expand Down
Loading