Skip to content

Commit

Permalink
Fix OSS builds with the following changes:
Browse files Browse the repository at this point in the history
1. Use CTAD to allow TriStatePtr to be used with non-unique_ptr data.
2. Migrate from old WORKSPACE to MODULE.bazel
3. Use pip python rule to manage pip dependencies required by array_record_datasource.

PiperOrigin-RevId: 709169059
  • Loading branch information
dryman authored and copybara-github committed Dec 23, 2024
1 parent 82e3d0f commit 50377fc
Show file tree
Hide file tree
Showing 17 changed files with 269 additions and 263 deletions.
7 changes: 7 additions & 0 deletions BUILD
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
load("@rules_python//python:pip.bzl", "compile_pip_requirements")


py_library(
name = "setup",
srcs = ["setup.py"],
)

compile_pip_requirements(
name = "requirements",
src = "requirements.in",
requirements_txt = "requirements_lock.txt",
)
65 changes: 65 additions & 0 deletions MODULE.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
###############################################################################
# Bazel now uses Bzlmod by default to manage external dependencies.
# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel.
#
# For more details, please check https://github.com/bazelbuild/bazel/issues/18958
###############################################################################

# TODO(fchern): automate version string alignment with setup.py
VERSION = "0.6.0"

module(
name = "array_record",
version = VERSION,
repo_name = "com_google_array_record",
)

bazel_dep(name = "abseil-cpp", version = "20240722.0", repo_name = "com_google_absl")
bazel_dep(name = "abseil-py", version = "2.1.0", repo_name = "com_google_absl_py")
bazel_dep(name = "pybind11_bazel", version = "2.12.0")
bazel_dep(name = "rules_proto", version = "7.0.2")
bazel_dep(name = "googletest", version = "1.15.2", repo_name = "com_google_googletest")
bazel_dep(name = "riegeli", version = "0.0.0-20240927-cdfb25a", repo_name = "com_google_riegeli")
bazel_dep(name = "platforms", version = "0.0.10")
bazel_dep(name = "protobuf", version = "29.0", repo_name = "com_google_protobuf")
bazel_dep(name = "rules_python", version = "0.40.0")

http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

# V3.4.0, 20210818
http_archive(
name = "eigen3",
build_file_content =
"""
cc_library(
name = 'eigen3',
srcs = [],
includes = ['.'],
hdrs = glob(['Eigen/**', 'unsupported/Eigen/**']),
visibility = ['//visibility:public'],
)
""",
sha256 = "b4c198460eba6f28d34894e3a5710998818515104d6e74e5cc331ce31e46e626",
strip_prefix = "eigen-3.4.0",
urls = [
"https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.bz2",
],
)

PYTHON_VERSION = "3.10"

python = use_extension("@rules_python//python/extensions:python.bzl", "python")
python.toolchain(
python_version = PYTHON_VERSION,
)

pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")

# requirements_lock.txt is generated by
# bazel run //:requirements.update
pip.parse(
hub_name = "pypi",
python_version = PYTHON_VERSION,
requirements_lock = "//:requirements_lock.txt",
)
use_repo(pip, "pypi")
168 changes: 0 additions & 168 deletions WORKSPACE

This file was deleted.

3 changes: 3 additions & 0 deletions cpp/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,10 @@ cc_library(
deps = [
":common",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/base:nullability",
"@com_google_absl//absl/synchronization",
"@com_google_riegeli//riegeli/base:dependency",
"@com_google_riegeli//riegeli/base:initializer",
],
)

Expand Down
8 changes: 4 additions & 4 deletions cpp/array_record_reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -342,8 +342,8 @@ absl::Status ArrayRecordReaderBase::ParallelReadRecords(
uint64_t chunk_idx_start = buf_idx * state_->chunk_group_size;
// inclusive index, not the conventional exclusive index.
uint64_t last_chunk_idx =
std::min((buf_idx + 1) * state_->chunk_group_size - 1,
state_->chunk_offsets.size() - 1);
std::min<uint64_t>((buf_idx + 1) * state_->chunk_group_size - 1,
state_->chunk_offsets.size() - 1);
uint64_t buf_len = state_->ChunkEndOffset(last_chunk_idx) -
state_->chunk_offsets[chunk_idx_start];
AR_ENDO_JOB(
Expand Down Expand Up @@ -708,8 +708,8 @@ bool ArrayRecordReaderBase::ReadAheadFromBuffer(uint64_t buffer_idx) {
chunk_offsets.reserve(state_->chunk_group_size);
uint64_t chunk_start = buffer_to_add * state_->chunk_group_size;
uint64_t chunk_end =
std::min(state_->chunk_offsets.size(),
(buffer_to_add + 1) * state_->chunk_group_size);
std::min<uint64_t>(state_->chunk_offsets.size(),
(buffer_to_add + 1) * state_->chunk_group_size);
for (uint64_t chunk_idx = chunk_start; chunk_idx < chunk_end; ++chunk_idx) {
chunk_offsets.push_back(state_->chunk_offsets[chunk_idx]);
}
Expand Down
16 changes: 10 additions & 6 deletions cpp/array_record_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ limitations under the License.
#ifndef ARRAY_RECORD_CPP_ARRAY_RECORD_READER_H_
#define ARRAY_RECORD_CPP_ARRAY_RECORD_READER_H_

#include <cstddef>
#include <cstdint>
#include <memory>
#include <optional>
Expand All @@ -46,8 +47,8 @@ limitations under the License.
#include "absl/strings/string_view.h"
#include "absl/types/span.h"
#include "cpp/common.h"
#include "cpp/tri_state_ptr.h"
#include "cpp/thread_pool.h"
#include "cpp/tri_state_ptr.h"
#include "google/protobuf/message_lite.h"
#include "riegeli/base/initializer.h"
#include "riegeli/base/object.h"
Expand Down Expand Up @@ -293,7 +294,7 @@ class ArrayRecordReaderBase : public riegeli::Object {

void Initialize();

virtual TriStatePtr<riegeli::Reader>::SharedRef get_backing_reader()
virtual TriStatePtrBase<riegeli::Reader>::SharedRef get_backing_reader()
const = 0;

private:
Expand Down Expand Up @@ -345,24 +346,27 @@ class ArrayRecordReader : public ArrayRecordReaderBase {
Options options = Options(),
ARThreadPool* pool = nullptr)
: ArrayRecordReaderBase(std::move(options), pool),
main_reader_(std::make_unique<TriStatePtr<riegeli::Reader>>(
main_reader_(std::make_unique<TriStatePtr<riegeli::Reader, Src>>(
std::move(src))) {
Initialize();
}

protected:
TriStatePtr<riegeli::Reader>::SharedRef get_backing_reader() const override {
TriStatePtrBase<riegeli::Reader>::SharedRef get_backing_reader()
const override {
return main_reader_->MakeShared();
}

void Done() override {
if (main_reader_ == nullptr) return;
if (main_reader_ == nullptr) {
return;
}
auto unique = main_reader_->WaitAndMakeUnique();
if (!unique->Close()) Fail(unique->status());
}

private:
std::unique_ptr<TriStatePtr<riegeli::Reader>> main_reader_;
std::unique_ptr<TriStatePtr<riegeli::Reader, Src>> main_reader_;
};

template <typename Src>
Expand Down
4 changes: 2 additions & 2 deletions cpp/array_record_writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ class ArrayRecordWriterBase::SubmitChunkCallback

// Aggregate the offsets information and write it to the file.
void WriteFooterAndPostscript(
TriStatePtr<SequencedChunkWriterBase>::SharedRef writer);
TriStatePtrBase<SequencedChunkWriterBase>::SharedRef writer);

private:
const Options options_;
Expand Down Expand Up @@ -489,7 +489,7 @@ void ArrayRecordWriterBase::SubmitChunkCallback::operator()(
}

void ArrayRecordWriterBase::SubmitChunkCallback::WriteFooterAndPostscript(
TriStatePtr<SequencedChunkWriterBase>::SharedRef writer) {
TriStatePtrBase<SequencedChunkWriterBase>::SharedRef writer) {
// Flushes prior chunks
writer->SubmitFutureChunks(true);
// Footer and postscript must pad to block boundary
Expand Down
Loading

0 comments on commit 50377fc

Please sign in to comment.