Skip to content

Commit

Permalink
feat: Revendor flatcc (#592)
Browse files Browse the repository at this point in the history
After dvidelabs/flatcc#289 flatcc will not
enforce alignment on empty vectors. This will let us unskip the
null_trivial case in the integration tests
  • Loading branch information
bkietz authored Aug 27, 2024
1 parent 31feee9 commit 3298ebc
Show file tree
Hide file tree
Showing 19 changed files with 9,904 additions and 4,107 deletions.
1 change: 1 addition & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,7 @@ foreach(target
-Wunused-result
-Wconversion
-Wno-sign-conversion>)
target_compile_options(${target} PRIVATE -Wno-misleading-indentation)
elseif(CMAKE_C_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_C_COMPILER_ID STREQUAL
"Clang")
target_compile_options(${target}
Expand Down
5 changes: 3 additions & 2 deletions dev/update_fbs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,16 @@
# under the License.

# get the .fbs files from the arrow repo
mkdir format && cd format
mkdir -p format && cd format

curl -L https://github.com/apache/arrow/raw/main/format/Schema.fbs --output Schema.fbs
curl -L https://github.com/apache/arrow/raw/main/format/Tensor.fbs --output Tensor.fbs
curl -L https://github.com/apache/arrow/raw/main/format/SparseTensor.fbs --output SparseTensor.fbs
curl -L https://github.com/apache/arrow/raw/main/format/Message.fbs --output Message.fbs
curl -L https://github.com/apache/arrow/raw/main/format/File.fbs --output File.fbs

# compile using flatcc
flatcc --common --reader --builder --verifier --recursive --outfile ../src/nanoarrow/nanoarrow_ipc_flatcc_generated.h *.fbs
flatcc --common --reader --builder --verifier --recursive --outfile ../../src/nanoarrow/ipc/flatcc_generated.h *.fbs

# clean up
cd ..
Expand Down
59 changes: 28 additions & 31 deletions dev/update_vendored_flatcc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,64 +15,61 @@
# specific language governing permissions and limitations
# under the License.

# Download a flatcc release
curl -L https://github.com/dvidelabs/flatcc/archive/refs/tags/v0.6.1.zip --output flatcc-src.zip
unzip flatcc-src.zip -d flatcc-src
FLATCC_SRC_DIR=flatcc-src/`ls flatcc-src`
REPO=`git rev-parse --show-toplevel`
if [ -z "$REPO" ]
then exit 1
fi

# Download flatcc
git clone https://github.com/dvidelabs/flatcc.git
git -C flatcc checkout bf4f67a16d85541e474f1d67b8fb64913ba72bc7

# Remove previous vendored flatcc
rm -rf ../thirdparty/flatcc
rm -rf $REPO/thirdparty/flatcc

# Create the folders we need to exist
mkdir -p ../thirdparty/flatcc/src/runtime
mkdir -p ../thirdparty/flatcc/include/flatcc/portable
mkdir -p $REPO/thirdparty/flatcc/src/runtime
mkdir -p $REPO/thirdparty/flatcc/include/flatcc/portable

# The only part of the runtime we need
cp $FLATCC_SRC_DIR/src/runtime/emitter.c \
$FLATCC_SRC_DIR/src/runtime/builder.c \
$FLATCC_SRC_DIR/src/runtime/verifier.c \
$FLATCC_SRC_DIR/src/runtime/refmap.c \
../thirdparty/flatcc/src/runtime/
cp flatcc/src/runtime/emitter.c \
flatcc/src/runtime/builder.c \
flatcc/src/runtime/verifier.c \
flatcc/src/runtime/refmap.c \
$REPO/thirdparty/flatcc/src/runtime/

# We also need the headers for those sources. makedepend
# can get those for us in topological order.
pushd $FLATCC_SRC_DIR/include

# List object dependencies (warns that it can't find system headers, which is OK)
# This list is in topological order and could be used for a single-file include;
# however, this approach is easier to support alongside a previous installation
# of the flatcc runtime.
makedepend -s#: -f- -- -I. -DFLATCC_PORTABLE -- 2>/dev/null \
../src/runtime/emitter.c \
../src/runtime/builder.c \
../src/runtime/verifier.c \
../src/runtime/refmap.c \
../src/nanoarrow/nanoarrow_ipc.c | \
makedepend -s#: -f- -- -Iflatcc/include -I$REPO/src -DFLATCC_PORTABLE -- 2>/dev/null \
`ls $REPO/thirdparty/flatcc/src/runtime/*.c` `ls $REPO/src/nanoarrow/ipc/*.c` | \
# Remove the '<src file>.o: ' prefix
sed 's/[^:]*: *//' | \
# Spaces to new lines
sed 's/ /\n/' | \
# Remove system headers
sed '/^\//d' | \
# Only unique lines (but don't sort to preserve topological order)
awk '!x[$0]++' | \
# Remove anything having to do with nanoarrow
sed '/nanoarrow/d' | \
# Remove blank lines
sed '/^$/d' | \
# Or anything having to do with nanoarrow
sed '/nanoarrow/d' > ../../../flatcc-headers.txt
sed '/^$/d' > flatcc-headers.txt

popd
cat flatcc-headers.txt

# Copy the headers we need. Using loops because cp --parents is not portable.
for file in $(cat flatcc-headers.txt | sed /portable/d); do
cp "$FLATCC_SRC_DIR/include/$file" ../thirdparty/flatcc/include/flatcc
done
for file in $(cat flatcc-headers.txt | sed -n /portable/p); do
cp "$FLATCC_SRC_DIR/include/$file" ../thirdparty/flatcc/include/flatcc/portable;
for file in $(cat flatcc-headers.txt); do
cp "$file" "$REPO/thirdparty/$file"
done

# And the license
cp $FLATCC_SRC_DIR/LICENSE ../thirdparty/flatcc
cp flatcc/LICENSE $REPO/thirdparty/flatcc/LICENSE

# clean up
rm -rf flatcc-src
rm flatcc-src.zip
rm -rf flatcc
rm flatcc-headers.txt
Loading

0 comments on commit 3298ebc

Please sign in to comment.