From 6ad16ba384fcac21b407fbdfdc58a1f30ef7f5dc Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Tue, 22 Oct 2024 22:00:13 -0400 Subject: [PATCH] Use bootstrap approach instead of subproject --- .gitignore | 3 -- dev/release/rat_exclude_files.txt | 1 - python/MANIFEST.in | 1 - python/bootstrap.py | 55 +++++++++++++++++------------- python/generate_dist.py | 53 ---------------------------- python/meson.build | 13 +++---- python/pyproject.toml | 5 +-- python/src/nanoarrow/_ipc_lib.pyx | 2 +- python/src/nanoarrow/meson.build | 24 ++----------- python/subprojects/arrow-nanoarrow | 1 - 10 files changed, 41 insertions(+), 117 deletions(-) delete mode 100644 python/generate_dist.py delete mode 120000 python/subprojects/arrow-nanoarrow diff --git a/.gitignore b/.gitignore index 1196540f1..89fb9fe2f 100644 --- a/.gitignore +++ b/.gitignore @@ -33,8 +33,5 @@ __pycache__ subprojects/* !subprojects/packagefiles !subprojects/*.wrap -python/subprojects/* -!python/subprojects/packagefiles -!python/subprojects/*.wrap compile_commands.json diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 819866645..cb0a83409 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -15,4 +15,3 @@ dist/flatcc.c src/nanoarrow/ipc/flatcc_generated.h thirdparty/* python/src/nanoarrow/dlpack_abi.h -python/subprojects/arrow-nanoarrow diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 1a0b7b480..26c529e0e 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -20,6 +20,5 @@ recursive-include src/nanoarrow *.pxd recursive-include src/nanoarrow *.h exclude bootstrap.py -exclude generate_dist.py exclude src/nanoarrow/*.c recursive-exclude src/nanoarrow *.o *.so diff --git a/python/bootstrap.py b/python/bootstrap.py index 32c0da30a..60c7a04f7 100644 --- a/python/bootstrap.py +++ b/python/bootstrap.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import argparse import pathlib import re import subprocess @@ -58,9 +59,7 @@ def generate_pxd(self, file_in, file_out): output.write(header.encode("UTF-8")) output.write( - f'\ncdef extern from "nanoarrow/{file_in_name}" nogil:\n'.encode( - "UTF-8" - ) + f'\ncdef extern from "{file_in_name}" nogil:\n'.encode("UTF-8") ) # A few things we add in manually @@ -228,13 +227,7 @@ def _write_defs(self, output): # from ../dist if it does not. Running cmake is safer because it will sync # any changes from nanoarrow C library sources in the checkout but is not # strictly necessary for things like installing from GitHub. -def copy_or_generate_nanoarrow_c(): - this_dir = pathlib.Path(__file__).parent.resolve() - subproj_dir = this_dir / "subprojects" - source_dir = subproj_dir / "arrow-nanoarrow" - - vendor_dir = this_dir / "vendor" - +def copy_or_generate_nanoarrow_c(target_dir: pathlib.Path): vendored_files = [ "nanoarrow.h", "nanoarrow.c", @@ -243,12 +236,20 @@ def copy_or_generate_nanoarrow_c(): "nanoarrow_device.h", "nanoarrow_device.c", ] - dst = {name: vendor_dir / name for name in vendored_files} + dst = {name: target_dir / name for name in vendored_files} + + this_dir = pathlib.Path(__file__).parent.resolve() + source_dir = this_dir.parent + is_cmake_dir = (source_dir / "CMakeLists.txt").exists() + is_in_nanoarrow_repo = ( + is_cmake_dir and (source_dir / "src" / "nanoarrow" / "nanoarrow.h").exists() + ) - for f in dst.values(): - f.unlink(missing_ok=True) + if not is_in_nanoarrow_repo: + raise ValueError( + "Attempt to build source distribution outside the nanoarrow repo" + ) - vendor_dir.mkdir(exist_ok=True) subprocess.run( [ sys.executable, @@ -258,9 +259,9 @@ def copy_or_generate_nanoarrow_c(): "--header-namespace", "", "--source-output-dir", - vendor_dir, + target_dir, "--include-output-dir", - vendor_dir, + target_dir, "--with-device", "--with-ipc", "--with-flatcc", @@ -272,18 +273,24 @@ def copy_or_generate_nanoarrow_c(): # Runs the pxd generator with some information about the file name -def generate_nanoarrow_pxds(): - this_dir = pathlib.Path(__file__).parent.resolve() - +def generate_nanoarrow_pxds(target_dir: pathlib.Path): NanoarrowPxdGenerator().generate_pxd( - this_dir / "vendor" / "nanoarrow.h", this_dir / "vendor" / "nanoarrow_c.pxd" + target_dir / "nanoarrow.h", target_dir / "nanoarrow_c.pxd" ) NanoarrowDevicePxdGenerator().generate_pxd( - this_dir / "vendor" / "nanoarrow_device.h", - this_dir / "vendor" / "nanoarrow_device_c.pxd", + target_dir / "nanoarrow_device.h", + target_dir / "nanoarrow_device_c.pxd", ) if __name__ == "__main__": - copy_or_generate_nanoarrow_c() - generate_nanoarrow_pxds() + parser = argparse.ArgumentParser() + parser.add_argument( + "--output-dir", help="Target directory where files should be written" + ) + + args = parser.parse_args() + target_dir = pathlib.Path(args.output_dir).resolve() + + copy_or_generate_nanoarrow_c(target_dir) + generate_nanoarrow_pxds(target_dir) diff --git a/python/generate_dist.py b/python/generate_dist.py deleted file mode 100644 index aabbf6c5c..000000000 --- a/python/generate_dist.py +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import os -import pathlib -import shutil - - -def main(): - src_dir = pathlib.Path(os.environ["MESON_SOURCE_ROOT"]).parent.resolve() - dist_dir = pathlib.Path(os.environ["MESON_DIST_ROOT"]).resolve() - subproj_dir = dist_dir / "subprojects" / "arrow-nanoarrow" - - if subproj_dir.is_symlink(): - subproj_dir.unlink() - - subproj_dir.mkdir() - shutil.copy(src_dir / "meson.build", subproj_dir / "meson.build") - shutil.copy(src_dir / "meson.options", subproj_dir / "meson.options") - - subproj_subproj_dir = subproj_dir / "subprojects" - subproj_subproj_dir.mkdir() - for f in (src_dir / "subprojects").glob("*.wrap"): - shutil.copy(f, subproj_subproj_dir / f.name) - - target_src_dir = subproj_dir / "src" - shutil.copytree(src_dir / "src", target_src_dir) - - # this files are only needed by bootstrap.py - shutil.copy(src_dir / "CMakeLists.txt", subproj_dir / "CMakeLists.txt") - subproj_ci_scripts_dir = subproj_dir / "ci" / "scripts" - subproj_ci_scripts_dir.mkdir(parents=True) - shutil.copy( - src_dir / "ci" / "scripts" / "bundle.py", subproj_ci_scripts_dir / "bundle.py" - ) - - -if __name__ == "__main__": - main() diff --git a/python/meson.build b/python/meson.build index 03e7b82e1..d2336be51 100644 --- a/python/meson.build +++ b/python/meson.build @@ -17,21 +17,18 @@ project( 'nanoarrow', - 'cpp', 'cython', + 'c', 'cython', version: '0.7.0.dev0', license: 'Apache-2.0', meson_version: '>=1.2.0', default_options: [ 'warning_level=2', - 'cpp_std=c++17', + 'c_std=c99', 'default_library=static', - # We need to set these options at the project default_option level - # due to https://github.com/mesonbuild/meson/issues/6728 - 'arrow-nanoarrow:ipc=true', - 'arrow-nanoarrow:device=true', ], ) -subdir('src/nanoarrow') +py = import('python').find_installation(pure: false) -meson.add_dist_script('python', meson.current_source_dir() / 'generate_dist.py') +subdir('vendor') +subdir('src/nanoarrow') diff --git a/python/pyproject.toml b/python/pyproject.toml index 4a55de5da..3cfeb6b95 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -40,10 +40,7 @@ Changelog = "https://github.com/apache/arrow-nanoarrow/blob/main/CHANGELOG.md" requires = [ "meson>=1.3.0", "meson-python", + "cmake>=3.14", "Cython" ] build-backend = "mesonpy" - -[tool.meson-python.args] -install = ['--skip-subprojects'] -dist = ['--include-subprojects'] diff --git a/python/src/nanoarrow/_ipc_lib.pyx b/python/src/nanoarrow/_ipc_lib.pyx index 163644d9e..fcb6530ad 100644 --- a/python/src/nanoarrow/_ipc_lib.pyx +++ b/python/src/nanoarrow/_ipc_lib.pyx @@ -38,7 +38,7 @@ from nanoarrow._array cimport CArrayView from nanoarrow._utils cimport Error -cdef extern from "nanoarrow/nanoarrow_ipc.h" nogil: +cdef extern from "nanoarrow_ipc.h" nogil: struct ArrowIpcInputStream: ArrowErrorCode (*read)(ArrowIpcInputStream* stream, uint8_t* buf, int64_t buf_size_bytes, int64_t* size_read_out, diff --git a/python/src/nanoarrow/meson.build b/python/src/nanoarrow/meson.build index ba4a392e4..e56bbcd3d 100644 --- a/python/src/nanoarrow/meson.build +++ b/python/src/nanoarrow/meson.build @@ -15,24 +15,6 @@ # specific language governing permissions and limitations # under the License. -# Try to resolve the symlink to a subproject first and fall back -# to the wrap entry if that is unsuccessful. Ideally Meson would -# take care of this for us, but there is a bug upstream -# https://github.com/mesonbuild/meson/issues/13746#issuecomment-2392510954 -nanoarrow_proj = subproject('arrow-nanoarrow') -nanoarrow_dep = nanoarrow_proj.get_variable('nanoarrow_dep') -nanoarrow_ipc_dep = nanoarrow_proj.get_variable('nanoarrow_ipc_dep') -nanoarrow_device_dep = nanoarrow_proj.get_variable('nanoarrow_device_dep') - -py = import('python').find_installation(pure: false) - -generated_pyx = custom_target( - 'generate-pyx', - output: 'nanoarrow_c.pxd', - command: [py, meson.current_source_dir() + '/../../bootstrap.py'], -) -nanoarrow_c_dep = declare_dependency(sources: generated_pyx) - cyfiles = [ '_array.pyx', '_array_stream.pyx', @@ -48,7 +30,7 @@ cython_args = [ '--include-dir', meson.current_source_dir(), '--include-dir', - meson.project_source_root() / 'vendor', # for generated nanoarrow_c file + meson.project_build_root() / 'vendor', ] if get_option('buildtype') == 'debug' cython_args += ['--gdb'] @@ -56,11 +38,11 @@ endif fs = import('fs') foreach cyf : cyfiles - cyfile_deps = [nanoarrow_c_dep, nanoarrow_dep] + cyfile_deps = [nanoarrow_pyx_dep] stem = fs.stem(cyf) if stem in ['_array', '_device'] - cyfile_deps += [nanoarrow_device_dep] + cyfile_deps += [nanoarrow_device_pyx_dep] elif stem == '_ipc_lib' cyfile_deps += [nanoarrow_ipc_dep] endif diff --git a/python/subprojects/arrow-nanoarrow b/python/subprojects/arrow-nanoarrow deleted file mode 120000 index c25bddb6d..000000000 --- a/python/subprojects/arrow-nanoarrow +++ /dev/null @@ -1 +0,0 @@ -../.. \ No newline at end of file