diff --git a/CMakeLists.txt b/CMakeLists.txt index e9953dd..a7da6e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,33 +1,19 @@ -cmake_minimum_required(VERSION 3.15...3.26) -project(kaitai_awkward_runtime CXX) +cmake_minimum_required(VERSION 3.19...3.26) +project(${SKBUILD_PROJECT_NAME} CXX) set(KSY "" CACHE STRING "Specify the KSY file path") -set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - -string(REGEX MATCH "([^/]+)$" package_name ${KSY}) -string(REGEX REPLACE "\\.ksy$" "" package_name ${package_name}) -set(PACKAGE_NAME ${package_name}) -# set(CMAKE_ARGS "-DPACKAGE_NAME=${PACKAGE_NAME}") - -set(BUILD_SHARED_LIBS OFF) -set(BUILD_STATIC_LIBS ON) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") +set(BUILD_SHARED_LIBS ON) find_package(pybind11 CONFIG REQUIRED) -add_subdirectory(kaitai_struct_cpp_stl_runtime) - -set (MAIN_HEADERS - kaitai_struct_cpp_stl_runtime/kaitai/kaitaistream.h - kaitai_struct_cpp_stl_runtime/kaitai/kaitaistruct.h - kaitai_struct_cpp_stl_runtime/kaitai/exceptions.h -) - -set (MAIN_SOURCES - kaitai_struct_cpp_stl_runtime/kaitai/kaitaistream.cpp -) +# Setup the RPATH for built libraries +if(APPLE) + set(CMAKE_INSTALL_RPATH "@loader_path") +else() + SET(CMAKE_INSTALL_RPATH "$ORIGIN") +endif() -add_library (${PROJECT_NAME} ${MAIN_HEADERS} ${MAIN_SOURCES}) +add_subdirectory(kaitai_struct_cpp_stl_runtime EXCLUDE_FROM_ALL) # Download awkward headers include(FetchContent) @@ -43,25 +29,32 @@ if(NOT awkward-headers_POPULATED) add_subdirectory(${awkward-headers_SOURCE_DIR} ${awkward-headers_BINARY_DIR} EXCLUDE_FROM_ALL) endif() -target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_BINARY_DIR}/_deps/awkward-headers-src/layout-builder) -target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_BINARY_DIR}/_deps/awkward-headers-src/builder-options) -target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_BINARY_DIR}/_deps/awkward-headers-src/growable-buffer) - # Run sbt package command to compile kaitai_struct_compiler code execute_process( COMMAND sbt package WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/kaitai_struct_compiler + COMMAND_ERROR_IS_FATAL ANY ) execute_process( - COMMAND ./kaitai-struct-compiler -t awkward --outdir src-${PACKAGE_NAME} ${KSY} #package name should be declared as a string + COMMAND ./kaitai-struct-compiler -t awkward --outdir src-${SKBUILD_PROJECT_NAME} ${KSY} #package name should be declared as a string WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMAND_ERROR_IS_FATAL ANY ) -pybind11_add_module(awkward_${PACKAGE_NAME} src-${PACKAGE_NAME}/${PACKAGE_NAME}_main.cpp src-${PACKAGE_NAME}/${PACKAGE_NAME}.cpp) -target_link_libraries(awkward_${PACKAGE_NAME} PRIVATE awkward::layout-builder) -target_include_directories(awkward_${PACKAGE_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/kaitai_struct_cpp_stl_runtime) -target_link_libraries(awkward_${PACKAGE_NAME} PUBLIC kaitai_struct_cpp_stl_runtime) +string(REGEX MATCH "([^/]+)$" KSY_MODULE ${KSY}) +string(REGEX REPLACE "\\.ksy$" "" KSY_MODULE ${KSY_MODULE}) + +pybind11_add_module(awkward_${KSY_MODULE} src-${SKBUILD_PROJECT_NAME}/${KSY_MODULE}_main.cpp src-${SKBUILD_PROJECT_NAME}/${KSY_MODULE}.cpp) +target_link_libraries(awkward_${KSY_MODULE} PRIVATE awkward::layout-builder kaitai_struct_cpp_stl_runtime) + +# Install pure-python files +file(GLOB_RECURSE PYTHON_SOURCES "src/${SKBUILD_PROJECT_NAME}/*.py") + +install( + TARGETS awkward_${KSY_MODULE} kaitai_struct_cpp_stl_runtime + LIBRARY DESTINATION "${SKBUILD_PROJECT_NAME}/lib" + RUNTIME DESTINATION "${SKBUILD_PROJECT_NAME}/lib" + ARCHIVE DESTINATION "${SKBUILD_PROJECT_NAME}/lib") -# Install into wheel -install(TARGETS awkward_${PACKAGE_NAME} DESTINATION .) \ No newline at end of file +install(FILES ${PYTHON_SOURCES} DESTINATION ${SKBUILD_PROJECT_NAME}) diff --git a/README.md b/README.md index 64f2929..d15f217 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # kaitai_awkward_runtime -# awkward-kaitai Building Awkward Arrays using Kaitai binary file descriptors. Steps for reproducing the environment: @@ -10,23 +9,25 @@ Steps for reproducing the environment: git clone --recursive https://github.com/ManasviGoyal/kaitai_awkward_runtime.git ``` -2. Change directory to `kaitai_awkward_runtime` and run CMake: +2. Change directory to `kaitai_awkward_runtime`: ``` cd kaitai_awkward_runtime -export CMAKE_ARGS="-DKSY=schemas/animal.ksy -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON" ``` -3. Install the library and open Python: +3. Install the library, and open Python: ``` -pip install . +pip install . python ``` 4. Print the returned `ak.Array`: ```python -import animal -awkward_array = animal.load("../data/animal.raw") +import awkward_animal +awkward_array = awkward_animal.load("data/animal.raw") print(awkward_array) ``` + +> **Info** +> `kaitai_awkward_runtime` depends upon `sbt`, and `gtest` dependencies. diff --git a/kaitai-struct-compiler b/kaitai-struct-compiler old mode 100644 new mode 100755 diff --git a/kaitai_struct_compiler b/kaitai_struct_compiler index 2f894bf..9aefa32 160000 --- a/kaitai_struct_compiler +++ b/kaitai_struct_compiler @@ -1 +1 @@ -Subproject commit 2f894bf7aba2a7214921deb3a8d75d3fa684e945 +Subproject commit 9aefa325ae2b57df1a81fb82dfd68be6deb5a5c3 diff --git a/pyproject.toml b/pyproject.toml index 0ef6b58..5de00d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,9 @@ classifiers = [ "Typing :: Typed", ] dynamic = ["version"] -dependencies = [] +dependencies = [ + "awkward>2.4.3" +] [project.optional-dependencies] test = [ @@ -50,10 +52,8 @@ docs = [ ] [project.urls] -Homepage = "https://github.com/org/kaitai_awkward_runtime" -"Bug Tracker" = "https://github.com/org/kaitai_awkward_runtime/issues" -Discussions = "https://github.com/org/kaitai_awkward_runtime/discussions" -Changelog = "https://github.com/org/kaitai_awkward_runtime/releases" +Homepage = "https://github.com/ManasviGoyal/kaitai_awkward_runtime" +"Bug Tracker" = "https://github.com/ManasviGoyal/kaitai_awkward_runtime/issues" [tool.scikit-build] @@ -61,6 +61,12 @@ minimum-version = "0.4" build-dir = "build/{wheel_tag}" metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" sdist.include = ["src/kaitai_awkward_runtime/_version.py"] +cmake.verbose = true +logging.level = "INFO" +cmake.build-type = "Debug" + +[tool.scikit-build.cmake.define] +KSY = "schemas/animal.ksy" [tool.setuptools_scm] diff --git a/src/kaitai_awkward_runtime/__init__.py b/src/kaitai_awkward_runtime/__init__.py index 252fa70..8c2070e 100644 --- a/src/kaitai_awkward_runtime/__init__.py +++ b/src/kaitai_awkward_runtime/__init__.py @@ -10,3 +10,6 @@ from ._version import version as __version__ __all__ = ("__version__",) + + +from .lib.awkward_animal import * diff --git a/src/main.cpp b/src/main.cpp deleted file mode 100644 index cd3f59d..0000000 --- a/src/main.cpp +++ /dev/null @@ -1,66 +0,0 @@ -#include -#include -#include -#include -#include "../build/_deps/awkward-headers-src/layout-builder/awkward/LayoutBuilder.h" -#include "../src-animal/animal.h" - -namespace py = pybind11; - -/** - * Create a snapshot of the given builder, and return an `ak.Array` pyobject - * @tparam T type of builder - * @param builder builder - * @return pyobject of Awkward Array - */ -template -py::object snapshot_builder(const T &builder) { - // How much memory to allocate? - std::map names_nbytes = {}; - builder.buffer_nbytes(names_nbytes); - - // Allocate memory - std::map buffers = {}; - for (auto it: names_nbytes) { - uint8_t *ptr = new uint8_t[it.second]; - buffers[it.first] = (void *) ptr; - } - - // Write non-contiguous contents to memory - builder.to_buffers(buffers); - auto from_buffers = py::module::import("awkward").attr("from_buffers"); - - // Build Python dictionary containing arrays - py::dict container; - for (auto it: buffers) { - - py::capsule free_when_done(it.second, [](void *data) { - uint8_t *dataPtr = reinterpret_cast(data); - delete[] dataPtr; - }); - - uint8_t *data = reinterpret_cast(it.second); - container[py::str(it.first)] = py::array_t( - {names_nbytes[it.first]}, - {sizeof(uint8_t)}, - data, - free_when_done - ); - } - return from_buffers(builder.form(), builder.length(), container); -} - -/** - * Create array, and return its snapshot - * @return pyobject of Awkward Array - */ -py::object load(std::string file_path) { - std::ifstream infile(file_path, std::ifstream::binary); - kaitai::kstream ks(&infile); - animal_t zoo = animal_t(&ks); - return snapshot_builder(zoo.animal_builder); -} - -PYBIND11_MODULE(awkward_animal, m) { - m.def("load", &load); -} \ No newline at end of file