diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 5524435b6..264c7ef13 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,8 @@ [bumpversion] -current_version = 0.0.3 -commit = True +current_version = 0.0.5 +commit = False tag = False +commit_args = -s [bumpversion:file:csp/__init__.py] search = __version__ = "{current_version}" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8e74c4868..ab2b1a8c2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,17 +2,18 @@ * @robambalu @AdamGlustein @svatasoiu @alexddobkin # Packaging code -.bumpversion.cfg @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807 -CMakeLists.txt @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807 -Makefile @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807 -MANIFEST.in @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807 -pyproject.toml @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807 -setup.py @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807 -vcpkg.json @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807 +.bumpversion.cfg @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine +conda @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine +CMakeLists.txt @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine +Makefile @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine +MANIFEST.in @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine +pyproject.toml @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine +setup.py @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine +vcpkg.json @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine # CI/CD -.github @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807 -ci @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807 +.github @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine +ci @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine # Administrative .github/CODEOWNERS @robambalu diff --git a/.github/actions/setup-caches/action.yml b/.github/actions/setup-caches/action.yml index b3296b5e4..4fa38a489 100644 --- a/.github/actions/setup-caches/action.yml +++ b/.github/actions/setup-caches/action.yml @@ -2,16 +2,10 @@ name: Setup Caches description: 'Ensure various caches are setup like homebrew, vcpkg, ccache, etc' inputs: - cibuildwheel: - type: choice - description: "cibuildwheel run" - options: - - 'cp38' - - 'cp39' - - 'cp310' - - 'cp311' - - 'cp312' - default: 'cp39' + vcpkg: + type: boolean + description: "cache vcpkg assets" + default: true runs: using: 'composite' @@ -38,7 +32,7 @@ runs: mkdir -p /home/runner/vcpkg_download_cache echo "VCPKG_DEFAULT_BINARY_CACHE=/home/runner/vcpkg_cache" >> $GITHUB_ENV echo "VCPKG_DOWNLOADS=/home/runner/vcpkg_download_cache" >> $GITHUB_ENV - if: ${{ runner.os == 'Linux' }} + if: ${{ runner.os == 'Linux' && inputs.vcpkg == 'true' }} - name: Setup vcpkg cache (Linux) uses: actions/cache@v4 @@ -48,7 +42,7 @@ runs: /home/runner/vcpkg_download_cache key: vcpkg-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('vcpkg.json') }} restore-keys: vcpkg-${{ runner.os }}-${{ runner.arch }}- - if: ${{ runner.os == 'Linux' }} + if: ${{ runner.os == 'Linux' && inputs.vcpkg == 'true' }} - name: Setup vcpkg cache in shell (macOS) shell: bash @@ -57,7 +51,7 @@ runs: mkdir -p /Users/runner/vcpkg_download_cache echo "VCPKG_DEFAULT_BINARY_CACHE=/Users/runner/vcpkg_cache" >> $GITHUB_ENV echo "VCPKG_DOWNLOADS=/Users/runner/vcpkg_download_cache" >> $GITHUB_ENV - if: ${{ runner.os == 'macOS' }} + if: ${{ runner.os == 'macOS' && inputs.vcpkg == 'true' }} - name: Setup vcpkg cache (macOS) uses: actions/cache@v4 @@ -67,4 +61,23 @@ runs: /Users/runner/vcpkg_download_cache key: vcpkg-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('vcpkg.json') }} restore-keys: vcpkg-${{ runner.os }}-${{ runner.arch }}- - if: ${{ runner.os == 'macOS' }} + if: ${{ runner.os == 'macOS' && inputs.vcpkg == 'true'}} + + - name: Setup vcpkg cache in shell (Windows) + shell: bash + run: | + mkdir C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_cache + mkdir C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache + echo "VCPKG_DEFAULT_BINARY_CACHE=C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_cache" >> $GITHUB_ENV + echo "VCPKG_DOWNLOADS=C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache" >> $GITHUB_ENV + if: ${{ runner.os == 'Windows' && inputs.vcpkg == 'true'}} + + - name: Setup vcpkg cache (Windows) + uses: actions/cache@v4 + with: + path: | + C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_cache + C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache + key: vcpkg-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('vcpkg.json') }} + restore-keys: vcpkg-${{ runner.os }}-${{ runner.arch }}- + if: ${{ runner.os == 'Windows' && inputs.vcpkg == 'true'}} diff --git a/.github/actions/setup-dependencies/action.yml b/.github/actions/setup-dependencies/action.yml index dac9f9051..eeaa95f94 100644 --- a/.github/actions/setup-dependencies/action.yml +++ b/.github/actions/setup-dependencies/action.yml @@ -22,7 +22,6 @@ runs: - name: Install python dependencies shell: bash run: make requirements - if: ${{ runner.os == 'Linux' }} ################ # Linux # NOTE: skip for manylinux image @@ -54,6 +53,6 @@ runs: shell: bash run: make dependencies-win env: - VCPKG_DEFAULT_TRIPLET: x64-windows + VCPKG_DEFAULT_TRIPLET: x64-windows-static-md VCPKG_PLATFORM_TOOLSET: v143 if: ${{ runner.os == 'Windows' }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c7c758f8e..2fa3d75ed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,18 +38,19 @@ permissions: id-token: write # for pypi test release jobs: - #################################### - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# - #~~~~~~~~~~/##########\~~~~~~~~~~~~# - #~~~~~~~~~|#|~~~~~~~|#|~~~~~~~~~~~~# - #~~~~~~~~~|#|~~~~~~~|#|~~~~~~~~~~~~# - #~~~~~~~~~|#|~~~~~~~|#|~~~~~~~~~~~~# - #~~~~~~~~~|#|~~~~~~~|#|~~~~~~~~~~~~# - #~~~~~~~~~|#|~~~~~~~|#|~~~~~~~~~~~~# - #~~~~~~~~~\##########/~~~~~~~~~~~~~# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# - # Stage One - Initialize the build # - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + ######################################################## + #......................................................# + #..|########|..|###\.....|##|..|########|..|########|..# + #.....|##|.....|##|##\...|##|.....|##|.....|########|..# + #.....|##|.....|##|\##\..|##|.....|##|........|##|.....# + #.....|##|.....|##|.\##\.|##|.....|##|........|##|.....# + #.....|##|.....|##|..\##\|##|.....|##|........|##|.....# + #.....|##|.....|##|...\##\#/......|##|........|##|.....# + #..|########|..|##|....\##/....|########|.....|##|.....# + #......................................................# + ######################################################## + # Stage One - Initialize the build # + ######################################################## # This is so we can inspect the latest commit message from # both push and pull_request events (there is no # github.event.head_commit.message otherwise on pull @@ -109,18 +110,19 @@ jobs: FULL_RUN: ${{ github.event.inputs.ci-full }} if: ${{ github.event_name == 'workflow_dispatch' }} - ################################### - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# - #~~~~~~~~~~~/#######|~~~~~~~~~~~~~# - #~~~~~~~~~~/########|~~~~~~~~~~~~~# - #~~~~~~~~~/###/~|###|~~~~~~~~~~~~~# - #~~~~~~~~~~~~~~~|###|~~~~~~~~~~~~~# - #~~~~~~~~~~~~~~~|###|~~~~~~~~~~~~~# - #~~~~~~~~~~~~~~~|###|~~~~~~~~~~~~~# - #~~~~~~~~~~~~~~~|###|~~~~~~~~~~~~~# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# - # Stage One - Lint Python / C++ # - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + ######################################################## + #......................................................# + #..|##|........|########|..|###\.....|##|..|########|..# + #..|##|...........|##|.....|##|##\...|##|..|########|..# + #..|##|...........|##|.....|##|\##\..|##|.....|##|.....# + #..|##|...........|##|.....|##|.\##\.|##|.....|##|.....# + #..|##|...........|##|.....|##|..\##\|##|.....|##|.....# + #..|########|.....|##|.....|##|...\##\#/......|##|.....# + #..|########|..|########|..|##|....\##/.......|##|.....# + #......................................................# + ######################################################## + # Stage One - Lint Python / C++ # + ######################################################## lint: needs: - initialize @@ -151,18 +153,19 @@ jobs: - name: Python Lint Steps (Linux) run: make lint - ############################ - #~~~~~~~~~~~~~~~~~~~~~~~~~~# - #~~~~~~~~~/########\~~~~~~~# - #~~~~~~~|###|~~~\###\~~~~~~# - #~~~~~~~~~~~~~~/###/~~~~~~~# - #~~~~~~~~~~~~/###/~~~~~~~~~# - #~~~~~~~~~~/###/~~~~~~~~~~~# - #~~~~~~~~/###/~~~~~~~~~~~~~# - #~~~~~~|#############|~~~~~# - #~~~~~~~~~~~~~~~~~~~~~~~~~~# - # Stage Two - Build Python # - #~~~~~~~~~~~~~~~~~~~~~~~~~~# + ################################################################ + #..............................................................# + #..|########\..|##|..|##|..|########|..|##|........|#######\...# + #..|##|../##/..|##|..|##|.....|##|.....|##|........|##|..\##\..# + #..|##|./##/...|##|..|##|.....|##|.....|##|........|##|..|##|..# + #..|##||#<.....|##|..|##|.....|##|.....|##|........|##|..|##|..# + #..|##|.\##\...|##|..|##|.....|##|.....|##|........|##|..|##|..# + #..|##|..\##\..|##|..|##|.....|##|.....|########|..|##|../##/..# + #..|########/..|########|..|########|..|########|..|#######/...# + #..............................................................# + ################################################################ + # Stage Two - Build Python # + ################################################################ build: needs: - initialize @@ -173,7 +176,7 @@ jobs: - ubuntu-22.04 # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md - macos-12 # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md - macos-14 # https://github.com/actions/runner-images/blob/main/images/macos/macos-14-arm64-Readme.md - # - windows-2022 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md + - windows-2019 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2019-Readme.md python-version: - "3.8" - "3.9" @@ -245,9 +248,22 @@ jobs: - os: macos-14 python-version: "3.9" - # windows is slow, so dont build unless its a full run - # - is-full-run: false - # os: windows-2022 + # Avoid extra resources for windows build + - is-full-run: false + os: windows-2019 + python-version: "3.8" + + - is-full-run: false + os: windows-2019 + python-version: "3.9" + + - is-full-run: false + os: windows-2019 + python-version: "3.10" + + - is-full-run: false + os: windows-2019 + python-version: "3.11" # avoid unnecessary use of mac resources - is-full-run: false @@ -281,14 +297,12 @@ jobs: - name: Set up Caches uses: ./.github/actions/setup-caches - with: - cibuildwheel: '${{ matrix.cibuildwheel }}' - name: Set up dependencies uses: ./.github/actions/setup-dependencies with: cibuildwheel: '${{ matrix.cibuildwheel }}' - if: ${{ runner.os == 'macOS' }} + if: ${{ runner.os != 'Linux' }} ######## # Linux @@ -322,13 +336,14 @@ jobs: ########## # Windows - - name: Python Build Steps (Windows vc14.3) + - name: Python Build Steps (Windows 2019) run: make dist-py-cibw env: CIBW_BUILD: "${{ matrix.cibuildwheel }}-win_amd64" - CMAKE_GENERATOR: Visual Studio 17 2022 - CIBW_ENVIRONMENT_WINDOWS: TODO="todo" - if: ${{ matrix.os == 'windows-2022' }} + CSP_GENERATOR: "Visual Studio 16 2019" + VCPKG_DEFAULT_BINARY_CACHE: C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_cache + VCPKG_DOWNLOADS: C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache + if: ${{ matrix.os == 'windows-2019' }} ########## # Common @@ -341,18 +356,28 @@ jobs: name: csp-dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }} path: dist/*.whl - ############################# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~# - #~~~~~~~~~~/########\~~~~~~~# - #~~~~~~~~~|##|~~~~|##|~~~~~~# - #~~~~~~~~~~~~~~~~~|##|~~~~~~# - #~~~~~~~~~~~~~|######|~~~~~~# - #~~~~~~~~~~~~~~~~~|##|~~~~~~# - #~~~~~~~~~|##|~~~~|##|~~~~~~# - #~~~~~~~~~\#########/~~~~~~~# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~# - # Stage Three - Build SDist # - #~~~~~~~~~~~~~~~~~~~~~~~~~~~# + + ################################################################ + #..............................................................# + #..|########\..|##|..|##|..|########|..|##|........|#######\...# + #..|##|../##/..|##|..|##|.....|##|.....|##|........|##|..\##\..# + #..|##|./##/...|##|..|##|.....|##|.....|##|........|##|..|##|..# + #..|##||#<.....|##|..|##|.....|##|.....|##|........|##|..|##|..# + #..|##|.\##\...|##|..|##|.....|##|.....|##|........|##|..|##|..# + #..|##|..\##\..|##|..|##|.....|##|.....|########|..|##|../##/..# + #..|########/..|########|..|########|..|########|..|#######/...# + #..............................................................# + #..../####\....|#######\...|########|..../####\....|########|..# + #../##/..\##\..|##|..\##\.....|##|...../##/..\##\..|########|..# + #...\##\.......|##|..|##|.....|##|......\##\..........|##|.....# + #.....\##\.....|##|..|##|.....|##|........\##\........|##|.....# + #.......\##\...|##|..|##|.....|##|..........\##\......|##|.....# + #..\##\./##/...|##|../##/.....|##|.....\##\./##/......|##|.....# + #...\####/.....|#######/...|########|...\####/........|##|.....# + #..............................................................# + ################################################################ + # Stage Three - Build SDist # + ################################################################ build_sdist: needs: - initialize @@ -392,18 +417,19 @@ jobs: name: csp-sdist path: dist/*.tar.gz - ############################# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~# - #~~~~~~~|##|~~~~|##|~~~~~~~~# - #~~~~~~~|##|~~~~|##|~~~~~~~~# - #~~~~~~~|##|~~~~|##|~~~~~~~~# - #~~~~~~~|##########|~~~~~~~~# - #~~~~~~~~~~~~~~~|##|~~~~~~~~# - #~~~~~~~~~~~~~~~|##|~~~~~~~~# - #~~~~~~~~~~~~~~~|##|~~~~~~~~# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~# - # Stage Four - Test Python # - #~~~~~~~~~~~~~~~~~~~~~~~~~~~# + #################################################### + #..................................................# + #..|########|..|########|..../####\....|########|..# + #..|########|..|##|......../##/..\##\..|########|..# + #.....|##|.....|##|.........\##\..........|##|.....# + #.....|##|.....|########|.....\##\........|##|.....# + #.....|##|.....|##|.............\##\......|##|.....# + #.....|##|.....|##|........\##\../##/.....|##|.....# + #.....|##|.....|########|...\####/........|##|.....# + #..................................................# + #################################################### + # Stage Four - Test Python # + #################################################### test: needs: - initialize @@ -415,7 +441,7 @@ jobs: - ubuntu-22.04 - macos-12 - macos-14 - # - windows-2022 + - windows-2019 python-version: - 3.8 - 3.9 @@ -434,9 +460,22 @@ jobs: - os: macos-14 python-version: "3.9" - # windows is slow, so dont build unless its a full run - # - is-full-run: false - # os: windows-2022 + # Avoid extra resources for windows build + - is-full-run: false + os: windows-2019 + python-version: "3.8" + + - is-full-run: false + os: windows-2019 + python-version: "3.9" + + - is-full-run: false + os: windows-2019 + python-version: "3.10" + + - is-full-run: false + os: windows-2019 + python-version: "3.11" # avoid unnecessary use of mac resources - is-full-run: false @@ -483,7 +522,9 @@ jobs: if: ${{ runner.os == 'Linux' }} - name: Install wheel (Linux) - run: python -m pip install -U *manylinux*.whl --target . + run: | + python -m pip install -U *manylinux*.whl + python -m pip install -U --no-deps *manylinux*.whl --target . if: ${{ runner.os == 'Linux' }} ######## @@ -494,17 +535,27 @@ jobs: if: ${{ runner.os == 'macOS' }} - name: Install wheel (OSX x86) - run: python -m pip install -U *x86*.whl --target . + run: | + python -m pip install -U *x86*.whl + python -m pip install -U --no-deps *x86*.whl --target . if: ${{ runner.os == 'macOS' && runner.arch == 'X64' }} - name: Install wheel (OSX arm) - run: python -m pip install -U *arm64*.whl --target . + run: | + python -m pip install -U *arm64*.whl + python -m pip install -U --no-deps *arm64*.whl --target . if: ${{ runner.os == 'macOS' && runner.arch == 'ARM64' }} ######## # Windows + - name: Install test dependencies (Windows) + run: choco install graphviz --no-progress -y + if: ${{ runner.os == 'Windows' }} + - name: Install wheel (windows) - run: python -m pip install -U (Get-ChildItem .\*.whl | Select-Object -Expand FullName) --target . + run: | + python -m pip install -U (Get-ChildItem .\*.whl | Select-Object -Expand FullName) + python -m pip install -U --no-deps (Get-ChildItem .\*.whl | Select-Object -Expand FullName) --target . if: ${{ runner.os == 'Windows' }} ########## @@ -512,16 +563,25 @@ jobs: - name: Python Test Steps run: make test - ######################################## - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# - #~~~~~~~~~~~~~~|##########|~~~~~~~~~~~~# - #~~~~~~~~~~~~~~|##|~~~~~~~~~~~~~~~~~~~~# - #~~~~~~~~~~~~~~|##|~~~~~~~~~~~~~~~~~~~~# - #~~~~~~~~~~~~~~|##########|~~~~~~~~~~~~# - #~~~~~~~~~~~~~~~~~~~~~~|##|~~~~~~~~~~~~# - #~~~~~~~~~~~~~~|##|~~~~|##|~~~~~~~~~~~~# - #~~~~~~~~~~~~~~|##########|~~~~~~~~~~~~# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + ################################################################ + #..............................................................# + #..|########|..|########|..../####\....|########|..............# + #..|########|..|##|......../##/..\##\..|########|..............# + #.....|##|.....|##|.........\##\..........|##|.................# + #.....|##|.....|########|.....\##\........|##|.................# + #.....|##|.....|##|.............\##\......|##|.................# + #.....|##|.....|##|........\##\../##/.....|##|.................# + #.....|##|.....|########|...\####/........|##|.................# + #..............................................................# + #..../####\....|#######\...|########|..../####\....|########|..# + #../##/..\##\..|##|..\##\.....|##|...../##/..\##\..|########|..# + #...\##\.......|##|..|##|.....|##|......\##\..........|##|.....# + #.....\##\.....|##|..|##|.....|##|........\##\........|##|.....# + #.......\##\...|##|..|##|.....|##|..........\##\......|##|.....# + #..\##\./##/...|##|../##/.....|##|.....\##\./##/......|##|.....# + #...\####/.....|#######/...|########|...\####/........|##|.....# + #..............................................................# + ################################################################ # Stage Four - Build / test the SDist # #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# test_sdist: @@ -553,8 +613,6 @@ jobs: - name: Set up Caches uses: ./.github/actions/setup-caches - with: - cibuildwheel: 'cp39' - name: Install python dependencies run: make requirements @@ -565,7 +623,9 @@ jobs: path: dist/ - name: Install sdist - run: python -m pip install -U -vvv dist/csp*.tar.gz --target . + run: | + python -m pip install -U -vvv dist/csp*.tar.gz + python -m pip install -U --no-deps -vvv dist/csp*.tar.gz --target . env: CCACHE_DIR: /home/runner/work/csp/csp/.ccache VCPKG_DEFAULT_BINARY_CACHE: /home/runner/vcpkg_cache @@ -577,19 +637,27 @@ jobs: CSP_TEST_SKIP_EXAMPLES: "1" - ################################# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# - #~~~~~~~~~|##########|~~~~~~~~~~# - #~~~~~~~~~|##|~~~~~~~~~~~~~~~~~~# - #~~~~~~~~~|##|~~~~~~~~~~~~~~~~~~# - #~~~~~~~~~|##########|~~~~~~~~~~# - #~~~~~~~~~|##|~~~~|##|~~~~~~~~~~# - #~~~~~~~~~|##|~~~~|##|~~~~~~~~~~# - #~~~~~~~~~|##########|~~~~~~~~~~# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# - # Test Dependencies/Regressions # - #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# + #################################################### + #..................................................# + #..|########|..|########|..../####\....|########|..# + #..|########|..|##|......../##/..\##\..|########|..# + #.....|##|.....|##|.........\##\..........|##|.....# + #.....|##|.....|########|.....\##\........|##|.....# + #.....|##|.....|##|.............\##\......|##|.....# + #.....|##|.....|##|........\##\../##/.....|##|.....# + #.....|##|.....|########|...\####/........|##|.....# + #..................................................# + #..|#######\...|########|..|########\..../####\....# + #..|##|..\##\..|##|........|##|../##/../##/..\##\..# + #..|##|..|##|..|##|........|##|./##/....\##\.......# + #..|##|..|##|..|########|..|##||##/.......\##\.....# + #..|##|..|##|..|##|........|##|.............\##\...# + #..|##|../##/..|##|........|##|........\##\../##/..# + #..|#######/...|########|..|##|.........\####/.....# + #..................................................# + #################################################### + # Test Dependencies/Regressions # + #################################################### test_dependencies: needs: - initialize @@ -632,7 +700,9 @@ jobs: name: csp-dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }} - name: Install wheel - run: python -m pip install -U *manylinux*.whl --target . + run: | + python -m pip install -U *manylinux*.whl + python -m pip install -U --no-deps *manylinux*.whl --target . - name: Install package - ${{ matrix.package }} run: python -m pip install -U "${{ matrix.package }}" @@ -645,31 +715,42 @@ jobs: run: make test if: ${{ contains( 'numpy', matrix.package )}} - ########################### - #~~~~~~~~~~~~~~~~~~~~~~~~~# - #~~~~~~|#############|~~~~# - #~~~~~~|#|~~~~~~~/##/~~~~~# - #~~~~~~|#|~~~~~/##/~~~~~~~# - #~~~~~~~~~~~~/##/~~~~~~~~~# - #~~~~~~~~~~/##/~~~~~~~~~~~# - #~~~~~~~~/##/~~~~~~~~~~~~~# - #~~~~~~/##/~~~~~~~~~~~~~~~# - #~~~~~~~~~~~~~~~~~~~~~~~~~# - # Test Service Adapters # - #~~~~~~~~~~~~~~~~~~~~~~~~~# + ########################################################################################################### + #.........................................................................................................# + #..|########|..|########|..../####\....|########|.........................................................# + #..|########|..|##|......../##/..\##\..|########|.........................................................# + #.....|##|.....|##|.........\##\..........|##|............................................................# + #.....|##|.....|########|.....\##\........|##|............................................................# + #.....|##|.....|##|.............\##\......|##|............................................................# + #.....|##|.....|##|........\##\../##/.....|##|............................................................# + #.....|##|.....|########|...\####/........|##|............................................................# + #.........................................................................................................# + #...../#####\.....|#######\....../#####\.....|########\..|########|..|########|..|########\...../####\....# + #..../##/.\##\....|##|..\##\..../##/.\##\....|##|../##/..|########|..|##|........|##|../##/.../##/..\##\..# + #.../##/...\##\...|##|..|##|.../##/...\##\...|##|./##/......|##|.....|##|........|##|./##/.....\##\.......# + #../###########\..|##|..|##|../###########\..|##||##/.......|##|.....|########|..|##||##<........\##\.....# + #..|##|.....|##|..|##|..|##|..|##|.....|##|..|##|...........|##|.....|##|........|##|.\##\.........\##\...# + #..|##|.....|##|..|##|../##/..|##|.....|##|..|##|...........|##|.....|##|........|##|..\##\...\##\../##/..# + #..|##|.....|##|..|#######/...|##|.....|##|..|##|...........|##|.....|########|..|##|...\##\...\####/.....# + #.........................................................................................................# + ########################################################################################################### + # Test Service Adapters # + ########################################################################################################### # Coming soon! - ############################# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~# - #~~~~~~|#############|~~~~~~# - #~~~~~~|#|~~~~~~~~|##|~~~~~~# - #~~~~~~|#|~~~~~~~~|##|~~~~~~# - #~~~~~~|#############|~~~~~~# - #~~~~~~|#|~~~~~~~~|##|~~~~~~# - #~~~~~~|#|~~~~~~~~|##|~~~~~~# - #~~~~~~|#############|~~~~~~# - #~Upload Release Artifacts~~# - #~~~~~~~~~~~~~~~~~~~~~~~~~~~# + ############################################################################################ + #..........................................................................................# + #..|########\...|########|..|##|........|########|...../#####\......./####\....|########|..# + #..|##|../##/...|##|........|##|........|##|........../##/.\##\..../##/..\##\..|##|........# + #..|##|./##/....|##|........|##|........|##|........./##/...\##\....\##\.......|##|........# + #..|##||##<.....|########|..|##|........|########|../###########\.....\##\.....|########|..# + #..|##|.\##\....|##|........|##|........|##|........|##|.....|##|.......\##\...|##|........# + #..|##|..\##\...|##|........|########|..|##|........|##|.....|##|..\##\./##/...|##|........# + #..|##|...\##\..|########|..|########|..|########|..|##|.....|##|...\####/.....|########|..# + #..........................................................................................# + ############################################################################################ + # Upload Release Artifacts # + ############################################################################################ # only publish artifacts on tags, but otherwise this always runs # Note this whole workflow only triggers on release tags (e.g. "v0.1.0") diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index d6073fd0a..c64fdb232 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -35,6 +35,7 @@ jobs: - ubuntu-22.04 - macos-14 - macos-12 + - windows-2019 runs-on: ${{ matrix.os }} steps: - name: Checkout @@ -48,22 +49,51 @@ jobs: bash cache-environment: true post-cleanup: 'all' + if: ${{ runner.os != 'Windows' }} + + - uses: mamba-org/setup-micromamba@v1 + with: + micromamba-version: '1.5.7-0' + environment-file: conda/dev-environment-win.yml + init-shell: >- + cmd.exe + cache-environment: true + post-cleanup: 'all' + if: ${{ runner.os == 'Windows' }} - name: Set up Caches uses: ./.github/actions/setup-caches with: - cibuildwheel: 'cp312' + vcpkg: false - name: Python Lint Steps run: make lint shell: micromamba-shell {0} + if: ${{ runner.os != 'Windows' }} + + - name: Python Lint Steps ( Windows ) + run: make lint + shell: cmd /C call {0} + if: ${{ runner.os == 'Windows' }} - name: Python Build Steps run: make build-conda shell: micromamba-shell {0} + if: ${{ runner.os != 'Windows' }} + + - name: Python Build Steps ( Windows ) + env: + CSP_GENERATOR: "Visual Studio 16 2019" + run: make build-conda + shell: cmd /C call {0} + if: ${{ runner.os == 'Windows' }} - name: Python Test Steps run: make test shell: micromamba-shell {0} + if: ${{ runner.os != 'Windows' }} - \ No newline at end of file + - name: Python Test Steps ( Windows ) + run: make test + shell: cmd /C call {0} + if: ${{ runner.os == 'Windows' }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 97cf3c088..4bb693281 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ # Project Configuration # ######################### cmake_minimum_required(VERSION 3.20.0) -project(csp VERSION "0.0.3") +project(csp VERSION "0.0.5") set(CMAKE_CXX_STANDARD 20) ################################################################################################################################################### @@ -152,28 +152,27 @@ endif() ################################################################################################################################################### # Flags # -######### # Optimization Flags if(WIN32) if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \ - /DEBUG \ - /Z7 \ - /Zi \ - ") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /DEBUG /Z7 /Zi") add_definitions(-DCSP_DEBUG) else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \ - /NDEBUG \ - /O2 \ - ") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2") add_definitions(-DNDEBUG) endif() + + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /MP /bigobj") + foreach(warning 4244 4251 4267 4275 4290 4786 4305 4996) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd${warning}") + endforeach(warning) + add_compile_definitions(WIN32 _WIN32) + else() if(CSP_BUILD_NO_CXX_ABI) - # TODO windows set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") if (COVERAGE) # TODO windows add_compile_options(--coverage) @@ -181,7 +180,6 @@ else() link_libraries(gcov) endif () if (GPROF_BUILD) - # TODO windows set(CMAKE_CXX_FLAGS "-pg ${CMAKE_CXX_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "-pg ${CMAKE_EXE_LINKER_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "-pg ${CMAKE_SHARED_LINKER_FLAGS}") @@ -197,6 +195,7 @@ else() -O3 \ -g0 \ -Wall \ + -Werror \ -Wno-deprecated-declarations \ -Wno-deprecated \ ") @@ -209,14 +208,6 @@ else() endif() endif() -# Other Flags -if(WIN32) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /MP /bigobj") - foreach(warning 4244 4251 4267 4275 4290 4786 4305 4996) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd${warning}") - endforeach(warning) - add_compile_definitions(WIN32 _WIN32) -endif() ################################################################################################################################################### # Messages # @@ -250,13 +241,25 @@ function(csp_autogen MODULE_NAME DEST_FILENAME HEADER_NAME_OUTVAR SOURCE_NAME_OU set(CSP_AUTOGEN_EXTRA_ARGS "") endif() - add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.cpp" "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.h" - COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${PROJECT_BINARY_DIR}/lib:${CMAKE_SOURCE_DIR}:$$PYTHONPATH" "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/lib:$$LD_LIBRARY_PATH}" ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/csp/build/csp_autogen.py -m ${MODULE_NAME} -d ${CMAKE_CURRENT_BINARY_DIR}/csp_autogen -o ${DEST_FILENAME} ${CSP_AUTOGEN_EXTRA_ARGS} - COMMENT "generating csp c++ types from module ${MODULE_NAME} ${PROJECT_BINARY_DIR} ${CMAKE_SOURCE_DIR}/csp/build/csp_autogen.py" - DEPENDS mkdir_autogen_${MODULE_NAME} + cmake_path(SET CSP_AUTOGEN_MODULE_PATH NORMALIZE "${CMAKE_SOURCE_DIR}/csp/build/csp_autogen.py") + cmake_path(SET CSP_AUTOGEN_DESTINATION_FOLDER NORMALIZE "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen") + cmake_path(SET CSP_AUTOTGEN_CPP_OUT NORMALIZE "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.cpp") + cmake_path(SET CSP_AUTOTGEN_H_OUT NORMALIZE "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.h") + + if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + set(CSP_AUTOGEN_PYTHONPATH ${PROJECT_BINARY_DIR}/lib/${CMAKE_BUILD_TYPE};${CMAKE_SOURCE_DIR};%PYTHONPATH% ) + else() + set(CSP_AUTOGEN_PYTHONPATH ${PROJECT_BINARY_DIR}/lib:${CMAKE_SOURCE_DIR}:$$PYTHONPATH ) + endif() + + add_custom_command(OUTPUT "${CSP_AUTOTGEN_CPP_OUT}" "${CSP_AUTOTGEN_H_OUT}" + COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CSP_AUTOGEN_PYTHONPATH}" ${Python_EXECUTABLE} ${CSP_AUTOGEN_MODULE_PATH} -m ${MODULE_NAME} -d ${CSP_AUTOGEN_DESTINATION_FOLDER} -o ${DEST_FILENAME} ${CSP_AUTOGEN_EXTRA_ARGS} + COMMENT "generating csp c++ types from module ${MODULE_NAME}" + DEPENDS mkdir_autogen_${MODULE_NAME} ${CMAKE_SOURCE_DIR}/csp/build/csp_autogen.py ${CMAKE_SOURCE_DIR}/${MODULE_FILENAME} - csptypesimpl ) + csptypesimpl + ) set(${SOURCE_NAME_OUTVAR} "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.cpp" PARENT_SCOPE ) set(${HEADER_NAME_OUTVAR} "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.h" PARENT_SCOPE ) @@ -329,6 +332,9 @@ if(NOT WIN32) # static suffix is _static.a # TODO decide if we want this set(CMAKE_STATIC_LIBRARY_SUFFIX _static.a) +else() + # shared suffix is .pyd for windows + set(CMAKE_SHARED_LIBRARY_SUFFIX .pyd) endif() @@ -341,7 +347,15 @@ include_directories("${CMAKE_BINARY_DIR}/cpp") set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +if(WIN32) + # On windows, force dlls into lib folder + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) + set(CSP_RUNTIME_INSTALL_SUBDIR lib/) +else() + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + set(CSP_RUNTIME_INSTALL_SUBDIR bin/) +endif() add_subdirectory(cpp/csp/adapters) add_subdirectory(cpp/csp/core) diff --git a/Makefile b/Makefile index c1897007e..6c8bf66f0 100644 --- a/Makefile +++ b/Makefile @@ -1,34 +1,33 @@ EXTRA_ARGS := -UNAME := $(shell uname) -ifeq ($(UNAME), Linux) -NPROC = $(shell nproc) -endif -ifeq ($(UNAME), Darwin) -NPROC = $(shell sysctl -n hw.physicalcpu) -endif - ######### # BUILD # ######### .PHONY: requirements develop build build-debug build-conda install requirements: ## install python dev and runtime dependencies +ifeq ($(OS),Windows_NT) + Powershell.exe -executionpolicy bypass -noprofile .\ci\scripts\windows\make_requirements.ps1 +else python -m pip install toml python -m pip install `python -c 'import toml; c = toml.load("pyproject.toml"); print("\n".join(c["build-system"]["requires"]))'` python -m pip install `python -c 'import toml; c = toml.load("pyproject.toml"); print("\n".join(c["project"]["optional-dependencies"]["develop"]))'` +endif develop: requirements ## install dependencies and build library python -m pip install -e .[develop] build: ## build the library - python setup.py build build_ext --inplace -- -- -j$(NPROC) + python setup.py build build_ext --inplace build-debug: ## build the library ( DEBUG ) - May need a make clean when switching from regular build to build-debug and vice versa - SKBUILD_CONFIGURE_OPTIONS="" DEBUG=1 python setup.py build build_ext --inplace -- -- -j$(NPROC) + SKBUILD_CONFIGURE_OPTIONS="" DEBUG=1 python setup.py build build_ext --inplace build-conda: ## build the library in Conda - CSP_USE_VCPKG=0 python setup.py build build_ext --inplace -- -- -j$(NPROC) + python setup.py build build_ext --csp-no-vcpkg --inplace + +build-conda-debug: ## build the library ( DEBUG ) - in Conda + SKBUILD_CONFIGURE_OPTIONS="" DEBUG=1 python setup.py build build_ext --csp-no-vcpkg --inplace install: ## install library python -m pip install . @@ -90,7 +89,11 @@ test-py: ## Clean and Make unit tests python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS) test-cpp: ## Make C++ unit tests +ifneq ($(OS),Windows_NT) for f in ./csp/tests/bin/*; do $$f; done || (echo "TEST FAILED" && exit 1) +else + .\ci\scripts\windows\run_cpp_tests.bat +endif coverage-py: python -m pytest -v csp/tests --junitxml=junit.xml --cov=csp --cov-report xml --cov-report html --cov-branch --cov-fail-under=80 --cov-report term-missing $(TEST_ARGS) @@ -177,8 +180,13 @@ deep-clean: ## clean everything from the repository git clean -fdx clean: ## clean the repository +ifneq ($(OS),Windows_NT) rm -rf .coverage coverage cover htmlcov logs build dist wheelhouse *.egg-info rm -rf csp/lib csp/bin csp/include _skbuild +else + del /s /q .coverage coverage cover htmlcov logs build dist wheelhouse *.egg-info + del /s/ q csp\lib csp\bin csp\include _skbuild +endif ################ # Dependencies # @@ -195,11 +203,11 @@ dependencies-debian: ## install dependencies for linux dependencies-fedora: ## install dependencies for linux yum install -y automake bison ccache cmake curl flex perl-IPC-Cmd tar unzip zip -dependencies-vcpkg: ## install dependnecies via vcpkg +dependencies-vcpkg: ## install dependencies via vcpkg cd vcpkg && ./bootstrap-vcpkg.sh && ./vcpkg install -dependencies-win: ## install dependnecies via windows (vcpkg) - cd vcpkg && ./bootstrap-vcpkg.bat && ./vcpkg install +dependencies-win: ## install dependencies via windows + choco install cmake curl winflexbison ninja unzip zip --no-progress -y ############################################################################################ # Thanks to Francoise at marmelab.com for this diff --git a/NOTICE b/NOTICE index 1a612c0e5..4f42bf946 100644 --- a/NOTICE +++ b/NOTICE @@ -2468,6 +2468,13 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java + +These file are derived from code from Netty, which is made available under the +Apache License 2.0. + vcpkg_installed/x64-linux/share/boost-algorithm/copyright @@ -2637,7 +2644,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/x64-linux/share/boost-bind/copyright +vcpkg_installed/x64-linux/share/boost-beast/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2665,7 +2672,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/x64-linux/share/boost-build/copyright +vcpkg_installed/x64-linux/share/boost-bind/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2721,6 +2728,34 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +vcpkg_installed/x64-linux/share/boost-cmake/copyright + + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + vcpkg_installed/x64-linux/share/boost-concept-check/copyright @@ -3057,6 +3092,34 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +vcpkg_installed/x64-linux/share/boost-endian/copyright + + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + vcpkg_installed/x64-linux/share/boost-exception/copyright @@ -3225,6 +3288,34 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +vcpkg_installed/x64-linux/share/boost-headers/copyright + + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + vcpkg_installed/x64-linux/share/boost-integer/copyright @@ -3393,6 +3484,34 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +vcpkg_installed/x64-linux/share/boost-logic/copyright + + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + vcpkg_installed/x64-linux/share/boost-math/copyright @@ -3757,7 +3876,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/x64-linux/share/boost-rational/copyright +vcpkg_installed/x64-linux/share/boost-regex/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3785,7 +3904,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/x64-linux/share/boost-regex/copyright +vcpkg_installed/x64-linux/share/boost-scope/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3897,6 +4016,34 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +vcpkg_installed/x64-linux/share/boost-static-string/copyright + + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + vcpkg_installed/x64-linux/share/boost-system/copyright @@ -4037,7 +4184,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/x64-linux/share/boost-typeof/copyright +vcpkg_installed/x64-linux/share/boost-type-index/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4065,7 +4212,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/x64-linux/share/boost-type-traits/copyright +vcpkg_installed/x64-linux/share/boost-typeof/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4093,7 +4240,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/x64-linux/share/boost-unordered/copyright +vcpkg_installed/x64-linux/share/boost-type-traits/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4121,7 +4268,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/x64-linux/share/boost-utility/copyright +vcpkg_installed/x64-linux/share/boost-unordered/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4149,7 +4296,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/x64-linux/share/boost-variant2/copyright +vcpkg_installed/x64-linux/share/boost-utility/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4177,32 +4324,32 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/x64-linux/share/boost-vcpkg-helpers/copyright - +vcpkg_installed/x64-linux/share/boost-variant2/copyright -Copyright (c) Microsoft Corporation -All rights reserved. - -MIT License +Boost Software License - Version 1.0 - August 17th, 2003 -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. -THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. vcpkg_installed/x64-linux/share/boost-winapi/copyright @@ -4546,7 +4693,8 @@ LICENSE -------------------------------------------------------------- librdkafka - Apache Kafka C driver library -Copyright (c) 2012-2020, Magnus Edenhill +Copyright (c) 2012-2022, Magnus Edenhill + 2023, Confluent Inc. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -5746,6 +5894,58 @@ registered in some jurisdictions. All other trademarks and registered trademarks mentioned herein are the property of their respective owners. +vcpkg_installed/x64-linux/share/utf8-range/copyright + + +MIT License + +Copyright (c) 2019 Yibo Cai +Copyright 2022 Google LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +vcpkg_installed/x64-linux/share/vcpkg-boost/copyright + + +MIT License + +Copyright (c) Microsoft Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be included in all copies +or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + vcpkg_installed/x64-linux/share/vcpkg-cmake-config/copyright @@ -5824,156 +6024,6 @@ CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFT OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/x64-linux/share/websocketpp/copyright - - -Main Library: - -Copyright (c) 2014, Peter Thorson. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the WebSocket++ Project nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL PETER THORSON BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Bundled Libraries: - -****** Base 64 Library (base64/base64.hpp) ****** -base64.hpp is a repackaging of the base64.cpp and base64.h files into a -single header suitable for use as a header only library. This conversion was -done by Peter Thorson (webmaster@zaphoyd.com) in 2012. All modifications to -the code are redistributed under the same license as the original, which is -listed below. - -base64.cpp and base64.h - -Copyright (C) 2004-2008 René Nyffenegger - -This source code is provided 'as-is', without any express or implied -warranty. In no event will the author be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - -1. The origin of this source code must not be misrepresented; you must not - claim that you wrote the original source code. If you use this source code - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - -2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original source code. - -3. This notice may not be removed or altered from any source distribution. - -René Nyffenegger rene.nyffenegger@adp-gmbh.ch - -****** SHA1 Library (sha1/sha1.hpp) ****** -sha1.hpp is a repackaging of the sha1.cpp and sha1.h files from the shallsha1 -library (http://code.google.com/p/smallsha1/) into a single header suitable for -use as a header only library. This conversion was done by Peter Thorson -(webmaster@zaphoyd.com) in 2013. All modifications to the code are redistributed -under the same license as the original, which is listed below. - - Copyright (c) 2011, Micael Hildenborg - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Micael Hildenborg nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY Micael Hildenborg ''AS IS'' AND ANY - EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL Micael Hildenborg BE LIABLE FOR ANY - DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -****** MD5 Library (common/md5.hpp) ****** -md5.hpp is a reformulation of the md5.h and md5.c code from -http://www.opensource.apple.com/source/cups/cups-59/cups/md5.c to allow it to -function as a component of a header only library. This conversion was done by -Peter Thorson (webmaster@zaphoyd.com) in 2012 for the WebSocket++ project. The -changes are released under the same license as the original (listed below) - -Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - -1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. -2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. -3. This notice may not be removed or altered from any source distribution. - -L. Peter Deutsch -ghost@aladdin.com - -****** UTF8 Validation logic (utf8_validation.hpp) ****** -utf8_validation.hpp is adapted from code originally written by Bjoern Hoehrmann -. See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for -details. - -The original license: - -Copyright (c) 2008-2009 Bjoern Hoehrmann - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - vcpkg_installed/x64-linux/share/xsimd/copyright @@ -6418,33 +6468,3 @@ Public License instead of this License. -vcpkg_installed/x64-linux/tools/boost-build/src/engine/debian/copyright - - -This package was debianized by Vladimir Prus on -Wed, 17 July 2002, 19:27:00 +0400. - -Copyright: - - /+\ - +\ Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc. - \+/ - - This is Release 2.4 of Jam/MR, a make-like program. - - License is hereby granted to use this software and distribute it - freely, as long as this copyright notice is retained and modifications - are clearly marked. - - ALL WARRANTIES ARE HEREBY DISCLAIMED. - -Some portions are also: - - Copyright 2001-2006 David Abrahams. - Copyright 2002-2006 Rene Rivera. - Copyright 2003-2006 Vladimir Prus. - - Distributed under the Boost Software License, Version 1.0. - (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - - diff --git a/README.md b/README.md index 1e9abd50b..2821d3289 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![PyPI](https://img.shields.io/pypi/v/csp.svg?style=flat)](https://pypi.python.org/pypi/csp) [![License](https://img.shields.io/badge/license-Apache--2.0-green)](https://github.com/Point72/csp/LICENSE) [![Build Status](https://github.com/Point72/csp/actions/workflows/build.yml/badge.svg)](https://github.com/Point72/csp/actions/workflows/build.yml) -[![Python Versions](https://img.shields.io/badge/python-3.8_%7C_3.9_%7C_3.10_%7C_3.11-blue)](https://github.com/Point72/csp/blob/main/pyproject.toml) +[![Python Versions](https://img.shields.io/badge/python-3.8_%7C_3.9_%7C_3.10_%7C_3.11_%7C_3.12-blue)](https://github.com/Point72/csp/blob/main/pyproject.toml)
@@ -18,7 +18,7 @@ The high level goal of `csp` is to make writing realtime code simple and performant. Write event driven code once, test it in simulation, then deploy as realtime without any code changes. -Here is a very simple example of a small `csp` program to calculate a [bid-ask spread](https://www.investopedia.com/terms/b/bid-askspread.asp). In this example, we use a constant bid and ask, but in the real world you might pipe these directly into your live streaming data source, or into your historical data source, without modifications to your core logic. +Here is a very simple example of a small `csp` program to calculate a [bid-ask spread](https://en.wikipedia.org/wiki/Bid%E2%80%93ask_spread). In this example, we use a constant bid and ask, but in the real world you might pipe these directly in from your live streaming data source, or in from your historical data source, without modifications to your core logic. ```python import csp diff --git a/ci/scripts/windows/make_requirements.ps1 b/ci/scripts/windows/make_requirements.ps1 new file mode 100644 index 000000000..6f17381a0 --- /dev/null +++ b/ci/scripts/windows/make_requirements.ps1 @@ -0,0 +1,3 @@ +python -m pip install toml +python -m pip install $(python -c "import toml; c = toml.load('pyproject.toml'); print('\n'.join(c['build-system']['requires']))") +python -m pip install $(python -c "import toml; c = toml.load('pyproject.toml'); print('\n'.join(c['project']['optional-dependencies']['develop']))") \ No newline at end of file diff --git a/ci/scripts/windows/run_cpp_tests.bat b/ci/scripts/windows/run_cpp_tests.bat new file mode 100644 index 000000000..592e05deb --- /dev/null +++ b/ci/scripts/windows/run_cpp_tests.bat @@ -0,0 +1,9 @@ +@echo off +for %%X in (.\csp\tests\bin\*.exe) do ( + echo Executing: %%X + call %%X + if errorlevel 1 ( + echo Failed to execute: %%X + exit /b 1 + ) +) \ No newline at end of file diff --git a/conda/dev-environment-unix.yml b/conda/dev-environment-unix.yml index 8f373d6a1..8211bfc0e 100644 --- a/conda/dev-environment-unix.yml +++ b/conda/dev-environment-unix.yml @@ -1,4 +1,4 @@ -name: csp-dev +name: csp channels: - conda-forge - nodefaults @@ -11,6 +11,7 @@ dependencies: - codespell>=2.2.6,<2.3 - compilers - cyrus-sasl + - deprecated - exprtk - flex - graphviz @@ -19,21 +20,20 @@ dependencies: - httpx>=0.20,<1 - isort>=5,<6 - libarrow=16 + - libboost>=1.80.0 + - libboost-headers>=1.80.0 - librdkafka - - libboost-headers - lz4-c - mamba - mdformat>=0.7.17,<0.8 - ninja - - numpy - - pillow - - psutil - - pyarrow=16 + - numpy<2 - pandas - pillow - polars - psutil - pydantic>2 + - pyarrow=16 - pytz - pytest - pytest-asyncio @@ -46,7 +46,6 @@ dependencies: - ruamel.yaml - ruff>=0.3,<0.4 - scikit-build - - slack-sdk - sqlalchemy - tar - threadpoolctl @@ -54,5 +53,4 @@ dependencies: - twine - unzip - wheel - - websocketpp - zip diff --git a/conda/dev-environment-win.yml b/conda/dev-environment-win.yml new file mode 100644 index 000000000..8c1ce5fdb --- /dev/null +++ b/conda/dev-environment-win.yml @@ -0,0 +1,51 @@ +name: csp +channels: + - conda-forge + - nodefaults +dependencies: + - brotli + - build + - bump2version>=1 + - cmake + - codespell>=2.2.6,<2.3 + - compilers + - cyrus-sasl + - deprecated + - exprtk + - graphviz + - gtest + - httpx>=0.20,<1 + - isort>=5,<6 + - libarrow=16 + - libboost>=1.80.0 + - libboost-headers>=1.80.0 + - librdkafka + - lz4-c + - make + - mamba + - mdformat>=0.7.17,<0.8 + - ninja + - numpy<2 + - pandas + - pillow + - polars + - psutil + - pyarrow=16 + - pytest + - pytest-asyncio + - pytest-cov + - pytest-sugar + - python<3.13 + - python-graphviz + - python-rapidjson + - pytz + - rapidjson + - requests + - ruamel.yaml + - ruff>=0.3,<0.4 + - scikit-build + - sqlalchemy + - threadpoolctl + - tornado + - twine + - wheel diff --git a/cpp/cmake/modules/FindBrotli.cmake b/cpp/cmake/modules/FindBrotli.cmake index 6f068a74d..3df448e0f 100644 --- a/cpp/cmake/modules/FindBrotli.cmake +++ b/cpp/cmake/modules/FindBrotli.cmake @@ -43,62 +43,62 @@ if(VCPKG_INSTALLED_DIR) endif() find_path( BROTLI_INCLUDE_DIR - NAMES brotli/decode.h - PATHS ${_brotli_roots} - PATH_SUFFIXES "include" ) - message("Include: ${BROTLI_INCLUDE_DIR}") + NAMES brotli/decode.h + PATHS ${_brotli_roots} + PATH_SUFFIXES "include" ) + message("Include: ${BROTLI_INCLUDE_DIR}") find_library( BROTLI_STATIC_LIB_ENC - NAMES libbrotlienc.a libbrotlienc-static.a brotlienc - PATHS ${_brotli_roots} - PATH_SUFFIXES "" "lib" ) + NAMES libbrotlienc.a libbrotlienc-static.a brotlienc + PATHS ${_brotli_roots} + PATH_SUFFIXES "" "lib" ) find_library( BROTLI_STATIC_LIB_DEC - NAMES libbrotlidec.a libbrotlidec-static.a brotlidec - PATHS ${_brotli_roots} - PATH_SUFFIXES "" "lib" ) + NAMES libbrotlidec.a libbrotlidec-static.a brotlidec + PATHS ${_brotli_roots} + PATH_SUFFIXES "" "lib" ) find_library( BROTLI_STATIC_LIB_COMMON - NAMES libbrotlicommon.a libbrotlicommon-static.a brotlicommon - PATHS ${_brotli_roots} - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" ) + NAMES libbrotlicommon.a libbrotlicommon-static.a brotlicommon + PATHS ${_brotli_roots} + PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" ) set(BROTLI_LIBRARIES ${BROTLI_STATIC_LIB_ENC} ${BROTLI_STATIC_LIB_DEC} ${BROTLI_STATIC_LIB_COMMON}) if (BROTLI_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR BROTLI_LIBRARIES)) - set(BROTLI_FOUND TRUE) - set(BROTLI_STATIC_LIB ${BROTLI_STATIC_LIB_ENC} ${BROTLI_STATIC_LIB_DEC} ${BROTLI_STATIC_LIB_COMMON}) + set(BROTLI_FOUND TRUE) + set(BROTLI_STATIC_LIB ${BROTLI_STATIC_LIB_ENC} ${BROTLI_STATIC_LIB_DEC} ${BROTLI_STATIC_LIB_COMMON}) else () - set(BROTLI_FOUND FALSE) + set(BROTLI_FOUND FALSE) endif () if (BROTLI_FOUND) - if (NOT Brotli_FIND_QUIETLY) + if (NOT Brotli_FIND_QUIETLY) if (PARQUET_MINIMAL_DEPENDENCY) - message(STATUS "Found the Brotli headers: ${BROTLI_INCLUDE_DIR}") + message(STATUS "Found the Brotli headers: ${BROTLI_INCLUDE_DIR}") else () - message(STATUS "Found the Brotli library: ${BROTLI_LIBRARIES}") + message(STATUS "Found the Brotli library: ${BROTLI_LIBRARIES}") + endif () endif () - endif () else () - if (NOT Brotli_FIND_QUIETLY) + if (NOT Brotli_FIND_QUIETLY) set(BROTLI_ERR_MSG "Could not find the Brotli library. Looked in ") if ( _brotli_roots ) - set(BROTLI_ERR_MSG "${BROTLI_ERR_MSG} in ${_brotli_roots}.") + set(BROTLI_ERR_MSG "${BROTLI_ERR_MSG} in ${_brotli_roots}.") else () - set(BROTLI_ERR_MSG "${BROTLI_ERR_MSG} system search paths.") + set(BROTLI_ERR_MSG "${BROTLI_ERR_MSG} system search paths.") endif () if (Brotli_FIND_REQUIRED) - message(FATAL_ERROR "${BROTLI_ERR_MSG}") + message(FATAL_ERROR "${BROTLI_ERR_MSG}") else (Brotli_FIND_REQUIRED) - message(STATUS "${BROTLI_ERR_MSG}") + message(STATUS "${BROTLI_ERR_MSG}") endif (Brotli_FIND_REQUIRED) - endif () + endif () endif () mark_as_advanced( - BROTLI_INCLUDE_DIR - BROTLI_LIBS - BROTLI_LIBRARIES - BROTLI_STATIC_LIB + BROTLI_INCLUDE_DIR + BROTLI_LIBS + BROTLI_LIBRARIES + BROTLI_STATIC_LIB ) diff --git a/cpp/cmake/modules/FindPyArrow.cmake b/cpp/cmake/modules/FindPyArrow.cmake index 22cd36aaa..813ce6ecb 100644 --- a/cpp/cmake/modules/FindPyArrow.cmake +++ b/cpp/cmake/modules/FindPyArrow.cmake @@ -13,27 +13,25 @@ find_package(Python ${CSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter) execute_process( COMMAND "${Python_EXECUTABLE}" -c "from __future__ import print_function;import pyarrow;print(pyarrow.get_include(), end='')" - OUTPUT_VARIABLE __pyarrow_include_path) + OUTPUT_VARIABLE PYARROW_INCLUDE_DIR) # Find out the lib path execute_process( COMMAND "${Python_EXECUTABLE}" -c "from __future__ import print_function;import pyarrow;print(pyarrow.get_library_dirs()[0], end='')" - OUTPUT_VARIABLE __pyarrow_libs_path) + OUTPUT_VARIABLE PYARROW_LIB_DIR) # Find out the version path execute_process( COMMAND "${Python_EXECUTABLE}" -c "from __future__ import print_function;import pyarrow;print(pyarrow.__version__, end='')" OUTPUT_VARIABLE __pyarrow_version) -find_path(PYARROW_INCLUDE_DIR arrow/python/pyarrow.h - HINTS "${__pyarrow_include_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH) - -find_path(PYARROW_LIB_DIR libarrow_python.so - HINTS "${__pyarrow_libs_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH) - if(PYARROW_INCLUDE_DIR AND PYARROW_LIB_DIR) set(PYARROW_FOUND 1 CACHE INTERNAL "Python pyarrow found") - set(PYARROW_LIBRARY libarrow_python.so) + if(NOT WIN32) + set(PYARROW_LIBRARY libarrow_python.so) + else() + set(PYARROW_LIBRARY ${PYARROW_LIB_DIR}/arrow_python.lib) + endif() endif() include(FindPackageHandleStandardArgs) diff --git a/cpp/cmake/modules/Findcsp_autogen.cmake b/cpp/cmake/modules/Findcsp_autogen.cmake index 98b3604d3..1c28a99f6 100644 --- a/cpp/cmake/modules/Findcsp_autogen.cmake +++ b/cpp/cmake/modules/Findcsp_autogen.cmake @@ -13,11 +13,26 @@ function(csp_autogen MODULE_NAME DEST_FILENAME HEADER_NAME_OUTVAR SOURCE_NAME_OU set(CSP_AUTOGEN_EXTRA_ARGS "") endif() - add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.cpp" "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.h" - COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${PROJECT_BINARY_DIR}/lib:${CMAKE_SOURCE_DIR}:$$PYTHONPATH" "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/lib:$$LD_LIBRARY_PATH}" ${Python_EXECUTABLE} ${CSP_AUTOGEN} -m ${MODULE_NAME} -d ${CMAKE_CURRENT_BINARY_DIR}/csp_autogen -o ${DEST_FILENAME} ${CSP_AUTOGEN_EXTRA_ARGS} - COMMENT "generating csp c++ types from module ${MODULE_NAME} ${PROJECT_BINARY_DIR} ${CSP_AUTOGEN}" - DEPENDS mkdir_autogen_${MODULE_NAME} ${CMAKE_SOURCE_DIR}/${MODULE_FILENAME}) + cmake_path(SET CSP_AUTOGEN_MODULE_PATH NORMALIZE "${CMAKE_SOURCE_DIR}/csp/build/csp_autogen.py") + cmake_path(SET CSP_AUTOGEN_DESTINATION_FOLDER NORMALIZE "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen") + cmake_path(SET CSP_AUTOTGEN_CPP_OUT NORMALIZE "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.cpp") + cmake_path(SET CSP_AUTOTGEN_H_OUT NORMALIZE "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.h") + + if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + set(CSP_AUTOGEN_PYTHONPATH ${PROJECT_BINARY_DIR}/lib/${CMAKE_BUILD_TYPE};${CMAKE_SOURCE_DIR};%PYTHONPATH% ) + else() + set(CSP_AUTOGEN_PYTHONPATH ${PROJECT_BINARY_DIR}/lib:${CMAKE_SOURCE_DIR}:$$PYTHONPATH ) + endif() + + add_custom_command(OUTPUT "${CSP_AUTOTGEN_CPP_OUT}" "${CSP_AUTOTGEN_H_OUT}" + COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CSP_AUTOGEN_PYTHONPATH}" ${Python_EXECUTABLE} ${CSP_AUTOGEN_MODULE_PATH} -m ${MODULE_NAME} -d ${CSP_AUTOGEN_DESTINATION_FOLDER} -o ${DEST_FILENAME} ${CSP_AUTOGEN_EXTRA_ARGS} + COMMENT "generating csp c++ types from module ${MODULE_NAME}" + DEPENDS mkdir_autogen_${MODULE_NAME} + ${CMAKE_SOURCE_DIR}/csp/build/csp_autogen.py + ${CMAKE_SOURCE_DIR}/${MODULE_FILENAME} + csptypesimpl + ) set(${SOURCE_NAME_OUTVAR} "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.cpp" PARENT_SCOPE ) set(${HEADER_NAME_OUTVAR} "${CMAKE_CURRENT_BINARY_DIR}/csp_autogen/${DEST_FILENAME}.h" PARENT_SCOPE ) -endfunction() +endfunction() \ No newline at end of file diff --git a/cpp/csp/adapters/kafka/CMakeLists.txt b/cpp/csp/adapters/kafka/CMakeLists.txt index 4984d87bf..1bf0ae402 100644 --- a/cpp/csp/adapters/kafka/CMakeLists.txt +++ b/cpp/csp/adapters/kafka/CMakeLists.txt @@ -26,6 +26,6 @@ target_link_libraries(csp_kafka_adapter PRIVATE csp_adapter_utils RdKafka::rdkaf install(TARGETS csp_kafka_adapter PUBLIC_HEADER DESTINATION include/csp/adapters/kafka - RUNTIME DESTINATION bin/ + RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR} LIBRARY DESTINATION lib/ ) diff --git a/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp b/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp index 5d6d78285..06d5fce9a 100644 --- a/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp +++ b/cpp/csp/adapters/kafka/KafkaAdapterManager.cpp @@ -75,7 +75,7 @@ KafkaAdapterManager::KafkaAdapterManager( csp::Engine * engine, const Dictionary m_consumerIdx( 0 ), m_producerPollThreadActive( false ) { - m_maxThreads = properties.get( "max_threads" ); + m_maxThreads = properties.get( "max_threads" ); m_pollTimeoutMs = properties.get( "poll_timeout" ).asMilliseconds(); m_eventCb = std::make_unique( this ); diff --git a/cpp/csp/adapters/kafka/KafkaAdapterManager.h b/cpp/csp/adapters/kafka/KafkaAdapterManager.h index 234f5fc11..bf4e23158 100644 --- a/cpp/csp/adapters/kafka/KafkaAdapterManager.h +++ b/cpp/csp/adapters/kafka/KafkaAdapterManager.h @@ -47,7 +47,7 @@ struct KafkaStatusMessageTypeTraits using KafkaStatusMessageType = csp::Enum; //Top level AdapterManager object for all kafka adapters in the engine -class KafkaAdapterManager final : public csp::AdapterManager +class CSP_PUBLIC KafkaAdapterManager final : public csp::AdapterManager { public: KafkaAdapterManager( csp::Engine * engine, const Dictionary & properties ); diff --git a/cpp/csp/adapters/parquet/CMakeLists.txt b/cpp/csp/adapters/parquet/CMakeLists.txt index d976a5538..0d029c397 100644 --- a/cpp/csp/adapters/parquet/CMakeLists.txt +++ b/cpp/csp/adapters/parquet/CMakeLists.txt @@ -50,18 +50,33 @@ find_package(lz4 REQUIRED) find_package(utf8proc REQUIRED) find_package(Brotli REQUIRED) -if(CSP_USE_VCPKG) - # use staic variants - set(ARROW_PACKAGES_TO_LINK parquet_static arrow_static) +if(WIN32) + if(CSP_USE_VCPKG) + set(ARROW_PACKAGES_TO_LINK Arrow::arrow_static Parquet::parquet_static ) + target_compile_definitions(csp_parquet_adapter PUBLIC ARROW_STATIC) + target_compile_definitions(csp_parquet_adapter PUBLIC PARQUET_STATIC) + else() + # use dynamic variants + # Until we manage to get the fix for ws3_32.dll in arrow-16 into conda, manually fix the error here + get_target_property(LINK_LIBS Arrow::arrow_shared INTERFACE_LINK_LIBRARIES) + string(REPLACE "ws2_32.dll" "ws2_32" FIXED_LINK_LIBS "${LINK_LIBS}") + set_target_properties(Arrow::arrow_shared PROPERTIES INTERFACE_LINK_LIBRARIES "${FIXED_LINK_LIBS}") + set(ARROW_PACKAGES_TO_LINK parquet_shared arrow_shared) + endif() else() - # use dynamic variants - set(ARROW_PACKAGES_TO_LINK parquet arrow) + if(CSP_USE_VCPKG) + # use static variants + set(ARROW_PACKAGES_TO_LINK parquet_static arrow_static) + else() + # use dynamic variants + set(ARROW_PACKAGES_TO_LINK parquet arrow) + endif() endif() target_link_libraries(csp_parquet_adapter PRIVATE csp_adapter_utils thrift::thrift lz4::lz4 utf8proc::utf8proc ${BROTLI_STATIC_LIB} ${ARROW_PACKAGES_TO_LINK}) install(TARGETS csp_parquet_adapter PUBLIC_HEADER DESTINATION include/csp/adapters/parquet - RUNTIME DESTINATION bin/ + RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR} LIBRARY DESTINATION lib/ ) diff --git a/cpp/csp/adapters/parquet/DialectGenericListReaderInterface.h b/cpp/csp/adapters/parquet/DialectGenericListReaderInterface.h index c88097f6f..61b77cd85 100644 --- a/cpp/csp/adapters/parquet/DialectGenericListReaderInterface.h +++ b/cpp/csp/adapters/parquet/DialectGenericListReaderInterface.h @@ -25,7 +25,7 @@ class DialectGenericListReaderInterface }; template< typename T > -class TypedDialectGenericListReaderInterface : public DialectGenericListReaderInterface +class CSP_PUBLIC TypedDialectGenericListReaderInterface : public DialectGenericListReaderInterface { public: using Ptr = std::shared_ptr>; @@ -45,4 +45,4 @@ class TypedDialectGenericListReaderInterface : public DialectGenericListReaderIn } -#endif \ No newline at end of file +#endif diff --git a/cpp/csp/adapters/parquet/ParquetInputAdapterManager.h b/cpp/csp/adapters/parquet/ParquetInputAdapterManager.h index baa67e19c..8f3effa1e 100644 --- a/cpp/csp/adapters/parquet/ParquetInputAdapterManager.h +++ b/cpp/csp/adapters/parquet/ParquetInputAdapterManager.h @@ -18,7 +18,7 @@ namespace csp::adapters::parquet //Top level AdapterManager object for all parquet adapters in the engine -class ParquetInputAdapterManager final : public csp::AdapterManager +class CSP_PUBLIC ParquetInputAdapterManager final : public csp::AdapterManager { public: using GeneratorPtr = csp::Generator::Ptr; diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp index 0061f300e..f73092254 100644 --- a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp +++ b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp @@ -140,9 +140,10 @@ template< typename A, typename V = typename A::value_type > inline std::shared_ptr<::arrow::ArrayBuilder> makeArrayAndAttachToWriter( DialectGenericListWriterInterface::Ptr &listWriterInterface ) { auto&& typedWriter = std::dynamic_pointer_cast>( listWriterInterface ); + auto& listWriterInterfaceRef = *listWriterInterface; CSP_TRUE_OR_THROW( typedWriter != nullptr, TypeError, "Expected " << typeid( TypedDialectGenericListWriterInterface ).name() << " " << " got " << - typeid( *listWriterInterface ).name() ); + typeid( listWriterInterfaceRef ).name() ); auto res = std::make_shared(); typedWriter -> setWriteFunction( diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h index a2b5da200..b7fe029f4 100644 --- a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h +++ b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h @@ -21,7 +21,7 @@ class ParquetOutputFilenameAdapter; class ParquetDictBasketOutputWriter; //Top level AdapterManager object for all parquet adapters in the engine -class ParquetOutputAdapterManager final : public csp::AdapterManager +class CSP_PUBLIC ParquetOutputAdapterManager final : public csp::AdapterManager { public: using FileVisitorCallback = std::function; diff --git a/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp b/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp index 978b96b32..0525ffbf9 100644 --- a/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp +++ b/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp @@ -22,7 +22,7 @@ static inline std::unique_ptr createDateColumnAdapter( new DateColumnAdapter<1000000, ArrowArrayType>( reader, columnName ) ); case arrow::DateUnit::DAY: return std::unique_ptr( - new DateColumnAdapter<1000000000L * 3600 * 24, ArrowArrayType>( reader, columnName ) ); + new DateColumnAdapter<1000000000LL * 3600 * 24, ArrowArrayType>( reader, columnName ) ); } CSP_THROW( csp::TypeError, "Unexpected day unit: " << ( int ) dateType -> unit() << " for column " << columnName ); } @@ -524,7 +524,7 @@ void NativeTypeColumnAdapter::readCurValue() } } -template< long UNIT > +template< int64_t UNIT > void DatetimeColumnAdapter::readCurValue() { auto curRow = this -> m_parquetReader.getCurRow(); @@ -539,7 +539,7 @@ void DatetimeColumnAdapter::readCurValue() } } -template< long UNIT > +template< int64_t UNIT > void DurationColumnAdapter::readCurValue() { auto curRow = this -> m_parquetReader.getCurRow(); @@ -554,7 +554,7 @@ void DurationColumnAdapter::readCurValue() } } -template< long UNIT, typename ArrowDateArray > +template< int64_t UNIT, typename ArrowDateArray > void DateColumnAdapter::readCurValue() { auto curRow = this -> m_parquetReader.getCurRow(); @@ -569,7 +569,7 @@ void DateColumnAdapter::readCurValue() } } -template< long UNIT, typename ArrowTimeArray > +template< int64_t UNIT, typename ArrowTimeArray > void TimeColumnAdapter::readCurValue() { auto curRow = this -> m_parquetReader.getCurRow(); @@ -734,7 +734,7 @@ void ListColumnAdapter::readCurValue() if( this -> m_curChunkArray -> IsValid( curRow ) ) { auto values = this -> m_curChunkArray -> value_slice( curRow ); - auto typedValues = std::dynamic_pointer_cast( values ); + auto typedValues = std::static_pointer_cast( values ); auto arrayValue = m_listReader -> create( typedValues -> length() ); auto* internalBuffer = m_listReader -> getRawDataBuffer( arrayValue ); diff --git a/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.h b/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.h index eb7c4a839..5d8ad8580 100644 --- a/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.h +++ b/cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.h @@ -197,7 +197,7 @@ class NativeTypeColumnAdapter : public BaseTypedColumnAdapter +template< int64_t UNIT > class DatetimeColumnAdapter : public BaseTypedColumnAdapter { public: @@ -207,7 +207,7 @@ class DatetimeColumnAdapter : public BaseTypedColumnAdapter +template< int64_t UNIT > class DurationColumnAdapter : public BaseTypedColumnAdapter { public: @@ -217,7 +217,7 @@ class DurationColumnAdapter : public BaseTypedColumnAdapter +template< int64_t UNIT, typename ArrowDateArray > class DateColumnAdapter : public BaseTypedColumnAdapter { public: @@ -227,7 +227,7 @@ class DateColumnAdapter : public BaseTypedColumnAdapter +template< int64_t UNIT, typename ArrowTimeArray > class TimeColumnAdapter : public BaseTypedColumnAdapter { public: diff --git a/cpp/csp/adapters/utils/CMakeLists.txt b/cpp/csp/adapters/utils/CMakeLists.txt index 08d6d81be..217b011d4 100644 --- a/cpp/csp/adapters/utils/CMakeLists.txt +++ b/cpp/csp/adapters/utils/CMakeLists.txt @@ -29,6 +29,6 @@ target_link_libraries(csp_adapter_utils PRIVATE protobuf::libprotoc protobuf::li install(TARGETS csp_adapter_utils PUBLIC_HEADER DESTINATION include/csp/adapters/utils - RUNTIME DESTINATION bin/ + RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR} LIBRARY DESTINATION lib/ ) diff --git a/cpp/csp/adapters/utils/ProtobufHelper.cpp b/cpp/csp/adapters/utils/ProtobufHelper.cpp index 0eba233a7..f1e1600b9 100644 --- a/cpp/csp/adapters/utils/ProtobufHelper.cpp +++ b/cpp/csp/adapters/utils/ProtobufHelper.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include diff --git a/cpp/csp/adapters/websocket/CMakeLists.txt b/cpp/csp/adapters/websocket/CMakeLists.txt index 879a156c4..402d01b69 100644 --- a/cpp/csp/adapters/websocket/CMakeLists.txt +++ b/cpp/csp/adapters/websocket/CMakeLists.txt @@ -1,8 +1,5 @@ csp_autogen( csp.adapters.websocket_types websocket_types WEBSOCKET_HEADER WEBSOCKET_SOURCE ) -# Need to build websocket adapter under cpp17 standard due to websocketpp incompatibility issues -set(CMAKE_CXX_STANDARD 17) - set(WS_CLIENT_HEADER_FILES ClientAdapterManager.h ClientInputAdapter.h @@ -25,20 +22,18 @@ set(WS_CLIENT_SOURCE_FILES add_library(csp_websocket_client_adapter STATIC ${WS_CLIENT_SOURCE_FILES}) set_target_properties(csp_websocket_client_adapter PROPERTIES PUBLIC_HEADER "${WS_CLIENT_SOURCE_FILES}") -find_package(websocketpp REQUIRED) -#set(OPENSSL_USE_STATIC_LIBS TRUE) +find_package(Boost REQUIRED) find_package(OpenSSL REQUIRED) target_link_libraries(csp_websocket_client_adapter PRIVATE csp_adapter_utils - ${OPENSSL_SSL_LIBRARY} - ${OPENSSL_CRYPTO_LIBRARY} - websocketpp::websocketpp + Boost::boost + OpenSSL::SSL ) install(TARGETS csp_websocket_client_adapter PUBLIC_HEADER DESTINATION include/csp/adapters/websocket - RUNTIME DESTINATION bin/ + RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR} LIBRARY DESTINATION lib/ ) diff --git a/cpp/csp/adapters/websocket/ClientAdapterManager.cpp b/cpp/csp/adapters/websocket/ClientAdapterManager.cpp index 5bdf5b18e..423f2a234 100644 --- a/cpp/csp/adapters/websocket/ClientAdapterManager.cpp +++ b/cpp/csp/adapters/websocket/ClientAdapterManager.cpp @@ -1,10 +1,5 @@ #include -#include -#include -#include -#include - namespace csp { INIT_CSP_ENUM( adapters::websocket::ClientStatusType, @@ -24,57 +19,62 @@ ClientAdapterManager::ClientAdapterManager( Engine* engine, const Dictionary & p : AdapterManager( engine ), m_active( false ), m_shouldRun( false ), - m_endpoint( nullptr ), + m_endpoint( std::make_unique( properties ) ), m_inputAdapter( nullptr ), m_outputAdapter( nullptr ), m_updateAdapter( nullptr ), m_thread( nullptr ), m_properties( properties ) -{ - if( m_properties.get( "use_tls" ) ) - { - m_endpoint = new WebsocketEndpointTLS( properties ); - } - else - { - m_endpoint = new WebsocketEndpointNoTLS( properties ); - } - - -}; +{ }; ClientAdapterManager::~ClientAdapterManager() { }; void ClientAdapterManager::start( DateTime starttime, DateTime endtime ) { - if(m_inputAdapter != nullptr) - { - m_endpoint -> setOnMessageCb( [ this ]( std::string msg ) { - PushBatch batch( m_engine -> rootEngine() ); - m_inputAdapter -> processMessage( msg, &batch ); - }); - } - m_endpoint -> setOnOpenCb( [ this ]() { - m_active = true; - pushStatus( StatusLevel::INFO, ClientStatusType::ACTIVE, "Connected successfully" ); - }); - m_endpoint -> setOnFailCb( [ this ]() { - m_active = false; - pushStatus( StatusLevel::ERROR, ClientStatusType::CONNECTION_FAILED, "Connection failed, will try to reconnect" ); - }); - m_endpoint -> setOnCloseCb( [ this ]() { - m_active = false; - pushStatus( StatusLevel::INFO, ClientStatusType::CLOSED, "Connection closed" ); - }); - m_endpoint -> setOnSendFailCb( [ this ]( const std::string& s ) { - std::stringstream ss; - ss << "Failed to send: " << s; - pushStatus( StatusLevel::ERROR, ClientStatusType::MESSAGE_SEND_FAIL, ss.str() ); - }); AdapterManager::start( starttime, endtime ); - // start the bg thread + m_shouldRun = true; + m_endpoint -> setOnOpen( + [ this ]() { + m_active = true; + pushStatus( StatusLevel::INFO, ClientStatusType::ACTIVE, "Connected successfully" ); + } + ); + m_endpoint -> setOnFail( + [ this ]( const std::string& reason ) { + std::stringstream ss; + ss << "Connection Failure: " << reason; + m_active = false; + pushStatus( StatusLevel::ERROR, ClientStatusType::CONNECTION_FAILED, ss.str() ); + } + ); + if( m_inputAdapter ) { + m_endpoint -> setOnMessage( + [ this ]( void* c, size_t t ) { + PushBatch batch( m_engine -> rootEngine() ); + m_inputAdapter -> processMessage( c, t, &batch ); + } + ); + } else { + // if a user doesn't call WebsocketAdapterManager.subscribe, no inputadapter will be created + // but we still need something to avoid on_message_cb not being set in the endpoint. + m_endpoint -> setOnMessage( []( void* c, size_t t ){} ); + } + m_endpoint -> setOnClose( + [ this ]() { + m_active = false; + pushStatus( StatusLevel::INFO, ClientStatusType::CLOSED, "Connection closed" ); + } + ); + m_endpoint -> setOnSendFail( + [ this ]( const std::string& s ) { + std::stringstream ss; + ss << "Failed to send: " << s; + pushStatus( StatusLevel::ERROR, ClientStatusType::MESSAGE_SEND_FAIL, ss.str() ); + } + ); + m_thread = std::make_unique( [ this ]() { while( m_shouldRun ) { @@ -89,7 +89,7 @@ void ClientAdapterManager::stop() { AdapterManager::stop(); m_shouldRun=false; - if( m_active ) m_endpoint->close(); + if( m_active ) m_endpoint->stop(); if( m_thread ) m_thread->join(); }; @@ -109,7 +109,7 @@ PushInputAdapter* ClientAdapterManager::getInputAdapter(CspTypePtr & type, PushM OutputAdapter* ClientAdapterManager::getOutputAdapter() { - if (m_outputAdapter == nullptr) m_outputAdapter = m_engine -> createOwnedObject(m_endpoint); + if (m_outputAdapter == nullptr) m_outputAdapter = m_engine -> createOwnedObject(*m_endpoint); return m_outputAdapter; } diff --git a/cpp/csp/adapters/websocket/ClientAdapterManager.h b/cpp/csp/adapters/websocket/ClientAdapterManager.h index ec80a356c..b2b15fa78 100644 --- a/cpp/csp/adapters/websocket/ClientAdapterManager.h +++ b/cpp/csp/adapters/websocket/ClientAdapterManager.h @@ -1,17 +1,21 @@ #ifndef _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_ADAPTERMGR_H #define _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_ADAPTERMGR_H +#include +#include +#include +#include #include #include #include #include #include +#include #include +#include +#include +#include -#include -#include -#include -#include namespace csp::adapters::websocket { @@ -36,10 +40,8 @@ struct WebsocketClientStatusTypeTraits using ClientStatusType = Enum; -class ClientAdapterManager final : public AdapterManager +class CSP_PUBLIC ClientAdapterManager final : public AdapterManager { - - public: ClientAdapterManager( Engine * engine, @@ -64,7 +66,7 @@ class ClientAdapterManager final : public AdapterManager bool m_active; bool m_shouldRun; - WebsocketEndpointBase* m_endpoint; + std::unique_ptr m_endpoint; ClientInputAdapter* m_inputAdapter; ClientOutputAdapter* m_outputAdapter; ClientHeaderUpdateOutputAdapter* m_updateAdapter; @@ -74,4 +76,4 @@ class ClientAdapterManager final : public AdapterManager } -#endif \ No newline at end of file +#endif diff --git a/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.h b/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.h index 96459aaa3..d2c898a1e 100644 --- a/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.h +++ b/cpp/csp/adapters/websocket/ClientHeaderUpdateAdapter.h @@ -1,8 +1,6 @@ #ifndef _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_HEADERUPDATEADAPTER_H #define _IN_CSP_ADAPTERS_WEBSOCKETS_CLIENT_HEADERUPDATEADAPTER_H -#include -#include #include #include #include diff --git a/cpp/csp/adapters/websocket/ClientInputAdapter.cpp b/cpp/csp/adapters/websocket/ClientInputAdapter.cpp index ad1db9afc..548b0bc04 100644 --- a/cpp/csp/adapters/websocket/ClientInputAdapter.cpp +++ b/cpp/csp/adapters/websocket/ClientInputAdapter.cpp @@ -25,16 +25,16 @@ ClientInputAdapter::ClientInputAdapter( m_converter = adapters::utils::MessageStructConverterCache::instance().create( type, properties ); }; -void ClientInputAdapter::processMessage( std::string payload, PushBatch* batch ) +void ClientInputAdapter::processMessage( void* c, size_t t, PushBatch* batch ) { if( type() -> type() == CspType::Type::STRUCT ) { - auto tick = m_converter -> asStruct( (void*)payload.data(), payload.length() ); + auto tick = m_converter -> asStruct( c, t ); pushTick( std::move(tick), batch ); } else if ( type() -> type() == CspType::Type::STRING ) { - pushTick( std::move(payload), batch ); + pushTick( std::string((char const*)c, t), batch ); } } diff --git a/cpp/csp/adapters/websocket/ClientInputAdapter.h b/cpp/csp/adapters/websocket/ClientInputAdapter.h index 93ae2614b..bf3cb295f 100644 --- a/cpp/csp/adapters/websocket/ClientInputAdapter.h +++ b/cpp/csp/adapters/websocket/ClientInputAdapter.h @@ -19,7 +19,7 @@ class ClientInputAdapter final: public PushInputAdapter { const Dictionary & properties ); - void processMessage( std::string payload, PushBatch* batch ); + void processMessage( void* c, size_t t, PushBatch* batch ); private: adapters::utils::MessageStructConverterPtr m_converter; diff --git a/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp b/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp index 48d2954dc..3ef3c91ac 100644 --- a/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp +++ b/cpp/csp/adapters/websocket/ClientOutputAdapter.cpp @@ -4,14 +4,14 @@ namespace csp::adapters::websocket { ClientOutputAdapter::ClientOutputAdapter( Engine * engine, - WebsocketEndpointBase * endpoint + WebsocketEndpoint& endpoint ) : OutputAdapter( engine ), m_endpoint( endpoint ) { }; void ClientOutputAdapter::executeImpl() { const std::string & value = input() -> lastValueTyped(); - m_endpoint->send( value ); + m_endpoint.send( value ); }; } \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/ClientOutputAdapter.h b/cpp/csp/adapters/websocket/ClientOutputAdapter.h index 865831c8f..905822e2f 100644 --- a/cpp/csp/adapters/websocket/ClientOutputAdapter.h +++ b/cpp/csp/adapters/websocket/ClientOutputAdapter.h @@ -17,7 +17,7 @@ class ClientOutputAdapter final: public OutputAdapter public: ClientOutputAdapter( Engine * engine, - WebsocketEndpointBase* endpoint + WebsocketEndpoint& endpoint ); void executeImpl() override; @@ -25,7 +25,7 @@ class ClientOutputAdapter final: public OutputAdapter const char * name() const override { return "WebsocketClientOutputAdapter"; } private: - WebsocketEndpointBase* m_endpoint; + WebsocketEndpoint& m_endpoint; }; } diff --git a/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp b/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp index 46cb61468..8e856c79e 100644 --- a/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp +++ b/cpp/csp/adapters/websocket/WebsocketEndpoint.cpp @@ -1,189 +1,71 @@ #include -using websocketpp::lib::placeholders::_1; -using websocketpp::lib::placeholders::_2; -using websocketpp::lib::bind; - namespace csp::adapters::websocket { using namespace csp; -/* -WebsocketEndpointBase -> base setup -*/ -WebsocketEndpointBase::WebsocketEndpointBase( csp::Dictionary properties ) -: m_properties( properties ) +WebsocketEndpoint::WebsocketEndpoint( + Dictionary properties +) : m_properties(properties) { }; - -csp::Dictionary& WebsocketEndpointBase::getProperties() { - return m_properties; -} - -void WebsocketEndpointBase::setOnMessageCb(on_message_cb cb) -{ m_on_message=cb; } - -void WebsocketEndpointBase::setOnOpenCb(void_cb cb) -{ m_on_open=cb; } - -void WebsocketEndpointBase::setOnFailCb(void_cb cb) -{ m_on_fail=cb; } - -void WebsocketEndpointBase::setOnCloseCb(void_cb cb) -{ m_on_close=cb; } - -void WebsocketEndpointBase::setOnSendFailCb(on_send_fail_cb cb) -{ m_on_send_fail=cb; } - -/* -WebsocketEndpointTLS -> tls impl -*/ -WebsocketEndpointTLS::WebsocketEndpointTLS(csp::Dictionary properties) -: WebsocketEndpointBase( std::move(properties) ) +void WebsocketEndpoint::setOnOpen(void_cb on_open) +{ m_on_open = std::move(on_open); } +void WebsocketEndpoint::setOnFail(string_cb on_fail) +{ m_on_fail = std::move(on_fail); } +void WebsocketEndpoint::setOnMessage(char_cb on_message) +{ m_on_message = std::move(on_message); } +void WebsocketEndpoint::setOnClose(void_cb on_close) +{ m_on_close = std::move(on_close); } +void WebsocketEndpoint::setOnSendFail(string_cb on_send_fail) +{ m_on_send_fail = std::move(on_send_fail); } + +void WebsocketEndpoint::run() { - if( m_properties.get("verbose_log") ) { - m_client.set_access_channels(websocketpp::log::alevel::all); - } else { - m_client.clear_access_channels(websocketpp::log::alevel::all); - } - m_client.init_asio(); - // need to set these with callbacks - m_client.set_open_handler( [ this ]( websocketpp::connection_hdl ){ - m_on_open(); - }); - m_client.set_message_handler( [ this ]( websocketpp::connection_hdl, message_ptr msg ) { - m_on_message( msg -> get_payload() ); - }); - m_client.set_fail_handler( [ this ]( websocketpp::connection_hdl ){ - m_on_fail(); - }); - m_client.set_close_handler( [ this ]( websocketpp::connection_hdl ){ - m_on_close(); - }); - m_client.set_tls_init_handler( []( websocketpp::connection_hdl ){ - auto ctx = websocketpp::lib::make_shared(boost::asio::ssl::context::tlsv12); - boost::system::error_code ec; - ctx->set_options( - boost::asio::ssl::context::default_workarounds | - boost::asio::ssl::context::no_sslv2 | - boost::asio::ssl::context::single_dh_use, ec + m_ioc.reset(); + if(m_properties.get("use_ssl")) { + ssl::context ctx{ssl::context::sslv23}; + ctx.set_verify_mode(ssl::context::verify_peer ); + ctx.set_default_verify_paths(); + + m_session = new WebsocketSessionTLS( + m_ioc, + ctx, + &m_properties, + m_on_open, + m_on_fail, + m_on_message, + m_on_close, + m_on_send_fail ); - if( ec ) { - CSP_THROW( csp::RuntimeException, "Init tls failed: "<< ec ); - } - return ctx; - }); -} -WebsocketEndpointTLS::~WebsocketEndpointTLS() -{ } - -void WebsocketEndpointTLS::send( const std::string& s ) -{ - websocketpp::lib::error_code ec; - m_client.send( m_hdl, s, websocketpp::frame::opcode::value::TEXT, ec ); - if( ec ) m_on_send_fail(s); -} - -void WebsocketEndpointTLS::run() -{ - auto uri = m_properties.get("uri"); - websocketpp::lib::error_code ec; - tls_client::connection_ptr con = m_client.get_connection( uri, ec ); - if( ec ) { - CSP_THROW(RuntimeException, "could not create connection because: " << ec.message()); - } - const csp::Dictionary &headers = *m_properties.get("headers"); - - for( auto it = headers.begin(); it != headers.end(); ++it ) - { - const std::string key = it.key(); - const std::string value = headers.get( key ); - con.get() -> append_header( key, value ); - } - - m_client.connect( con ); - m_hdl = con -> get_handle(); - m_client.run(); - m_client.reset(); -} - -void WebsocketEndpointTLS::close() -{ - websocketpp::lib::error_code ec; - m_client.close( m_hdl , websocketpp::close::status::going_away, "", ec ); - if( ec ) { - CSP_THROW( RuntimeException, "could not close connection because: " << ec.message() ); - } -} - - -/* -WebsocketEndpointNoTLS -> tls impl -*/ -WebsocketEndpointNoTLS::WebsocketEndpointNoTLS( csp::Dictionary properties ) -: WebsocketEndpointBase( properties ) -{ - if (m_properties.get("verbose_log")) { - m_client.set_access_channels(websocketpp::log::alevel::all); - // m_client.clear_access_channels(websocketpp::log::alevel::frame_payload); } else { - m_client.clear_access_channels(websocketpp::log::alevel::all); + m_session = new WebsocketSessionNoTLS( + m_ioc, + &m_properties, + m_on_open, + m_on_fail, + m_on_message, + m_on_close, + m_on_send_fail + ); } - m_client.init_asio(); + m_session->run(); - // need to set these with callbacks - m_client.set_open_handler( [ this ]( websocketpp::connection_hdl ){ - m_on_open(); - }); - m_client.set_message_handler( [ this ]( websocketpp::connection_hdl, message_ptr msg ) { - m_on_message( msg -> get_payload() ); - }); - m_client.set_fail_handler( [ this ]( websocketpp::connection_hdl ){ - m_on_fail(); - }); - m_client.set_close_handler( [ this ]( websocketpp::connection_hdl ){ - m_on_close(); - }); + m_ioc.run(); } -WebsocketEndpointNoTLS::~WebsocketEndpointNoTLS() -{ } -void WebsocketEndpointNoTLS::send( const std::string& s ) -{ - websocketpp::lib::error_code ec; - m_client.send( m_hdl, s, websocketpp::frame::opcode::value::TEXT, ec ); - if( ec ) m_on_send_fail( s ); +void WebsocketEndpoint::stop() +{ + m_ioc.stop(); + if(m_session) m_session->stop(); } -void WebsocketEndpointNoTLS::run() -{ - auto uri = m_properties.get( "uri" ); - websocketpp::lib::error_code ec; - client::connection_ptr con = m_client.get_connection( uri, ec ); - if(ec) { - CSP_THROW( RuntimeException, "could not create connection because: " << ec.message() ); - } - const csp::Dictionary &headers = *m_properties.get("headers"); - - for( auto it = headers.begin(); it != headers.end(); ++it ) - { - const std::string key = it.key(); - const std::string value = headers.get( key ); - con.get() -> append_header( key, value ); - } - m_client.connect( con ); - m_hdl = con -> get_handle(); - m_client.run(); - m_client.reset(); +csp::Dictionary& WebsocketEndpoint::getProperties() { + return m_properties; } -void WebsocketEndpointNoTLS::close() -{ - websocketpp::lib::error_code ec; - m_client.close( m_hdl, websocketpp::close::status::going_away, "Good bye", ec ); - if( ec ) { - CSP_THROW( RuntimeException, "could not close connection because: " << ec.message() ); - } -} +void WebsocketEndpoint::send(const std::string& s) +{ if(m_session) m_session->send(s); } + } \ No newline at end of file diff --git a/cpp/csp/adapters/websocket/WebsocketEndpoint.h b/cpp/csp/adapters/websocket/WebsocketEndpoint.h index e4b1e095f..cfca08742 100644 --- a/cpp/csp/adapters/websocket/WebsocketEndpoint.h +++ b/cpp/csp/adapters/websocket/WebsocketEndpoint.h @@ -1,84 +1,370 @@ #ifndef _IN_CSP_ADAPTERS_WEBSOCKETS_ENDPOINT_H #define _IN_CSP_ADAPTERS_WEBSOCKETS_ENDPOINT_H -// need a base -> TLS, base -> No TLS -#include -#include -#include -#include +#include +#include +#include +#include + +#include +#include + +#include #include +#include +#include namespace csp::adapters::websocket { +using namespace csp; -using on_message_cb = std::function; -using on_send_fail_cb = std::function; -using void_cb = std::function; +namespace beast = boost::beast; // from +namespace http = beast::http; // from +namespace net = boost::asio; // from +namespace ssl = boost::asio::ssl; // from +namespace websocket = beast::websocket; // from +using tcp = boost::asio::ip::tcp; // from -using message_ptr = websocketpp::config::core_client::message_type::ptr; +using string_cb = std::function; +using char_cb = std::function; +using void_cb = std::function; +class BaseWebsocketSession { +public: + virtual void stop() { }; + virtual void send( const std::string& ) { }; + virtual void do_read() { }; + virtual void do_write(const std::string& ) { }; + virtual void run() { }; +}; -class WebsocketEndpointBase { +template +class WebsocketSession : public BaseWebsocketSession { public: - WebsocketEndpointBase(csp::Dictionary properties); - virtual ~WebsocketEndpointBase() { }; + WebsocketSession( + net::io_context& ioc, + Dictionary* properties, + void_cb& on_open, + string_cb& on_fail, + char_cb& on_message, + void_cb& on_close, + string_cb& on_send_fail + ) : m_resolver( net::make_strand( ioc ) ), + m_properties( properties ), + m_on_open( on_open ), + m_on_fail( on_fail ), + m_on_message( on_message ), + m_on_close( on_close ), + m_on_send_fail( on_send_fail ) + { }; - virtual void run() { }; - virtual void send(const std::string&){ }; - virtual void close() { }; + Derived& derived(){ return static_cast(*this); } + + void handle_message( beast::error_code& ec, std::size_t& bytes_transfered ) { + if( ec ) { + m_on_close(); + return; + } + m_on_message(beast::buffers_front(m_buffer.data()).data(), m_buffer.size()); + m_buffer.consume(m_buffer.size()); + do_read(); + } - void setOnMessageCb(on_message_cb cb); - void setOnOpenCb(void_cb cb); - void setOnFailCb(void_cb cb); - void setOnCloseCb(void_cb cb); - void setOnSendFailCb(on_send_fail_cb cb); + void set_headers(websocket::request_type& req) { + const csp::Dictionary &headers = *m_properties->get("headers"); + + for( auto it = headers.begin(); it != headers.end(); ++it ) + { + const std::string key = it.key(); + const std::string value = headers.get( key ); + req.set(key, value); + } + } + + void do_read() override { + derived().ws().async_read( + m_buffer, + [ this ]( beast::error_code ec, std::size_t bytes_transfered ) + { handle_message( ec, bytes_transfered ); } + ); + } + + void stop() override + { + derived().ws().async_close( websocket::close_code::normal, [ this ]( beast::error_code ec ) { + if(ec) CSP_THROW(RuntimeException, ec.message()); + m_on_close(); + }); + } + + void send( const std::string& s ) override + { + net::post( + derived().ws().get_executor(), + [this, s]() + { + m_queue.push_back(s); + if (m_queue.size() > 1) return; + do_write(m_queue.front()); + } + ); + } + + void do_write(const std::string& s) override + { + derived().ws().async_write( + net::buffer(s), + [this](beast::error_code ec, std::size_t bytes_transfered) + { + // add logging here? + m_queue.erase(m_queue.begin()); + boost::ignore_unused(bytes_transfered); + if(ec) m_on_send_fail(ec.message()); + if(m_queue.size() >0) do_write(m_queue.front()); + } + ); + } - csp::Dictionary& getProperties(); public: - csp::Dictionary m_properties; + tcp::resolver m_resolver; + Dictionary* m_properties; void_cb m_on_open; - on_message_cb m_on_message; - void_cb m_on_fail; + string_cb m_on_fail; + char_cb m_on_message; void_cb m_on_close; - on_send_fail_cb m_on_send_fail; + string_cb m_on_send_fail; + beast::flat_buffer m_buffer; + std::vector m_queue; }; +class WebsocketSessionNoTLS final: public WebsocketSession { +public: + WebsocketSessionNoTLS( + net::io_context& ioc, + Dictionary* properties, + void_cb& on_open, + string_cb& on_fail, + char_cb& on_message, + void_cb& on_close, + string_cb& on_send_fail + ) : WebsocketSession( + ioc, + properties, + on_open, + on_fail, + on_message, + on_close, + on_send_fail + ), + m_ws(net::make_strand(ioc)) + { } + + void run() override { + m_resolver.async_resolve( + m_properties->get("host").c_str(), + m_properties->get("port").c_str(), + [this]( beast::error_code ec, tcp::resolver::results_type results ) { + if(ec) { + m_on_fail(ec.message()); + return; + } + // Set the timeout for the operation + beast::get_lowest_layer(m_ws).expires_after(std::chrono::seconds(5)); -class WebsocketEndpointTLS final: public WebsocketEndpointBase { + // Make the connection on the IP address we get from a lookup + beast::get_lowest_layer(m_ws).async_connect( + results, + [this]( beast::error_code ec, tcp::resolver::results_type::endpoint_type ep ) + { + // Turn off the timeout on the tcp_stream, because + // the websocket stream has its own timeout system. + if(ec) { + m_on_fail(ec.message()); + return; + } -using tls_client = websocketpp::client; + beast::get_lowest_layer(m_ws).expires_never(); -public: - WebsocketEndpointTLS(const csp::Dictionary properties); - ~WebsocketEndpointTLS(); + m_ws.set_option( + websocket::stream_base::timeout::suggested( + beast::role_type::client)); - void run() override; - void send(const std::string& s) override; - void close() override; + m_ws.set_option(websocket::stream_base::decorator( + [this](websocket::request_type& req) + { + set_headers(req); + req.set(http::field::user_agent, "CSP WebsocketEndpoint"); + } + )); + std::string host_ = m_properties->get("host") + ':' + std::to_string(ep.port()); + m_ws.async_handshake( + host_, + m_properties->get("route"), + [this]( beast::error_code ec ) { + if(ec) { + m_on_fail(ec.message()); + return; + } + m_on_open(); + m_ws.async_read( + m_buffer, + [ this ]( beast::error_code ec, std::size_t bytes_transfered ) + { handle_message( ec, bytes_transfered ); } + ); + } + ); + } + ); + } + ); + } + + websocket::stream& ws() + { return m_ws; } private: - tls_client m_client; - websocketpp::connection_hdl m_hdl; + websocket::stream m_ws; }; +class WebsocketSessionTLS final: public WebsocketSession { +public: + WebsocketSessionTLS( + net::io_context& ioc, + ssl::context& ctx, + Dictionary* properties, + void_cb& on_open, + string_cb& on_fail, + char_cb& on_message, + void_cb& on_close, + string_cb& on_send_fail + ) : WebsocketSession( + ioc, + properties, + on_open, + on_fail, + on_message, + on_close, + on_send_fail + ), + m_ws(net::make_strand(ioc), ctx) + { } + + void run() override { + m_resolver.async_resolve( + m_properties->get("host").c_str(), + m_properties->get("port").c_str(), + [this]( beast::error_code ec, tcp::resolver::results_type results ) { + if(ec) { + m_on_fail(ec.message()); + return; + } + // Set the timeout for the operation + beast::get_lowest_layer(m_ws).expires_after(std::chrono::seconds(5)); + + // Make the connection on the IP address we get from a lookup + beast::get_lowest_layer(m_ws).async_connect( + results, + [this]( beast::error_code ec, tcp::resolver::results_type::endpoint_type ep ) + { + if(ec) { + m_on_fail(ec.message()); + return; + } + + if(! SSL_set_tlsext_host_name( + m_ws.next_layer().native_handle(), + m_properties->get("host").c_str())) + { + ec = beast::error_code(static_cast(::ERR_get_error()), + net::error::get_ssl_category()); + m_on_fail(ec.message()); + return; + } + + m_complete_host = m_properties->get("host") + ':' + std::to_string(ep.port()); + + // ssl handler + m_ws.next_layer().async_handshake( + ssl::stream_base::client, + [this]( beast::error_code ec ) { + if(ec) { + m_on_fail(ec.message()); + return; + } + + beast::get_lowest_layer(m_ws).expires_never(); + // Set suggested timeout settings for the websocket + m_ws.set_option(websocket::stream_base::timeout::suggested(beast::role_type::client)); -class WebsocketEndpointNoTLS final: public WebsocketEndpointBase { + // Set a decorator to change the User-Agent of the handshake + m_ws.set_option(websocket::stream_base::decorator( + [this](websocket::request_type& req) + { + set_headers(req); + req.set(http::field::user_agent, "CSP WebsocketAdapter"); + })); + + m_ws.async_handshake( + m_complete_host, + m_properties->get("route"), + [this]( beast::error_code ec ) { + if(ec) { + m_on_fail(ec.message()); + return; + } + m_on_open(); + m_ws.async_read( + m_buffer, + [ this ]( beast::error_code ec, std::size_t bytes_transfered ) + { handle_message( ec, bytes_transfered ); } + ); + } + ); -using client = websocketpp::client; + } + ); + } + ); + } + ); + } + websocket::stream>& ws() + { return m_ws; } + +private: + websocket::stream> m_ws; + std::string m_complete_host; +}; + +class WebsocketEndpoint { public: - WebsocketEndpointNoTLS(csp::Dictionary properties); - ~WebsocketEndpointNoTLS(); + WebsocketEndpoint( Dictionary properties ); + virtual ~WebsocketEndpoint() { }; - void run() override; - void send(const std::string& s) override; - void close() override; + void setOnOpen(void_cb on_open); + void setOnFail(string_cb on_fail); + void setOnMessage(char_cb on_message); + void setOnClose(void_cb on_close); + void setOnSendFail(string_cb on_send_fail); + Dictionary& getProperties(); + void run(); + void stop(); + void send(const std::string& s); private: - client m_client; - websocketpp::connection_hdl m_hdl; + Dictionary m_properties; + BaseWebsocketSession* m_session; + net::io_context m_ioc; + void_cb m_on_open; + string_cb m_on_fail; + char_cb m_on_message; + void_cb m_on_close; + string_cb m_on_send_fail; }; + + } #endif \ No newline at end of file diff --git a/cpp/csp/core/BasicAllocator.h b/cpp/csp/core/BasicAllocator.h index 91f02cb38..da3aac792 100644 --- a/cpp/csp/core/BasicAllocator.h +++ b/cpp/csp/core/BasicAllocator.h @@ -1,12 +1,16 @@ #ifndef _IN_CSP_CORE_BASIC_ALLOCATOR_H #define _IN_CSP_CORE_BASIC_ALLOCATOR_H +#include #include #include -#include #include #include +#ifdef __linux__ +#include +#endif + namespace csp { @@ -68,9 +72,11 @@ inline BasicAllocator::~BasicAllocator() { for( auto & entry : m_arenas ) { +#ifdef __linux__ if( entry.mmap ) munmap( entry.buffer, entry.size ); else +#endif ::free( entry.buffer ); } } diff --git a/cpp/csp/core/CMakeLists.txt b/cpp/csp/core/CMakeLists.txt index 672323794..592ca9b60 100644 --- a/cpp/csp/core/CMakeLists.txt +++ b/cpp/csp/core/CMakeLists.txt @@ -30,7 +30,7 @@ set_target_properties(csp_core PROPERTIES PUBLIC_HEADER "${CORE_PUBLIC_HEADERS}" install(TARGETS csp_core PUBLIC_HEADER DESTINATION include/csp/core - RUNTIME DESTINATION bin/ + RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR} LIBRARY DESTINATION lib/ ) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/cpp/csp/core/Config.h DESTINATION include/csp/core) diff --git a/cpp/csp/core/DynamicBitSet.h b/cpp/csp/core/DynamicBitSet.h index e8f7f303c..5fb120f06 100644 --- a/cpp/csp/core/DynamicBitSet.h +++ b/cpp/csp/core/DynamicBitSet.h @@ -2,6 +2,7 @@ #define _IN_CSP_CORE_DYNAMICBITSET_H #include +#include #include namespace csp @@ -154,27 +155,19 @@ class DynamicBitSet private: using nbit_type = uint8_t; +#ifndef WIN32 +#define CLZ_CONSTEXPR constexpr +#else +#define CLZ_CONSTEXPR +#endif + template::value, bool> = true> static constexpr nbit_type nbits() { return sizeof( value_type ) * 8; } - template::value, bool> = true> - static constexpr nbit_type log2( U n ) { return nbits() - __builtin_clz( n ) - 1; } - static constexpr nbit_type log2( uint64_t n ) { return nbits() - __builtin_clzl( n ) - 1; } - - // clz (count leading zeros) returns number of leading zeros before MSB (i.e. clz(00110..) = 2 ) - // __builtin_clz auto-promotes to 32-bits: need to subtract off extra leading zeros - static constexpr nbit_type clz( uint8_t n ) { return __builtin_clz( n ) - 24; } - static constexpr nbit_type clz( uint16_t n ) { return __builtin_clz( n ) - 16; } - static constexpr nbit_type clz( uint32_t n ) { return __builtin_clz( n ); } - static constexpr nbit_type clz( uint64_t n ) { return __builtin_clzl( n ); } - - // ffs (find first set) returns offset of first set bit (i.e. ffs(..0110) = 2 ), with ffs(0) = 0 - template::value, bool> = true> - static constexpr nbit_type ffs( U n ) { return __builtin_ffs( n ); } - static constexpr nbit_type ffs( uint64_t n ) { return __builtin_ffsl( n ); } + template::value, bool> = true> + static CLZ_CONSTEXPR nbit_type log2( U n ) { return nbits() - clz(static_cast( n )) - 1; } //upcast to 32 bit to avoid truncation for log2 + static CLZ_CONSTEXPR nbit_type log2(uint64_t n) { return nbits() - clz(n) - 1; } static constexpr node_type mask( nbit_type bitIndex ) { return ( node_type )1 << bitIndex; } @@ -188,8 +181,8 @@ class DynamicBitSet return ( index_type )bit + ( node << _logBits ); } - static constexpr nbit_type _bits = nbits(); - static constexpr nbit_type _logBits = log2( _bits ); + static constexpr nbit_type _bits = nbits(); + static inline CLZ_CONSTEXPR nbit_type _logBits = log2( _bits ); node_type * m_nodes; index_type m_size; @@ -200,4 +193,4 @@ class DynamicBitSet } -#endif \ No newline at end of file +#endif diff --git a/cpp/csp/core/Enum.h b/cpp/csp/core/Enum.h index ca0a129af..d6dac5ee9 100644 --- a/cpp/csp/core/Enum.h +++ b/cpp/csp/core/Enum.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -70,13 +71,13 @@ auto UnknownOnInvalidValue(int) -> decltype(T::UNKNOWN_ON_INVALID_VALUE) { retur template bool UnknownOnInvalidValue(long) { return false; } +START_PACKED template struct Enum : public EnumTraits { using EnumV = typename EnumTraits::_enum; using Mapping = std::vector; - //these checks for enum is for the SubVenue "hack" using UType = typename EnumUTypeHelper::value>::type; template::value> > @@ -169,22 +170,6 @@ struct Enum : public EnumTraits return it->second; } - - void addAliases( const Aliases &aliases ) - { - for( auto &entry : aliases ) - { - auto it = this -> find( entry.first.c_str() ); - if( it == this -> end() ) - CSP_THROW(ValueError, "Invalid Enum alias, unrecognized enum value " << entry.first); - - if( this -> find( entry.second.c_str() ) != this -> end() ) - CSP_THROW(ConfigException, "Attempting to add conflicting alias " << entry.second << " to " - << typeid(EnumTraits).name()); - - (*this)[strdup(entry.second.c_str())] = it -> second; - } - } }; //This is defined by INIT macro @@ -196,7 +181,7 @@ struct Enum : public EnumTraits return s_reverseMap; } -} __attribute__((packed)); +} END_PACKED; template Enum::Enum( UType v ) diff --git a/cpp/csp/core/Exception.cpp b/cpp/csp/core/Exception.cpp index 0c7489ef7..e4480754b 100644 --- a/cpp/csp/core/Exception.cpp +++ b/cpp/csp/core/Exception.cpp @@ -1,7 +1,6 @@ #include +#include -#include -#include #include #include @@ -10,6 +9,10 @@ #include #include +#ifndef WIN32 +#include +#endif + //From https://stackoverflow.com/questions/2443135/how-do-i-find-where-an-exception-was-thrown-in-c/2443366 static void csp_terminate( void ); static bool set_sigabrt_handler(); @@ -53,12 +56,14 @@ static void printBacktrace( char ** messages, int size, std::ostream & dest ) begin_name++; *begin_offset = '\0'; +#ifndef WIN32 int status; - char* demangled = abi::__cxa_demangle(begin_name, - NULL, NULL, &status); - + char* demangled = abi::__cxa_demangle(begin_name, NULL, NULL, &status); dest << "[bt]: (" << i << ") " << ( status == 0 ? demangled : messages[i] ) << std::endl; free( demangled ); +#else + dest << "[bt]: (" << i << ") " << messages[i] << std::endl; +#endif } else { @@ -71,11 +76,14 @@ static void printBacktrace( char ** messages, int size, std::ostream & dest ) void printBacktrace() { +//TODO get stack traces on windows +#ifndef WIN32 void *array[50]; int size = backtrace( array, 50 ); auto messages = backtrace_symbols( array, size ); printBacktrace( messages, size, std::cerr ); free( messages ); +#endif } void csp_terminate() @@ -115,41 +123,61 @@ void csp_terminate() } //This is for coredumps -void sigabrt_handler( int sig_num, siginfo_t * info, void * ctx ) +#ifndef WIN32 +void sigabrt_handler(int sig_num, siginfo_t* info, void* ctx) { - std::cerr << "signal " << sig_num - << " (" << strsignal( sig_num ) << "), address is " - << info -> si_addr << " from " << std::endl; + std::cerr << "signal " << sig_num + << " (" << strsignal(sig_num) << "), address is " + << info -> si_addr << " from " << std::endl; printBacktrace(); - signal( SIGABRT, SIG_DFL ); - signal( SIGSEGV, SIG_DFL ); - signal( SIGBUS, SIG_DFL ); + signal(SIGABRT, SIG_DFL); + signal(SIGSEGV, SIG_DFL); + signal(SIGBUS, SIG_DFL); abort(); } bool set_sigabrt_handler() { - static struct sigaction sigact; sigact.sa_sigaction = sigabrt_handler; - sigact.sa_flags = SA_RESTART | SA_SIGINFO; + sigact.sa_flags = SA_RESTART | SA_SIGINFO; - sigaction( SIGABRT, &sigact, NULL ); - sigaction( SIGSEGV, &sigact, NULL ); - sigaction( SIGBUS, &sigact, NULL ); + sigaction(SIGABRT, &sigact, NULL); + sigaction(SIGSEGV, &sigact, NULL); + sigaction(SIGBUS, &sigact, NULL); + return true; +} +#else +void sigabrt_handler(int sig_num) +{ + std::cerr << "signal " << sig_num << " from " << std::endl; + printBacktrace(); + + signal(SIGABRT, SIG_DFL); + signal(SIGSEGV, SIG_DFL); + abort(); +} + +bool set_sigabrt_handler() +{ + signal(SIGABRT, sigabrt_handler); + signal(SIGSEGV, sigabrt_handler); return true; } +#endif void csp::Exception::setbt() { +#ifndef WIN32 void *array[50]; m_backtracesize = backtrace( array, 50 ); char **messages = backtrace_symbols( array, m_backtracesize ); m_backtracemessages = messages; +#endif } static char ** dupe_backtraces( char** bt, int n ) diff --git a/cpp/csp/core/Exception.h b/cpp/csp/core/Exception.h index d03f863b0..5227acf34 100644 --- a/cpp/csp/core/Exception.h +++ b/cpp/csp/core/Exception.h @@ -5,16 +5,22 @@ #include #include #include +#include namespace csp { -class Exception : public std::exception +class CSP_PUBLIC Exception : public std::exception { public: - Exception( const char * exType, const std::string & description ) : m_exType( exType ), m_description( description ), m_line ( -1 ) { setbt(); } Exception( const char * exType, const std::string & description, const char * file, const char * func, int line ) : - m_exType( exType ), m_description( description ), m_file( file ), m_function( func ), m_line( line ) { setbt(); } + m_exType( exType ), m_description( description ), m_file( file ), m_function( func ), m_line( line ), m_backtracemessages( nullptr ) + { + setbt(); + } + + Exception(const char* exType, const std::string& description) : Exception(exType, description, "", "", -1) + {} ~Exception() { free( m_backtracemessages ); } Exception( const Exception & ); Exception( Exception&& ); @@ -53,7 +59,7 @@ class Exception : public std::exception }; #define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) -#define CSP_DECLARE_EXCEPTION( DerivedException, BaseException ) class DerivedException : public BaseException { public: DerivedException( const char * exType, const std::string &r, const char * file, const char * func, int line ) : BaseException( exType, r, file, func, line ) {} }; +#define CSP_DECLARE_EXCEPTION( DerivedException, BaseException ) class CSP_PUBLIC DerivedException : public BaseException { public: DerivedException( const char * exType, const std::string &r, const char * file, const char * func, int line ) : BaseException( exType, r, file, func, line ) {} }; CSP_DECLARE_EXCEPTION( AssertionError, Exception ) CSP_DECLARE_EXCEPTION( RuntimeException, Exception ) @@ -71,9 +77,9 @@ CSP_DECLARE_EXCEPTION( OSError, RuntimeException ) CSP_DECLARE_EXCEPTION( OutOfMemoryError, RuntimeException ) CSP_DECLARE_EXCEPTION( FileNotFoundError, IOError ) - template -[[noreturn]] void throw_exc(T&& e) __attribute__ ((noinline)); +[[noreturn]] NO_INLINE void throw_exc(T&& e); + template [[noreturn]] inline void throw_exc(T&& e) {throw e;} diff --git a/cpp/csp/core/FileUtils.h b/cpp/csp/core/FileUtils.h index 59a6dd260..a4fa9e6ca 100644 --- a/cpp/csp/core/FileUtils.h +++ b/cpp/csp/core/FileUtils.h @@ -8,9 +8,7 @@ namespace csp::utils inline std::string dirname( const std::string & path ) { - if( path.length() >= 2 ) - return path.substr( 0, path.rfind( "/", path.length() - 2 ) ); - return ""; + return std::filesystem::path(path).parent_path().string(); } //files and directories are treated equally @@ -19,16 +17,16 @@ inline bool fileExists( const std::string & fileOrDir ) return std::filesystem::exists(fileOrDir); } -inline void mkdir( const std::string & path, mode_t mode = S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH ) +inline void mkdir( const std::string & path, + std::filesystem::perms perms = std::filesystem::perms::owner_all | std::filesystem::perms::group_all | std::filesystem::perms::others_read | std::filesystem::perms::others_exec ) { - size_t pos = 0; - do + if (!fileExists(path)) { - pos = path.find( '/', pos + 1 ); - std::string subpath = path.substr( 0, pos ); - if( !fileExists( subpath) && ( ::mkdir( subpath.c_str(), mode ) == -1 && errno != EEXIST ) ) - CSP_THROW( IOError, "Failed to create path " << subpath << ": " << strerror( errno ) ); - } while( pos != std::string::npos ); + std::error_code err; + if (!std::filesystem::create_directories(path, err)) + CSP_THROW(IOError, "Failed to create path " << path << ": " << err.message()); + std::filesystem::permissions(path, perms); + } } } diff --git a/cpp/csp/core/Likely.h b/cpp/csp/core/Likely.h index 2d0589eb4..74829c5d0 100644 --- a/cpp/csp/core/Likely.h +++ b/cpp/csp/core/Likely.h @@ -1,7 +1,13 @@ #ifndef _IN_CSP_CORE_LIKELY_H #define _IN_CSP_CORE_LIKELY_H +// We should move to [[likely]] [[unlikely]] attributes once we enable c++20 +#ifndef WIN32 #define likely(x) __builtin_expect ( (x), 1 ) #define unlikely(x) __builtin_expect ( (x), 0 ) +#else +#define likely(x) x +#define unlikely(x) x +#endif #endif diff --git a/cpp/csp/core/Platform.h b/cpp/csp/core/Platform.h index 18757d658..3ef861259 100644 --- a/cpp/csp/core/Platform.h +++ b/cpp/csp/core/Platform.h @@ -1,11 +1,120 @@ #ifndef _IN_CSP_CORE_PLATFORM_H #define _IN_CSP_CORE_PLATFORM_H +#include #include +#include -#ifdef __linux__ - typedef uint64_t binding_int_t; +//TODO move Likely.h defines into Platform.h +#ifdef WIN32 +#define NOMINMAX +#include +#include +#include + +#undef ERROR +#undef GetMessage + +#define CSP_LOCAL +#define CSP_PUBLIC __declspec(dllexport) + +#ifdef CSPTYPESIMPL_EXPORTS +#define CSPTYPESIMPL_EXPORT __declspec(dllexport) +#else +#define CSPTYPESIMPL_EXPORT __declspec(dllimport) +#endif + +#ifdef CSPIMPL_EXPORTS +#define CSPIMPL_EXPORT __declspec(dllexport) #else - typedef int64_t binding_int_t; +#define CSPIMPL_EXPORT __declspec(dllimport) #endif -#endif \ No newline at end of file +#define START_PACKED __pragma( pack(push, 1) ) +#define END_PACKED __pragma( pack(pop)) + +#define NO_INLINE __declspec(noinline) + +inline tm * localtime_r( const time_t * timep, tm * result ) +{ + tm * rv = localtime(timep); + if (rv) + *result = *rv; + + return result; +} + +#define timegm _mkgmtime + +inline int nanosleep(const timespec* req, timespec* rem) +{ + assert(rem == nullptr); + int64_t millis = req->tv_sec * 1000 + req->tv_nsec * 1000000; + Sleep(millis); + return 0; +} + +inline uint8_t clz(uint64_t n) +{ + unsigned long index = 0; + if (_BitScanReverse64(&index, n)) + return 64 - index - 1; + return 0; +} + +inline uint8_t clz(uint32_t n) +{ + unsigned long index = 0; + if (_BitScanReverse(&index, n)) + return 32 - index - 1; + return 0; +} + +inline uint8_t clz(uint16_t n) { return clz(static_cast(n)) - 16; } +inline uint8_t clz(uint8_t n) { return clz(static_cast(n)) - 24; } + +template::value, bool> = true> +inline uint8_t ffs(U n) +{ + unsigned long index = 0; + if (_BitScanForward(&index, n)) + return index + 1; + return 0; +} + +inline uint8_t ffs(uint64_t n) +{ + unsigned long index = 0; + if (_BitScanForward64(&index, n)) + return index + 1; + return 0; +} + +#else + +#define CSPIMPL_EXPORT __attribute__ ((visibility ("default"))) +#define CSPTYPESIMPL_EXPORT __attribute__ ((visibility ("default"))) + +#define CSP_LOCAL __attribute__ ((visibility ("hidden"))) +#define CSP_PUBLIC __attribute__ ((visibility ("default"))) + +#define START_PACKED +#define END_PACKED __attribute__((packed)) + +#define NO_INLINE __attribute__ ((noinline)) + +inline constexpr uint8_t clz(uint32_t n) { return __builtin_clz(n); } +inline constexpr uint8_t clz(uint64_t n) { return __builtin_clzl(n); } + +// clz (count leading zeros) returns number of leading zeros before MSB (i.e. clz(00110..) = 2 ) +// __builtin_clz auto-promotes to 32-bits: need to subtract off extra leading zeros +inline constexpr uint8_t clz(uint16_t n) { return clz(static_cast(n)) - 16; } +inline constexpr uint8_t clz(uint8_t n) { return clz(static_cast(n)) - 24; } + +// ffs (find first set) returns offset of first set bit (i.e. ffs(..0110) = 2 ), with ffs(0) = 0 +template::value, bool> = true> +inline constexpr uint8_t ffs( U n ) { return __builtin_ffs(n); } +inline constexpr uint8_t ffs( uint64_t n ) { return __builtin_ffsl(n); } + +#endif + +#endif diff --git a/cpp/csp/core/System.h b/cpp/csp/core/System.h index 55beecbbf..90f56f5aa 100644 --- a/cpp/csp/core/System.h +++ b/cpp/csp/core/System.h @@ -3,17 +3,14 @@ //Common low level system methods / defines #include -#include -#include -#include -#include #include #include -#include -#include -#include #include +#ifndef WIN32 +#include +#endif + namespace csp { @@ -25,12 +22,14 @@ std::string cpp_type_name() { int status = 0; std::string result = typeid(*(T*)nullptr).name(); +#ifndef WIN32 char * demangled = abi::__cxa_demangle(result.c_str(), NULL, NULL, &status); if( demangled ) { result = demangled; free( demangled ); } +#endif return result; } diff --git a/cpp/csp/core/Time.h b/cpp/csp/core/Time.h index c262f4dbd..f2e73d990 100644 --- a/cpp/csp/core/Time.h +++ b/cpp/csp/core/Time.h @@ -2,13 +2,13 @@ #define _IN_CSP_CORE_TIME_H #include +#include #include #include #include #include #include #include -#include #include #include #include @@ -394,7 +394,7 @@ inline Time::Time( int hour, int minute, int second, int32_t nanosecond ) if( nanosecond >= NANOS_PER_SECOND || nanosecond < 0 ) CSP_THROW( ValueError, "Nanosecond out of range: " << nanosecond ); - m_ticks = ( hour * 3600 + minute * 60 + second ) * NANOS_PER_SECOND + nanosecond; + m_ticks = ( int64_t( hour ) * 3600 + int64_t( minute ) * 60 + int64_t( second ) ) * NANOS_PER_SECOND + nanosecond; } inline Time& Time::operator +=( const TimeDelta & delta ) @@ -450,7 +450,6 @@ class DateTime { public: DateTime() : DateTime( DateTime::NONE() ) {} - DateTime( const timeval & tv ); DateTime( int year, int month, int day, int hour = 0, int minute = 0, int second = 0, int nanosecond = 0 ); DateTime( Date date, Time time ); @@ -506,7 +505,7 @@ class DateTime DateTime& operator +=( const TimeDelta & delta ) { m_ticks += delta.asNanoseconds(); return *this; } DateTime& operator -=( const TimeDelta & delta ) { m_ticks -= delta.asNanoseconds(); return *this; } - static constexpr DateTime NONE() { return DateTime( std::numeric_limits::min() ); } + static constexpr DateTime NONE() { return DateTime(std::numeric_limits::min()); } static constexpr DateTime MIN_VALUE() { return DateTime( std::numeric_limits::min() + 1 ); } //min reserved for NONE static constexpr DateTime MAX_VALUE() { return DateTime( std::numeric_limits::max() ); } @@ -536,11 +535,6 @@ inline DateTime::DateTime( int year, int month, int day, } -inline DateTime::DateTime( const timeval & tv ) -{ - m_ticks = tv.tv_sec * NANOS_PER_SECOND + tv.tv_usec * NANOS_PER_MICROSECOND; -} - inline DateTime::DateTime( Date date, Time time ) : DateTime( date.year(), date.month(), date.day(), time.hour(), time.minute(), time.second(), time.nanosecond() ) { @@ -549,7 +543,11 @@ inline DateTime::DateTime( Date date, Time time ) : inline DateTime DateTime::now() { timespec ts; +#ifdef WIN32 + timespec_get(&ts, TIME_UTC); +#else clock_gettime( CLOCK_REALTIME, &ts ); +#endif return DateTime( ts.tv_sec * NANOS_PER_SECOND + ts.tv_nsec ); } diff --git a/cpp/csp/cppnodes/CMakeLists.txt b/cpp/csp/cppnodes/CMakeLists.txt index 32fb9c7e1..10af9fee6 100644 --- a/cpp/csp/cppnodes/CMakeLists.txt +++ b/cpp/csp/cppnodes/CMakeLists.txt @@ -13,6 +13,6 @@ target_link_libraries(statsimpl baselibimpl csp_core csp_engine) install(TARGETS baselibimpl basketlibimpl mathimpl statsimpl PUBLIC_HEADER DESTINATION include/csp/cppnodes - RUNTIME DESTINATION bin/ + RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR} LIBRARY DESTINATION lib/ ) diff --git a/cpp/csp/cppnodes/baselibimpl.cpp b/cpp/csp/cppnodes/baselibimpl.cpp index a0cd2a029..52a5537d9 100644 --- a/cpp/csp/cppnodes/baselibimpl.cpp +++ b/cpp/csp/cppnodes/baselibimpl.cpp @@ -445,13 +445,14 @@ DECLARE_CPPNODE( collect ) //we expect all elements to be of the same type switchCspType( elemType, [this]( auto tag ) { - using ElemT = typename decltype(tag)::type; - using ArrayT = typename CspType::Type::toCArrayType::type; - - ArrayT & out = unnamed_output().reserveSpace(); - out.clear(); - for( auto it = x.tickedinputs(); it; ++it ) - out.emplace_back( it -> lastValueTyped() ); + using TagType = decltype( tag ); + using ElemT = typename TagType::type; + using ArrayT = typename CspType::Type::toCArrayType::type; + + ArrayT & out = unnamed_output().reserveSpace(); + out.clear(); + for( auto it = x.tickedinputs(); it; ++it ) + out.emplace_back( it -> lastValueTyped() ); } ); } }; diff --git a/cpp/csp/cppnodes/statsimpl.h b/cpp/csp/cppnodes/statsimpl.h index 765b3aac2..61cda2346 100644 --- a/cpp/csp/cppnodes/statsimpl.h +++ b/cpp/csp/cppnodes/statsimpl.h @@ -1,3 +1,6 @@ +#ifndef _IN_CSP_CPPNODES_STATSIMPL_H +#define _IN_CSP_CPPNODES_STATSIMPL_H + #include #include @@ -5,11 +8,9 @@ #include #include #include - -#ifndef __clang__ -#include -#include -#endif +#include +#include +#include namespace csp::cppnodes { @@ -825,11 +826,11 @@ double kurtCompute( double count, double mx, double mx2, double mx3, double mx4, double kfactor = ( count + 1 ) * ( count - 1 ) / ( ( count - 2 ) * ( count - 3 ) ); double ub_kurt = kfactor * bias_kurt; - if( excess ) - { - double gfactor = ( count - 1 ) / ( count + 1 ) * kfactor; - ub_kurt -= 3 * gfactor; - } + double gfactor = ( count - 1 ) / ( count + 1 ) * kfactor; + ub_kurt -= 3 * gfactor; + if( !excess ) + ub_kurt += 3; + return ub_kurt; } @@ -1084,19 +1085,8 @@ class WeightedKurtosis bool m_excess; }; -#ifndef __clang__ -template -using ost = __gnu_pbds::tree; - -template -void ost_erase( ost &t, double & v ) -{ - int rank = t.order_of_key( v ); - auto it = t.find_by_order( rank ); - t.erase( it ); -} -#endif +template +using ost = boost::multi_index::multi_index_container, Comparator>>>; class Quantile { @@ -1139,11 +1129,7 @@ class Quantile void remove( double x ) { - #ifndef __clang__ - ost_erase( m_tree, x ); - #else m_tree.erase( m_tree.find( x ) ); - #endif } void reset() @@ -1162,113 +1148,60 @@ class Quantile double target = std::get( m_quants[index]._data ) * ( m_tree.size() - 1 ); int ft = floor( target ); int ct = ceil( target ); + auto fIt = m_tree.get<0>().nth( ft ); + auto cIt = ( ft == ct ) ? fIt : std::next( fIt ); double qtl = 0.0; - #ifndef __clang__ - switch ( m_interpolation ) - { - case LINEAR: - if( ft == target ) - { - qtl = *m_tree.find_by_order( ft ); - } - else - { - double lower = *m_tree.find_by_order( ft ); - double higher = *m_tree.find_by_order( ct ); - qtl = ( 1 - target + ft ) * lower + ( 1 - ct + target ) * higher; - } - break; - case LOWER: - qtl = *m_tree.find_by_order( ft ); - break; - case HIGHER: - qtl = *m_tree.find_by_order( ct ); - break; - case MIDPOINT: - if( ft == target ) - { - qtl = *m_tree.find_by_order( ft ); - } - else - { - double lower = *m_tree.find_by_order( ft ); - double higher = *m_tree.find_by_order( ct ); - qtl = ( higher+lower ) / 2; - } - break; - case NEAREST: - if( target - ft < ct - target ) - { - qtl = *m_tree.find_by_order( ft ); - } - else - { - qtl = *m_tree.find_by_order( ct ); - } - break; - default: - break; - } - #else - auto it = m_tree.begin(); - std::advance( it, ft ); switch ( m_interpolation ) { - case LINEAR: - if( ft == target ) - { - qtl = *it; - } - else - { - double lower = *it; - double higher = *++it; - qtl = ( 1 - target + ft ) * lower + ( 1 - ct + target ) * higher; - } - break; - case LOWER: - qtl = *it; - break; - case HIGHER: - qtl = ( ft == ct ? *it : *++it ); - break; - case MIDPOINT: - if( ft == target ) - { - qtl = *it; - } - else - { - double lower = *it; - double higher = *++it; - qtl = ( higher+lower ) / 2; - } - break; - case NEAREST: - if( target - ft <= ct - target ) - { - qtl = *it; - } - else - { - qtl = *++it; - } - break; - default: - break; + case LINEAR: + if ( ft == target ) + { + qtl = *fIt; + } + else + { + double lower = *fIt; + double higher = *cIt; + qtl = ( 1 - target + ft ) * lower + ( 1 - ct + target ) * higher; + } + break; + case LOWER: + qtl = *fIt; + break; + case HIGHER: + qtl = *cIt; + break; + case MIDPOINT: + if ( ft == target ) + { + qtl = *fIt; + } + else + { + double lower = *fIt; + double higher = *cIt; + qtl = ( higher + lower ) / 2; + } + break; + case NEAREST: + if ( target - ft < ct - target ) + { + qtl = *fIt; + } + else + { + qtl = *cIt; + } + break; + default: + break; } - #endif return qtl; } private: - - #ifndef __clang__ - ost> m_tree; - #else - std::multiset m_tree; - #endif + ost> m_tree; std::vector m_quants; int64_t m_interpolation; }; @@ -1356,14 +1289,10 @@ class Rank else { m_lastval = x; - #ifndef __clang__ if( m_method == MAX ) m_maxtree.insert( x ); else m_mintree.insert( x ); - #else - m_tree.insert( x ); - #endif } } @@ -1371,104 +1300,63 @@ class Rank { if( likely( !isnan( x ) ) ) { - #ifndef __clang__ - if( m_method == MAX ) - ost_erase( m_maxtree, x ); + if ( m_method == MAX ) + m_maxtree.erase ( m_maxtree.find( x ) ); else - ost_erase( m_mintree, x ); - #else - m_tree.erase( m_tree.find( x ) ); - #endif + m_mintree.erase ( m_mintree.find( x ) ); } } void reset() { - #ifndef __clang__ if( m_method == MAX ) m_maxtree.clear(); else m_mintree.clear(); - #else - m_tree.clear(); - #endif } double compute() const { // Verify tree is not empty and lastValue is valid // Last value can only ever be NaN if the "keep" nan option is used - #ifndef __clang__ if( likely( !isnan( m_lastval ) && ( ( m_method == MAX && m_maxtree.size() > 0 ) || m_mintree.size() > 0 ) ) ) { switch( m_method ) { case MIN: { - if( m_mintree.size() == 1 ) + if ( m_mintree.size() == 1 ) return 0; - return m_mintree.order_of_key( m_lastval ); + return m_mintree.get<0>().find_rank( m_lastval ); } case MAX: { - if( m_maxtree.size() == 1 ) + if ( m_maxtree.size() == 1 ) return 0; - return m_maxtree.size() - 1 - m_maxtree.order_of_key( m_lastval ); + return m_maxtree.size() - 1 - m_maxtree.get<0>().find_rank( m_lastval ); } case AVG: { - // Need to iterate to find average rank - if( m_mintree.size() == 1 ) + if ( m_mintree.size() == 1 ) return 0; - - int min_rank = m_mintree.order_of_key( m_lastval ); + + int min_rank = m_mintree.get<0>().find_rank( m_lastval ); int max_rank = min_rank; - auto it = m_mintree.find_by_order( min_rank ); + auto it = m_mintree.get<0>().nth( min_rank ); it++; - for( ; it != m_mintree.end() && *it == m_lastval ; it++ ) max_rank++; + for( ; it != m_mintree.end() && *it == m_lastval ; it++ ) max_rank++; // While this is in theory O(n), in reality this loop is only interated once, since there are likely no duplicate values or very few. return ( double )( min_rank + max_rank ) / 2; } - default: break; } } - #else - if( likely( !isnan( m_lastval ) && m_tree.size() > 0 ) ) - { - switch( m_method ) - { - case MIN: - { - return std::distance( m_tree.begin(), m_tree.find( m_lastval ) ); - } - case MAX: - { - auto end_range = m_tree.equal_range( m_lastval ).second; - return std::distance( m_tree.begin(), std::prev( end_range ) ); - } - case AVG: - { - auto range = m_tree.equal_range( m_lastval ); - return std::distance( m_tree.begin(), range.first ) + ( double )std::distance( range.first, std::prev( range.second ) ) / 2; - } - default: - break; - } - } - #endif - return std::numeric_limits::quiet_NaN(); } private: - - #ifndef __clang__ - ost> m_mintree; - ost> m_maxtree; - #else - std::multiset m_tree; - #endif + ost> m_mintree; + ost> m_maxtree; double m_lastval; int64_t m_method; @@ -1569,7 +1457,7 @@ class EMA m_ema = x; m_first = false; } - else if( unlikely( isnan( x ) ) && !m_ignore_na ) + else if( unlikely( isnan( x ) ) && !m_ignore_na && likely( !m_first ) ) { m_offset++; } @@ -1582,8 +1470,9 @@ class EMA } else { - m_ema = ( pow( m_ema * ( 1 - m_alpha ), m_offset ) + m_alpha * x ) / + m_ema = ( m_ema * pow( ( 1 - m_alpha ), m_offset ) + m_alpha * x ) / ( pow( 1 - m_alpha, m_offset ) + m_alpha ); + m_offset = 1; } } } @@ -1599,7 +1488,7 @@ class EMA double compute() const { - return m_ema; + return unlikely( m_first ) ? std::numeric_limits::quiet_NaN() : m_ema; } private: @@ -1710,7 +1599,7 @@ class AlphaDebiasEMA void add( double x ) { - if( m_first ) + if( m_first && likely( !isnan( x ) ) ) { m_wsum = 1; m_sqsum = 1; @@ -1728,10 +1617,16 @@ class AlphaDebiasEMA w0 = 1.0; else w0 = 1 - m_decay; - m_sqsum += pow( w0, 2 ); + m_sqsum += w0 * w0; m_wsum += w0; + if( !m_adjust ) + { + double correction = decay_factor + w0; + m_wsum /= correction; + m_sqsum /= ( correction * correction ); + } } - else + else if ( likely( !m_first ) ) { m_offset++; m_nan_count++; @@ -1746,7 +1641,7 @@ class AlphaDebiasEMA double wh = pow( m_decay, lookback ); if( !m_adjust ) wh *= ( 1- m_decay ); - m_sqsum -= pow( wh, 2 ); + m_sqsum -= wh * wh; m_wsum -= wh; if( m_wsum < EPSILON || m_sqsum < EPSILON ) { @@ -1767,7 +1662,7 @@ class AlphaDebiasEMA double compute() const { - double wsum_sq = pow( m_wsum, 2 ); + double wsum_sq = m_wsum * m_wsum; if( abs( wsum_sq - m_sqsum ) > EPSILON ) return wsum_sq / ( wsum_sq - m_sqsum ); else @@ -1794,7 +1689,7 @@ class HalflifeEMA HalflifeEMA( TimeDelta halflife, DateTime start ) { - m_halflife = halflife; + m_decay_factor = log( 0.5 ) / halflife.asNanoseconds(); m_last_tick = start; reset(); } @@ -1808,12 +1703,9 @@ class HalflifeEMA if( likely( !isnan( x ) ) ) { TimeDelta delta_t = now - m_last_tick; - double decay_duration = ( double )( delta_t.asNanoseconds() ) / ( m_halflife.asNanoseconds() ); - double decay = exp( -( decay_duration ) * log( 2 ) ); - m_ema *= decay; - m_norm *= decay; - m_ema += x; - m_norm++; + double decay = exp( m_decay_factor * delta_t.asNanoseconds() ); + m_ema = decay * m_ema + x; + m_norm = decay * m_norm + 1.0; m_last_tick = now; } } @@ -1832,7 +1724,7 @@ class HalflifeEMA double m_ema; double m_norm; - TimeDelta m_halflife; + double m_decay_factor; DateTime m_last_tick; }; @@ -1844,7 +1736,7 @@ class HalflifeDebiasEMA HalflifeDebiasEMA( TimeDelta halflife, DateTime start ) { - m_halflife = halflife; + m_decay_factor = log( 0.5 ) / halflife.asNanoseconds(); m_last_tick = start; reset(); } @@ -1858,11 +1750,9 @@ class HalflifeDebiasEMA if( likely( !isnan( x ) ) ) { TimeDelta delta_t = now - m_last_tick; - double decay = exp( -( delta_t/m_halflife ) * log( 2 ) ); - m_sqsum *= pow( decay, 2 ); - m_wsum *= decay; - m_sqsum++; - m_wsum++; + double decay = exp( m_decay_factor * delta_t.asNanoseconds() ); + m_sqsum = decay * decay * m_sqsum + 1.0; + m_wsum = decay * m_wsum + 1.0; m_last_tick = now; } } @@ -1874,7 +1764,7 @@ class HalflifeDebiasEMA double compute() const { - double wsum_sq = pow( m_wsum, 2 ); + double wsum_sq = m_wsum * m_wsum; if( wsum_sq != m_sqsum ) return wsum_sq / ( wsum_sq - m_sqsum ); else @@ -1885,7 +1775,7 @@ class HalflifeDebiasEMA double m_wsum; double m_sqsum; - TimeDelta m_halflife; + double m_decay_factor; DateTime m_last_tick; }; @@ -2061,11 +1951,6 @@ DECLARE_CPPNODE( _generic_tick_window_updates ) s_pending_recalc = true; } - if( csp.ticked( recalc ) ) - { - s_pending_recalc = true; - } - if( csp.ticked( sampler ) ) { // Handle removals if needed @@ -2096,8 +1981,10 @@ DECLARE_CPPNODE( _generic_tick_window_updates ) // Handle removals if( !s_removals.empty() && !s_pending_recalc ) + { std::swap( removals_.reserveSpace(), s_removals ); - s_removals.clear(); + s_removals.clear(); + } if( s_pending_recalc && !s_value_buffer.empty() ) { @@ -2186,9 +2073,12 @@ DECLARE_CPPNODE( _generic_time_window_updates ) STATE_VAR( bool, s_first{true} ); STATE_VAR( bool, s_expanding{false} ); + STATE_VAR( VariableSizeWindowBuffer, s_value_buffer{} ); STATE_VAR( VariableSizeWindowBuffer, s_time_buffer{} ); + STATE_VAR( std::vector, s_additions{} ); // only used for expanding window optimization + TS_NAMED_OUTPUT_RENAMED( std::vector, additions, additions_ ); TS_NAMED_OUTPUT_RENAMED( std::vector, removals, removals_ ); @@ -2213,9 +2103,14 @@ DECLARE_CPPNODE( _generic_time_window_updates ) { if( csp.ticked( reset ) ) { - s_value_buffer.clear(); - s_time_buffer.clear(); - s_pending_removals = 0; + if( s_expanding ) + s_additions.clear(); + else + { + s_value_buffer.clear(); + s_time_buffer.clear(); + s_pending_removals = 0; + } } if( csp.ticked( recalc ) ) @@ -2229,21 +2124,40 @@ DECLARE_CPPNODE( _generic_time_window_updates ) if( csp.ticked( x ) ) { node -> validateShape(); - s_value_buffer.push( x ); + if( s_expanding ) + s_additions.push_back( x ); + else + s_value_buffer.push( x ); } else { node -> checkValid(); - s_value_buffer.push( node -> createNan() ); + if( s_expanding ) + s_additions.push_back( node -> createNan() ); + else + s_value_buffer.push( node -> createNan() ); } - s_time_buffer.push( now() ); + if( !s_expanding ) + s_time_buffer.push( now() ); } if( csp.ticked( trigger ) || ( s_first && csp.ticked( x ) && csp.ticked( sampler ) ) ) { s_first = false; - DateTime threshold = ( s_expanding ? DateTime::MIN_VALUE() : now() - interval ); + if( s_expanding ) + { + // fast track expanding window calculations + // we just need to swap in all additions with no checks + if( !s_additions.empty() ) + { + std::swap( additions_.reserveSpace(), s_additions ); + s_additions.clear(); // keep allocated memory + } + return; + } + + DateTime threshold = now() - interval; // Handle removals int64_t count = csp.count( sampler ); std::vector* removals = nullptr; @@ -2334,3 +2248,5 @@ DECLARE_CPPNODE( _generic_cross_sectional ) }; } + +#endif // _IN_CSP_CPPNODES_STATSIMPL_H diff --git a/cpp/csp/engine/AdapterManager.h b/cpp/csp/engine/AdapterManager.h index a0c1531ee..fd71126c9 100644 --- a/cpp/csp/engine/AdapterManager.h +++ b/cpp/csp/engine/AdapterManager.h @@ -93,7 +93,7 @@ bool ManagedSimInputAdapter::pushNullTick() return true; } -class AdapterManager : public EngineOwned +class CSP_PUBLIC AdapterManager : public EngineOwned { public: AdapterManager( csp::Engine * ); diff --git a/cpp/csp/engine/CMakeLists.txt b/cpp/csp/engine/CMakeLists.txt index deb91fbe4..ce035b691 100644 --- a/cpp/csp/engine/CMakeLists.txt +++ b/cpp/csp/engine/CMakeLists.txt @@ -94,6 +94,6 @@ install(FILES ${CSP_TYPES_PUBLIC_HEADERS} ${ENGINE_PUBLIC_HEADERS} DESTINATION i install(TARGETS csp_types csp_engine PUBLIC_HEADER DESTINATION include/csp/engine - RUNTIME DESTINATION bin/ + RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR} LIBRARY DESTINATION lib/ ) diff --git a/cpp/csp/engine/CppNode.h b/cpp/csp/engine/CppNode.h index 554b0695a..f4d33f58e 100644 --- a/cpp/csp/engine/CppNode.h +++ b/cpp/csp/engine/CppNode.h @@ -1,6 +1,7 @@ #ifndef _IN_CSP_ENGINE_CPPNODE_H #define _IN_CSP_ENGINE_CPPNODE_H +#include #include #include #include @@ -672,7 +673,7 @@ public: \ #define STATE_VAR( Type, Name ) Type Name; -#define START() void start() override __attribute__((noinline)) +#define START() NO_INLINE void start() override #define STOP() void stop() override //wrap _executeImpl so we clear basket flags at the end of execute call diff --git a/cpp/csp/engine/CspEnum.cpp b/cpp/csp/engine/CspEnum.cpp index cb20ebc71..9bed122a8 100644 --- a/cpp/csp/engine/CspEnum.cpp +++ b/cpp/csp/engine/CspEnum.cpp @@ -14,11 +14,11 @@ CspEnumMeta::CspEnumMeta( const std::string & name, const ValueDef & def ) : m_n { for( auto [ key,value ] : def ) { - auto [rit, inserted] = m_instanceMap.emplace( value, CspEnumInstance( key, value, this ) ); + auto [rit, inserted] = m_instanceMap.emplace( value, std::make_shared( key, value, this ) ); if( !inserted ) CSP_THROW( TypeError, "CspEnum type " << name << " defined with multiple entries for " << value ); - m_mapping[ rit -> second.name().c_str() ] = rit; + m_mapping[ rit -> second -> name().c_str() ] = rit; } } diff --git a/cpp/csp/engine/CspEnum.h b/cpp/csp/engine/CspEnum.h index b33dcb04b..93f083369 100644 --- a/cpp/csp/engine/CspEnum.h +++ b/cpp/csp/engine/CspEnum.h @@ -76,7 +76,7 @@ class CspEnumMeta auto it = m_mapping.find( key ); if( it == m_mapping.end() ) CSP_THROW( ValueError, "Unrecognized enum name " << key << " for enum " << m_name ); - return CspEnum( &it -> second -> second ); + return CspEnum( it -> second -> second.get() ); } CspEnum create( int64_t value ) const @@ -84,12 +84,12 @@ class CspEnumMeta auto found = m_instanceMap.find( value ); if( found == m_instanceMap.end() ) CSP_THROW( RuntimeException, "Unrecognized value " << value << " for enum " << m_name ); - return CspEnum( &found -> second ); + return CspEnum( found -> second.get() ); } private: - using InstanceMapping = std::unordered_map; - using Mapping = std::unordered_map; + using InstanceMapping = std::unordered_map>; + using Mapping = std::unordered_map; std::string m_name; Mapping m_mapping; diff --git a/cpp/csp/engine/CspType.cpp b/cpp/csp/engine/CspType.cpp index c5d2599e7..884fc17cc 100644 --- a/cpp/csp/engine/CspType.cpp +++ b/cpp/csp/engine/CspType.cpp @@ -27,16 +27,19 @@ INIT_CSP_ENUM( CspType::Type, "DIALECT_GENERIC" ); -CspTypePtr & CspArrayType::create( const CspTypePtr & elemType ) +CspTypePtr & CspArrayType::create( const CspTypePtr & elemType, bool isPyStructFastList ) { - using Cache = std::unordered_map; + using Cache = std::unordered_map; static std::mutex s_mutex; static Cache s_cache; + static Cache s_pyStructFastListCache; + + auto & cache = isPyStructFastList ? s_pyStructFastListCache : s_cache; std::lock_guard guard( s_mutex ); - auto rv = s_cache.emplace( elemType.get(), nullptr ); + auto rv = cache.emplace( elemType.get(), nullptr ); if( rv.second ) - rv.first -> second = std::make_shared( elemType ); + rv.first -> second = std::make_shared( elemType, isPyStructFastList ); return rv.first -> second; } diff --git a/cpp/csp/engine/CspType.h b/cpp/csp/engine/CspType.h index 225dc958b..b85ab4148 100644 --- a/cpp/csp/engine/CspType.h +++ b/cpp/csp/engine/CspType.h @@ -115,7 +115,7 @@ class CspType Type m_type; }; -class CspStringType:public CspType +class CspStringType : public CspType { public: CspStringType(bool isBytes) @@ -128,7 +128,6 @@ class CspStringType:public CspType inline CspType::Ptr & CspType::STRING() { static CspType::Ptr s_type = std::make_shared( false ); return s_type; } inline CspType::Ptr & CspType::BYTES() { static CspType::Ptr s_type = std::make_shared( true ); return s_type; } - using CspTypePtr = CspType::Ptr; class CspEnum; @@ -171,18 +170,21 @@ class CspStructType : public CspType class CspArrayType : public CspType { public: - CspArrayType( CspTypePtr elemType ) : CspType( CspType::Type::ARRAY ), - m_elemType( elemType ) + CspArrayType( CspTypePtr elemType, bool isPyStructFastList = false ) : + CspType( CspType::Type::ARRAY ), m_elemType( elemType ), m_isPyStructFastList( isPyStructFastList ) {} + ~CspArrayType() {} const CspTypePtr & elemType() const { return m_elemType; } + bool isPyStructFastList() const { return m_isPyStructFastList; } //Used by BURST mode to avoid creating more instances of CspArrayTypes than needed //returns CspArrayType with the given elemType - static CspTypePtr & create( const CspTypePtr & elemType ); + static CspTypePtr & create( const CspTypePtr & elemType, bool isPyStructFastList = false ); private: CspTypePtr m_elemType; + bool m_isPyStructFastList; }; template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::BOOL; }; @@ -204,6 +206,7 @@ template<> struct CspType::TypeTraits::fromCType { sta template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::STRUCT; }; template struct CspType::TypeTraits::fromCType> { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::STRUCT; }; template<> struct CspType::TypeTraits::fromCType { static constexpr CspType::TypeTraits::_enum type = CspType::TypeTraits::DIALECT_GENERIC; }; + template struct CspType::TypeTraits::fromCType> { static_assert( !std::is_same::value, "vector should not be getting instantiated" ); diff --git a/cpp/csp/engine/DialectGenericType.h b/cpp/csp/engine/DialectGenericType.h index 0527be5af..0aa0dfad0 100644 --- a/cpp/csp/engine/DialectGenericType.h +++ b/cpp/csp/engine/DialectGenericType.h @@ -1,13 +1,15 @@ #ifndef _IN_CSP_CORE_DIALECTGENERICTYPE_H #define _IN_CSP_CORE_DIALECTGENERICTYPE_H +#include #include #include namespace csp { -struct DialectGenericType +// Note that this is intentionally exported with CSPTYPESIMPL_EXPORT because the actual impl is compiled into the dialect specific impl library +struct CSPTYPESIMPL_EXPORT DialectGenericType { public: DialectGenericType(); @@ -29,7 +31,7 @@ struct DialectGenericType [[maybe_unused]] void* m_data; }; -std::ostream & operator<<( std::ostream & o, const DialectGenericType & obj ); +CSPTYPESIMPL_EXPORT std::ostream & operator<<( std::ostream & o, const DialectGenericType & obj ); } diff --git a/cpp/csp/engine/Dictionary.h b/cpp/csp/engine/Dictionary.h index 66e49b184..1e7a2c82a 100644 --- a/cpp/csp/engine/Dictionary.h +++ b/cpp/csp/engine/Dictionary.h @@ -38,7 +38,7 @@ class Dictionary Value _data; }; private: - using Map = std::unordered_map; + using Map = std::unordered_map; using DataVector = std::vector>; public: diff --git a/cpp/csp/engine/Engine.h b/cpp/csp/engine/Engine.h index 9eb0e3ea4..ed8799625 100644 --- a/cpp/csp/engine/Engine.h +++ b/cpp/csp/engine/Engine.h @@ -29,6 +29,9 @@ struct EngineOwned //little trick here to force all derivations to fail void * operator new( size_t sz ) = delete; void * operator new( size_t sz, Engine * ) { return ::operator new( sz ); } + + void operator delete( void * ptr, Engine * ) { ::operator delete(ptr); } + void operator delete( void* ptr ) { ::operator delete(ptr); } }; class Engine diff --git a/cpp/csp/engine/Feedback.h b/cpp/csp/engine/Feedback.h index 0194d9257..ae02925aa 100644 --- a/cpp/csp/engine/Feedback.h +++ b/cpp/csp/engine/Feedback.h @@ -28,7 +28,7 @@ class FeedbackOutputAdapter final : public OutputAdapter }; template -class FeedbackInputAdapter final : public InputAdapter +class CSP_PUBLIC FeedbackInputAdapter final : public InputAdapter { public: using InputAdapter::InputAdapter; diff --git a/cpp/csp/engine/PartialSwitchCspType.h b/cpp/csp/engine/PartialSwitchCspType.h index 7a6d32e69..3765cd8b6 100644 --- a/cpp/csp/engine/PartialSwitchCspType.h +++ b/cpp/csp/engine/PartialSwitchCspType.h @@ -145,7 +145,7 @@ struct PartialSwitchCspType typename R_T, typename ArraySubTypeSwitchT, csp::CspType::Type::_enum T = CspType::Type::ARRAY, - std::enable_if_t(), int> = 1 > + std::enable_if_t(), void *> = nullptr > static R_T handleArrayType( const CspType *type, F &&f ) { const auto *arrayType = static_cast( type ); @@ -160,7 +160,7 @@ struct PartialSwitchCspType typename R_T, typename ArraySubTypeSwitchT, csp::CspType::Type::_enum T = CspType::Type::ARRAY, - std::enable_if_t(), int> = 0 > + std::enable_if_t(), void *> = nullptr > static R_T handleArrayType( const CspType *type, F &&f ) { CSP_THROW( UnsupportedSwitchType, "Unsupported type " << CspType::Type( T )); @@ -169,7 +169,7 @@ struct PartialSwitchCspType template< csp::CspType::Type::_enum T, typename F, typename R_T, - std::enable_if_t(), int> = 0 > + std::enable_if_t(), void *> = nullptr > static R_T handleType( const CspType *type, F &&f ) { return f( CspType::Type::toCType()); @@ -178,7 +178,7 @@ struct PartialSwitchCspType template< csp::CspType::Type::_enum T, typename F, typename R_T, - std::enable_if_t(), int> = 0 > + std::enable_if_t(), void *> = nullptr > static R_T handleType( const CspType *type, F &&f ) { CSP_THROW( UnsupportedSwitchType, "Unsupported type " << CspType::Type( T )); diff --git a/cpp/csp/engine/Profiler.h b/cpp/csp/engine/Profiler.h index 0d4629e71..6fb5dfee8 100644 --- a/cpp/csp/engine/Profiler.h +++ b/cpp/csp/engine/Profiler.h @@ -1,13 +1,14 @@ #ifndef _IN_CSP_ENGINE_PROFILER_H #define _IN_CSP_ENGINE_PROFILER_H +#include +#include +#include #include #include #include #include #include -#include -#include namespace csp { @@ -151,7 +152,7 @@ class Profiler struct ProfStat { - int64_t m_exec; + int64_t m_exec = 0; TimeDelta m_maxTime = TimeDelta::ZERO(); TimeDelta m_totalTime = TimeDelta::ZERO(); diff --git a/cpp/csp/engine/RootEngine.cpp b/cpp/csp/engine/RootEngine.cpp index 2ddc0eaca..cc0beca9b 100644 --- a/cpp/csp/engine/RootEngine.cpp +++ b/cpp/csp/engine/RootEngine.cpp @@ -18,19 +18,20 @@ The signal count variable is maintained to ensure that multiple engine threads s An interrupt should cause all running engines to stop, but should not affect future runs in the same process. Thus, each root engine keeps track of the signal count when its created. When an interrupt occurs, one engine thread handles the interrupt by incrementing the count. Then, all other root engines detect the signal by comparing their -initial count to the current count. +initial count to the current count. Future runs after the interrupt remain unaffected since they are initialized with the updated signal count, and will only consider themselves "interupted" if another signal is received during their execution. */ -static struct sigaction g_prevSIGTERMaction; +static void (*g_prevSIGTERMhandler)(int) = nullptr; static void handle_SIGTERM( int signum ) { g_SIGNAL_COUNT++; - if( g_prevSIGTERMaction.sa_handler ) - (*g_prevSIGTERMaction.sa_handler)( signum ); + + if(g_prevSIGTERMhandler) + (*g_prevSIGTERMhandler)( signum ); } static bool install_signal_handlers() @@ -42,14 +43,18 @@ static bool install_signal_handlers() std::lock_guard guard( s_lock ); if( !s_installed ) { - struct sigaction newhandler; +#ifndef WIN32 + struct sigaction newhandler, prev_handler; sigemptyset( &newhandler.sa_mask ); newhandler.sa_handler = handle_SIGTERM; newhandler.sa_flags = 0; - if( sigaction(SIGINT,&newhandler, &g_prevSIGTERMaction ) != 0 ) + if( sigaction(SIGINT,&newhandler, &prev_handler) != 0 ) printf( "Failed to set SIGTERM handler: %s", strerror( errno ) ); - + g_prevSIGTERMhandler = prev_handler.sa_handler; +#else + g_prevSIGTERMhandler = signal(SIGINT, handle_SIGTERM); +#endif s_installed = true; } } diff --git a/cpp/csp/engine/Struct.cpp b/cpp/csp/engine/Struct.cpp index 040c4745b..837d519b7 100644 --- a/cpp/csp/engine/Struct.cpp +++ b/cpp/csp/engine/Struct.cpp @@ -36,7 +36,7 @@ and adjustments required for the hidden fields StructMeta::StructMeta( const std::string & name, const Fields & fields, std::shared_ptr base ) : m_name( name ), m_base( base ), m_fields( fields ), m_size( 0 ), m_partialSize( 0 ), m_partialStart( 0 ), m_nativeStart( 0 ), m_basePadding( 0 ), - m_maskLoc( 0 ), m_maskSize( 0 ), m_firstPartialField( 0 ), + m_maskLoc( 0 ), m_maskSize( 0 ), m_firstPartialField( 0 ), m_firstNativePartialField( 0 ), m_isPartialNative( true ), m_isFullyNative( true ) { if( m_fields.empty() && !m_base) diff --git a/cpp/csp/engine/Struct.h b/cpp/csp/engine/Struct.h index 0cd83762f..1b09cb97b 100644 --- a/cpp/csp/engine/Struct.h +++ b/cpp/csp/engine/Struct.h @@ -313,7 +313,6 @@ class ArrayStructField : public NonNativeStructField { using ElemT = typename CType::value_type; - //template::type), bool> = true> template static std::enable_if_t::type), void> deepcopy( const std::vector & src, std::vector & dest ) { @@ -335,6 +334,14 @@ class ArrayStructField : public NonNativeStructField dest[i] = src[i].deepcopy(); } + template + static void deepcopy( const std::vector> & src, std::vector> & dest ) + { + dest.resize( src.size() ); + for( size_t i = 0; i < src.size(); ++i ) + deepcopy( src[i], dest[i] ); + } + public: ArrayStructField( CspTypePtr arrayType, const std::string & fieldname ) : NonNativeStructField( arrayType, fieldname, sizeof( CType ), alignof( CType ) ) @@ -735,7 +742,8 @@ class Struct private: static void * operator new( std::size_t count, const std::shared_ptr & meta ); - static void operator delete( void * ptr ); + static void operator delete( void * ptr, const std::shared_ptr & meta ) { delete( ( Struct * ) ptr ); } + static void operator delete( void * ptr ); Struct( const std::shared_ptr & meta ); ~Struct() @@ -748,11 +756,15 @@ class Struct void decref() { //Work around GCC12 bug mis-identifying this code as use-after-free -//#pragma GCC diagnostic push -//#pragma GCC diagnostic ignored "-Wuse-after-free" +#if defined(__linux__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuse-after-free" +#endif if( --hidden() -> refcount == 0 ) delete this; -//#pragma GCC diagnostic pop +#if defined(__linux__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif } diff --git a/cpp/csp/python/CMakeLists.txt b/cpp/csp/python/CMakeLists.txt index ae4328ada..d5f863e1a 100644 --- a/cpp/csp/python/CMakeLists.txt +++ b/cpp/csp/python/CMakeLists.txt @@ -5,7 +5,10 @@ set(CSPTYPESIMPL_PUBLIC_HEADERS PyCspType.h PyStruct.h PyStructList.h - PyStructList_impl.h) + PyStructList_impl.h + PyStructFastList.h + PyStructFastList_impl.h + VectorWrapper.h ) add_library(csptypesimpl csptypesimpl.cpp @@ -13,10 +16,14 @@ add_library(csptypesimpl PyCspEnum.cpp PyCspType.cpp PyStruct.cpp - PyStructToJson.cpp) + PyStructToJson.cpp + PyStructToDict.cpp) + set_target_properties(csptypesimpl PROPERTIES PUBLIC_HEADER "${CSPTYPESIMPL_PUBLIC_HEADERS}") target_compile_definitions(csptypesimpl PUBLIC RAPIDJSON_HAS_STDSTRING=1) target_link_libraries(csptypesimpl csp_core csp_types) +target_compile_definitions(csptypesimpl PRIVATE CSPTYPESIMPL_EXPORTS=1) + set(CSPIMPL_PUBLIC_HEADERS Common.h @@ -42,8 +49,7 @@ set(CSPIMPL_PUBLIC_HEADERS PyOutputProxy.h PyConstants.h PyStructToJson.h - PyStructList.h - PyStructList_impl.h) + PyStructToDict.h) add_library(cspimpl SHARED cspimpl.cpp @@ -78,8 +84,11 @@ add_library(cspimpl SHARED set_target_properties(cspimpl PROPERTIES PUBLIC_HEADER "${CSPIMPL_PUBLIC_HEADERS}") -target_link_libraries(cspimpl csptypesimpl csp_core csp_engine) +target_link_libraries(cspimpl csptypesimpl csp_core csp_engine ) + target_compile_definitions(cspimpl PUBLIC NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION) +target_compile_definitions(cspimpl PRIVATE CSPIMPL_EXPORTS=1) + ## Baselib c++ module add_library(cspbaselibimpl SHARED cspbaselibimpl.cpp) @@ -113,6 +122,6 @@ target_include_directories(cspnpstatsimpl PRIVATE ${NUMPY_INCLUDE_DIRS}) install(TARGETS csptypesimpl cspimpl cspbaselibimpl cspbasketlibimpl cspmathimpl cspstatsimpl csptestlibimpl cspnpstatsimpl PUBLIC_HEADER DESTINATION include/csp/python - RUNTIME DESTINATION bin/ + RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR} LIBRARY DESTINATION lib/ ) diff --git a/cpp/csp/python/Common.h b/cpp/csp/python/Common.h index a505e41ed..e8ecb0b1f 100644 --- a/cpp/csp/python/Common.h +++ b/cpp/csp/python/Common.h @@ -5,6 +5,8 @@ #define IS_PRE_PYTHON_3_11 (PY_MAJOR_VERSION < 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 11) ) +#define INIT_PYDATETIME if( !PyDateTimeAPI ) { PyDateTime_IMPORT; } + namespace csp::python { diff --git a/cpp/csp/python/Conversions.h b/cpp/csp/python/Conversions.h index 31d473be9..2422aaea1 100644 --- a/cpp/csp/python/Conversions.h +++ b/cpp/csp/python/Conversions.h @@ -1,20 +1,23 @@ #ifndef _IN_CSP_PYTHON_CONVERSIONS_H #define _IN_CSP_PYTHON_CONVERSIONS_H +#include #include #include #include #include #include +#include #include #include #include #include #include #include +#include #include -#include #include +#include #include #include @@ -80,7 +83,7 @@ inline T fromPython( PyObject * o ) return T{}; } -//work around for inabilty to partially spcialize fromPython on vector +//work around for inabilty to partially specialize fromPython on vector template struct FromPython { @@ -187,7 +190,7 @@ inline double fromPython( PyObject * o ) //allow ints as floats if( PyLong_Check( o ) ) { - int64_t rv = PyLong_AsLong( o ); + int64_t rv = PyLong_AsLongLong( o ); if( rv == -1 && PyErr_Occurred() ) CSP_THROW( PythonPassthrough, "" ); return rv; @@ -209,7 +212,7 @@ inline PyObject * toPython( const double & value ) template<> inline PyObject * toPython( const int64_t & value ) { - return toPythonCheck( PyLong_FromLong( value ) ); + return toPythonCheck( PyLong_FromLongLong( value ) ); } template<> @@ -218,7 +221,7 @@ inline int64_t fromPython( PyObject * o ) if( !PyLong_Check( o ) ) CSP_THROW( TypeError, "Invalid int type, expected long (int) got " << Py_TYPE( o ) -> tp_name ); - int64_t rv = PyLong_AsLong( o ); + int64_t rv = PyLong_AsLongLong( o ); if( rv == -1 && PyErr_Occurred() ) CSP_THROW( PythonPassthrough, "" ); return rv; @@ -229,7 +232,7 @@ inline int64_t fromPython( PyObject * o ) template<> inline PyObject * toPython( const uint64_t & value ) { - return toPythonCheck( PyLong_FromUnsignedLong( value ) ); + return toPythonCheck( PyLong_FromUnsignedLongLong( value ) ); } template<> @@ -238,7 +241,7 @@ inline uint64_t fromPython( PyObject * o ) if( !PyLong_Check( o ) ) CSP_THROW( TypeError, "Invalid int type, expected long (int) got " << Py_TYPE( o ) -> tp_name ); - uint64_t rv = PyLong_AsUnsignedLong( o ); + uint64_t rv = PyLong_AsUnsignedLongLong( o ); if( rv == ( uint64_t ) -1 && PyErr_Occurred() ) CSP_THROW( PythonPassthrough, "" ); return rv; @@ -465,6 +468,8 @@ inline CspEnum fromPython( PyObject * o, const CspType & type ) template<> inline PyObject * toPython( const TimeDelta & td ) { + INIT_PYDATETIME; + if( td == TimeDelta::NONE() ) Py_RETURN_NONE; @@ -474,6 +479,8 @@ inline PyObject * toPython( const TimeDelta & td ) template<> inline TimeDelta fromPython( PyObject * o ) { + INIT_PYDATETIME; + if( o == Py_None ) return TimeDelta::NONE(); @@ -499,6 +506,8 @@ inline TimeDelta fromPython( PyObject * o ) template<> inline PyObject * toPython( const Date & d ) { + INIT_PYDATETIME; + if ( d == Date::NONE()) Py_RETURN_NONE; @@ -508,6 +517,8 @@ inline PyObject * toPython( const Date & d ) template<> inline Date fromPython( PyObject * o ) { + INIT_PYDATETIME; + if( o == Py_None ) return Date::NONE(); @@ -525,6 +536,8 @@ inline Date fromPython( PyObject * o ) template<> inline PyObject * toPython( const Time & t ) { + INIT_PYDATETIME; + if( t == Time::NONE()) Py_RETURN_NONE; @@ -534,6 +547,8 @@ inline PyObject * toPython( const Time & t ) template<> inline Time fromPython( PyObject * o ) { + INIT_PYDATETIME; + if( o == Py_None ) return Time::NONE(); @@ -555,6 +570,8 @@ inline Time fromPython( PyObject * o ) template<> inline PyObject * toPython( const DateTime & dt ) { + INIT_PYDATETIME; + DateTimeEx dtEx( dt ); return toPythonCheck( PyDateTime_FromDateAndTime( dtEx.year(), dtEx.month(), dtEx.day(), dtEx.hour(), dtEx.minute(), dtEx.second(), dtEx.microseconds() ) ); } @@ -562,6 +579,8 @@ inline PyObject * toPython( const DateTime & dt ) template<> inline DateTime fromPython( PyObject * o ) { + INIT_PYDATETIME; + if( o == Py_None ) return DateTime::NONE(); @@ -609,6 +628,8 @@ inline DateTime fromPython( PyObject * o ) template<> inline DateTimeOrTimeDelta fromPython( PyObject * o ) { + INIT_PYDATETIME; + if( PyDateTime_Check( o ) ) return fromPython( o ); @@ -666,6 +687,8 @@ inline std::vector fromPython( PyObject * o ) template<> inline Dictionary::Value fromPython( PyObject * o ) { + INIT_PYDATETIME; + if( PyBool_Check( o ) ) return fromPython( o ); else if( PyLong_Check( o ) ) @@ -747,11 +770,11 @@ inline PyObject * toPython( const DictionaryPtr & value) } template -inline PyObject * toPython( const std::vector & v, const CspType & type ) +inline PyObject * toPython( const std::vector & v, const CspType & arrayType ) { - assert( type.type() == CspType::Type::ARRAY ); + assert( arrayType.type() == CspType::Type::ARRAY ); - const CspType & elemType = *static_cast( type ).elemType(); + const CspType & elemType = *static_cast( arrayType ).elemType(); size_t size = v.size(); PyObjectPtr list = PyObjectPtr::check( PyList_New( size ) ); @@ -764,18 +787,27 @@ inline PyObject * toPython( const std::vector & v, const CspType & typ } template -inline PyObject * toPython( const std::vector & v, const CspType & type, const PyStruct * pystruct ) +inline PyObject * toPython( const std::vector & v, const CspType & arrayType, const PyStruct * pystruct ) { - assert( type.type() == CspType::Type::ARRAY ); + assert( arrayType.type() == CspType::Type::ARRAY ); - const CspTypePtr elemType = static_cast( type ).elemType(); + const CspArrayType & cspArrayType = static_cast( arrayType ); + const CspTypePtr elemType = cspArrayType.elemType(); using ElemT = typename CspType::Type::toCArrayElemType::type; size_t sz = v.size(); + // Create PyStructFastList when requested + if( cspArrayType.isPyStructFastList() ) + { + PyObject * fl = PyStructFastList::PyType.tp_alloc( &PyStructFastList::PyType, 0 ); + new ( fl ) PyStructFastList( const_cast( pystruct ), const_cast &>( v ), cspArrayType ); + return fl; + } + // Create PyStructList otherwise // TODO: Implement more efficient list allocation by pre-allocating the space and filling it using PyList_SET_ITEM. // As of now, the problem is that Python is not allowing to resize the list via API, and it cannot allocate the list at the base of PyStructList, it can only allocate it somewhere in memory not under control. PyObject * psl = PyStructList::PyType.tp_alloc( &PyStructList::PyType, 0 ); - new ( psl ) PyStructList( const_cast( pystruct ), const_cast &>( v ), *elemType ); + new ( psl ) PyStructList( const_cast( pystruct ), const_cast &>( v ), cspArrayType ); for( size_t index = 0; index < sz; ++index ) { @@ -789,15 +821,16 @@ inline PyObject * toPython( const std::vector & v, const CspType & typ template struct FromPython> { - static std::vector impl( PyObject * o, const CspType & type ) + static std::vector impl( PyObject * o, const CspType & arrayType ) { - assert( type.type() == CspType::Type::ARRAY ); - const CspType & elemType = *static_cast( type ).elemType(); + assert( arrayType.type() == CspType::Type::ARRAY ); using ElemT = typename CspType::Type::toCArrayElemType::type; + const CspType & elemType = *static_cast( arrayType ).elemType(); + std::vector out; - //fast path list and tuples since we can size up front + //fast path for list and tuple since we can size up front if( PyList_Check( o ) ) { size_t size = PyList_GET_SIZE( o ); @@ -841,15 +874,6 @@ struct FromPython> PyObject * lastValueToPython( const csp::TimeSeriesProvider * ts ); PyObject * valueAtIndexToPython( const csp::TimeSeriesProvider * ts, int32_t index ); -static bool _initPyDateTimeAPI() -{ - PyDateTime_IMPORT; - return true; -} - -//init datetime API once up front for this cpp object -static bool _initDateTimeAPI = _initPyDateTimeAPI(); - } #endif diff --git a/cpp/csp/python/CspTypeFactory.cpp b/cpp/csp/python/CspTypeFactory.cpp index ef0f0dc78..a1bfd0273 100644 --- a/cpp/csp/python/CspTypeFactory.cpp +++ b/cpp/csp/python/CspTypeFactory.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -14,17 +15,29 @@ CspTypeFactory & CspTypeFactory::instance() CspTypePtr & CspTypeFactory::typeFromPyType( PyObject * pyTypeObj ) { + INIT_PYDATETIME; + // List objects shouldn't be cached since they are temporary objects if( PyList_Check( pyTypeObj ) ) { - if( PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) != 1 ) - CSP_THROW( TypeError, "Expected list types to be single element of sub-type" ); + if( PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) != 1 && PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) != 2 ) + CSP_THROW( TypeError, "Expected list types post-normalization to be one or two elements: sub-type and optional FastList flag" ); PyObject *pySubType = PyList_GET_ITEM( pyTypeObj, 0 ); if( !PyType_Check( pySubType ) ) CSP_THROW( TypeError, "nested typed lists are not supported" ); + + bool useFastList = false; + if( PyList_GET_SIZE( ( PyObject * ) pyTypeObj ) == 2 ) + { + PyObject *pyUseFastList = PyList_GET_ITEM( pyTypeObj, 1 ); + if( !PyBool_Check( pyUseFastList ) || pyUseFastList != Py_True ) + CSP_THROW( TypeError, "expected bool True as second list type argument" ); + useFastList = true; + } + CspTypePtr elemType = typeFromPyType( pySubType ); - return CspArrayType::create( elemType ); + return CspArrayType::create( elemType, useFastList ); } PyTypeObject *pyType = (PyTypeObject*) pyTypeObj; diff --git a/cpp/csp/python/CspTypeFactory.h b/cpp/csp/python/CspTypeFactory.h index b95f30d92..dd44d06b7 100644 --- a/cpp/csp/python/CspTypeFactory.h +++ b/cpp/csp/python/CspTypeFactory.h @@ -1,6 +1,7 @@ #ifndef _IN_CSP_PYTHON_CSPTYPEFACTORY_H #define _IN_CSP_PYTHON_CSPTYPEFACTORY_H +#include #include #include #include @@ -8,7 +9,7 @@ namespace csp::python { -class CspTypeFactory +class CSPTYPESIMPL_EXPORT CspTypeFactory { public: static CspTypeFactory & instance(); diff --git a/cpp/csp/python/Exception.h b/cpp/csp/python/Exception.h index 104a3509a..6040c79a2 100644 --- a/cpp/csp/python/Exception.h +++ b/cpp/csp/python/Exception.h @@ -8,7 +8,7 @@ namespace csp::python { -class PythonPassthrough : public csp::Exception +class CSP_PUBLIC PythonPassthrough : public csp::Exception { public: PythonPassthrough( const char * exType, const std::string &r, const char * file, @@ -16,9 +16,20 @@ class PythonPassthrough : public csp::Exception csp::Exception( exType, r, file, func, line ) { //Fetch the current error to clear out the error indicator while the stack gets unwound + //We own the references to all the members assigned in PyErr_Fetch + //We need to hold the reference since PyErr_Restore takes back a reference to each of its arguments PyErr_Fetch( &m_type, &m_value, &m_traceback ); } + PythonPassthrough( PyObject * pyException ) : + csp::Exception( "", "" ) + { + // Note: all of these methods return strong references, so we own them like in the other constructor + m_type = PyObject_Type( pyException ); + m_value = PyObject_Str( pyException ); + m_traceback = PyException_GetTraceback( pyException ); + } + void restore() { if( !description().empty() ) @@ -39,7 +50,6 @@ class PythonPassthrough : public csp::Exception PyObject * m_type; PyObject * m_value; PyObject * m_traceback; - }; CSP_DECLARE_EXCEPTION( AttributeError, ::csp::Exception ); diff --git a/cpp/csp/python/InitHelper.h b/cpp/csp/python/InitHelper.h index 5df1ed956..1665f847f 100644 --- a/cpp/csp/python/InitHelper.h +++ b/cpp/csp/python/InitHelper.h @@ -1,6 +1,7 @@ #ifndef _IN_CSP_PYTHON_INITHELPER_H #define _IN_CSP_PYTHON_INITHELPER_H +#include #include #include #include @@ -9,7 +10,7 @@ namespace csp::python { -class __attribute__ ((visibility ("hidden"))) InitHelper +class CSP_LOCAL InitHelper { public: ~InitHelper() {} @@ -110,4 +111,21 @@ inline bool InitHelper::execute( PyObject * module ) } } + +//PyMODINIT_FUNC in Python <3.9 doesn't export the function/make visible +//this is required since we build with hidden visibility by default +//the below macro code can be removed once 3.8 support is dropped +// +//see similar issues: +//https://github.com/scipy/scipy/issues/15996 +//https://github.com/mesonbuild/meson/pull/10369 + +#if PY_VERSION_HEX < 0x03090000 +#ifdef PyMODINIT_FUNC +#undef PyMODINIT_FUNC +#endif + +#define PyMODINIT_FUNC extern "C" CSP_PUBLIC PyObject* +#endif + #endif diff --git a/cpp/csp/python/NumpyConversions.cpp b/cpp/csp/python/NumpyConversions.cpp index 0ed3191f9..982544038 100644 --- a/cpp/csp/python/NumpyConversions.cpp +++ b/cpp/csp/python/NumpyConversions.cpp @@ -64,7 +64,7 @@ NPY_DATETIMEUNIT datetimeUnitFromDescr( PyArray_Descr* descr ) return dtMeta -> base; } -static std::wstring_convert, wchar_t> wstr_converter; +static std::wstring_convert, char32_t> wstr_converter; void stringFromNumpyStr( void* data, std::string& out, char numpy_type, int elem_size_bytes ) { @@ -73,17 +73,17 @@ void stringFromNumpyStr( void* data, std::string& out, char numpy_type, int elem if( numpy_type == NPY_UNICODELTR) { - const wchar_t * const raw_value = (const wchar_t *) data; - const int field_size = elem_size_bytes / __SIZEOF_WCHAR_T__; + const char32_t* const raw_value = (const char32_t *) data; + const int field_size = elem_size_bytes / sizeof( char32_t ); if( raw_value[field_size - 1] == 0 ) { - std::wstring wstr( raw_value ); + std::u32string wstr( raw_value ); out = wstr_converter.to_bytes( wstr ); } else { - std::wstring wstr( raw_value, field_size ); + std::u32string wstr( raw_value, field_size ); out = wstr_converter.to_bytes( wstr ); } } diff --git a/cpp/csp/python/NumpyConversions.h b/cpp/csp/python/NumpyConversions.h index fe119024b..04ce787b8 100644 --- a/cpp/csp/python/NumpyConversions.h +++ b/cpp/csp/python/NumpyConversions.h @@ -197,7 +197,14 @@ inline PyObject * createNumpyArray( ValueType valueType, const csp::TimeSeriesPr endIndex < ts -> numTicks() && ts -> timeAtIndex( endIndex ) < endDt; - T lastValue = ( ts -> valid() ? ts -> lastValueTyped() : T() ); + //MSVC bug!! They dont eval ternary operators correctly and this code actually evaluates both sides + //which leads to a crash since lastValueTyped should not be called at all if ts is not valid... + //T lastValue = ( ts -> valid() ? ts -> lastValueTyped() : T() ); + + T lastValue; + if( ts -> valid() ) + lastValue = ts -> lastValueTyped(); + DateTime lastTime = ( ts -> valid() ? ts -> lastTime() : DateTime() ); switch( valueType ) { @@ -233,7 +240,6 @@ void stringFromNumpyStr( void* data, std::string& out, char numpy_type, int elem void validateNumpyTypeVsCspType( const CspTypePtr & type, char numpy_type_char ); - } #endif diff --git a/cpp/csp/python/NumpyInputAdapter.h b/cpp/csp/python/NumpyInputAdapter.h index 55ce7d2b8..0d4bd7c8b 100644 --- a/cpp/csp/python/NumpyInputAdapter.h +++ b/cpp/csp/python/NumpyInputAdapter.h @@ -286,7 +286,8 @@ void NumpyInputAdapter::setValue( int64_t & value, void * data ) } case NPY_LONGLTR: { - value = *(int64_t *)(data); + const long * const val = (const long*)data; + value = static_cast(*val); break; } default: diff --git a/cpp/csp/python/PyAdapterManager.cpp b/cpp/csp/python/PyAdapterManager.cpp index 3a9ec7211..d6547e135 100644 --- a/cpp/csp/python/PyAdapterManager.cpp +++ b/cpp/csp/python/PyAdapterManager.cpp @@ -68,6 +68,15 @@ class PyAdapterManager : public AdapterManager static PyObject * PyAdapterManager_PyObject_starttime( PyAdapterManager_PyObject * self ) { return toPython( self -> manager -> starttime() ); } static PyObject * PyAdapterManager_PyObject_endtime( PyAdapterManager_PyObject * self ) { return toPython( self -> manager -> endtime() ); } +static PyObject * PyAdapterManager_PyObject_shutdown_engine( PyAdapterManager_PyObject * self, PyObject * pyException ) +{ + CSP_BEGIN_METHOD; + + self -> manager -> rootEngine() -> shutdown( PyEngine_shutdown_make_exception( pyException ) ); + + CSP_RETURN_NONE; +} + static int PyAdapterManager_init( PyAdapterManager_PyObject *self, PyObject *args, PyObject *kwds ) { CSP_BEGIN_METHOD; @@ -83,8 +92,9 @@ static int PyAdapterManager_init( PyAdapterManager_PyObject *self, PyObject *arg } static PyMethodDef PyAdapterManager_methods[] = { - { "starttime", (PyCFunction) PyAdapterManager_PyObject_starttime, METH_NOARGS, "starttime" }, - { "endtime", (PyCFunction) PyAdapterManager_PyObject_endtime, METH_NOARGS, "endtime" }, + { "starttime", (PyCFunction) PyAdapterManager_PyObject_starttime, METH_NOARGS, "starttime" }, + { "endtime", (PyCFunction) PyAdapterManager_PyObject_endtime, METH_NOARGS, "endtime" }, + { "shutdown_engine", (PyCFunction) PyAdapterManager_PyObject_shutdown_engine, METH_O, "shutdown_engine" }, {NULL} }; diff --git a/cpp/csp/python/PyAdapterManagerWrapper.h b/cpp/csp/python/PyAdapterManagerWrapper.h index 4cce8f854..9503f0473 100644 --- a/cpp/csp/python/PyAdapterManagerWrapper.h +++ b/cpp/csp/python/PyAdapterManagerWrapper.h @@ -12,7 +12,7 @@ namespace csp::python class PyEngine; -class PyAdapterManagerWrapper +class CSPIMPL_EXPORT PyAdapterManagerWrapper { public: using Creator = std::function; diff --git a/cpp/csp/python/PyCppNode.h b/cpp/csp/python/PyCppNode.h index 5e6e88d40..0ff6f1acc 100644 --- a/cpp/csp/python/PyCppNode.h +++ b/cpp/csp/python/PyCppNode.h @@ -19,7 +19,7 @@ in c++, we have macros to fwd declare the creator methods that create the C++ no and relies on linking in the c+ node library properly. */ -PyObject * pycppnode_create( PyObject * args, csp::CppNode::Creator creatorFn ); +CSPIMPL_EXPORT PyObject * pycppnode_create( PyObject * args, csp::CppNode::Creator creatorFn ); #define _REGISTER_CPPNODE( Name, Creator ) \ static PyObject * Name##_cppnode_create( PyObject * module, PyObject * args ) \ diff --git a/cpp/csp/python/PyCspEnum.h b/cpp/csp/python/PyCspEnum.h index 2f3239e08..fb098735c 100644 --- a/cpp/csp/python/PyCspEnum.h +++ b/cpp/csp/python/PyCspEnum.h @@ -1,6 +1,7 @@ #ifndef _IN_CSP_PYTHON_CSPENUM_H #define _IN_CSP_PYTHON_CSPENUM_H +#include #include #include #include @@ -10,7 +11,7 @@ namespace csp::python { //This is the base class of csp.Enum -struct PyCspEnumMeta : public PyHeapTypeObject +struct CSPTYPESIMPL_EXPORT PyCspEnumMeta : public PyHeapTypeObject { //convert to PyObject ( new ref ) PyObject * toPyEnum( CspEnum e ) const; @@ -26,9 +27,11 @@ struct PyCspEnumMeta : public PyHeapTypeObject static PyTypeObject PyType; }; +//TODO Windows - need to figure out why adding CSP_PUBLIC to this class leads to weird compilation errors on CspEnumMeta's unordered_map... + //This is an extension of csp::CspEnumMeta for python dialect, we need it in order to //keep a reference to the python enum type from conversion to/from csp::CspEnumMeta <-> PyObject properly -class DialectCspEnumMeta : public CspEnumMeta +class CSPTYPESIMPL_EXPORT DialectCspEnumMeta : public CspEnumMeta { public: DialectCspEnumMeta( PyTypeObjectPtr pyType, const std::string & name, @@ -44,7 +47,7 @@ class DialectCspEnumMeta : public CspEnumMeta PyTypeObjectPtr m_pyType; }; -struct PyCspEnum : public PyObject +struct CSPTYPESIMPL_EXPORT PyCspEnum : public PyObject { PyCspEnum( const CspEnum & e ) : enum_( e ) {} ~PyCspEnum() {} diff --git a/cpp/csp/python/PyCspType.h b/cpp/csp/python/PyCspType.h index bb53ac3c7..d972148eb 100644 --- a/cpp/csp/python/PyCspType.h +++ b/cpp/csp/python/PyCspType.h @@ -1,6 +1,7 @@ #ifndef _IN_CSP_PYTHON_PYCSPTYPE_H #define _IN_CSP_PYTHON_PYCSPTYPE_H +#include #include #include diff --git a/cpp/csp/python/PyEngine.h b/cpp/csp/python/PyEngine.h index fd2013cc3..d58289501 100644 --- a/cpp/csp/python/PyEngine.h +++ b/cpp/csp/python/PyEngine.h @@ -2,6 +2,7 @@ #define _IN_CSP_PYTHON_PYENGINE_H #include +#include #include #include #include @@ -32,7 +33,7 @@ class PythonEngine final : public csp::RootEngine }; //This is the root engine wrapper object -class PyEngine final: public PyObject +class CSPIMPL_EXPORT PyEngine final: public PyObject { public: PyEngine( const Dictionary & settings ); @@ -54,6 +55,21 @@ class PyEngine final: public PyObject Engine * m_engine; }; +inline std::exception_ptr PyEngine_shutdown_make_exception( PyObject * pyException ) +{ + if( !PyExceptionInstance_Check( pyException ) ) + { + PyObjectPtr pyExceptionStr = PyObjectPtr::own( PyObject_Str( pyException ) ); + if( !pyExceptionStr.ptr() ) + CSP_THROW( PythonPassthrough, "" ); + std::string pyExceptionString = PyUnicode_AsUTF8( pyExceptionStr.ptr() ); + std::string desc = "Expected Exception object as argument for shutdown_engine: got " + pyExceptionString + " of type " + Py_TYPE( pyException ) -> tp_name; + return std::make_exception_ptr( csp::Exception( "TypeError", desc ) ); + } + else + return std::make_exception_ptr( PythonPassthrough( pyException ) ); +} + }; #endif diff --git a/cpp/csp/python/PyInputAdapterWrapper.h b/cpp/csp/python/PyInputAdapterWrapper.h index e6a1971b0..686cc75d9 100644 --- a/cpp/csp/python/PyInputAdapterWrapper.h +++ b/cpp/csp/python/PyInputAdapterWrapper.h @@ -13,7 +13,7 @@ namespace csp::python class PyEngine; -class PyInputAdapterWrapper : public PyObject +class CSPIMPL_EXPORT PyInputAdapterWrapper : public PyObject { public: PyInputAdapterWrapper( InputAdapter * adapter ) : m_adapter( adapter ) diff --git a/cpp/csp/python/PyObjectPtr.h b/cpp/csp/python/PyObjectPtr.h index 0fa66bff8..448b10f34 100644 --- a/cpp/csp/python/PyObjectPtr.h +++ b/cpp/csp/python/PyObjectPtr.h @@ -38,19 +38,11 @@ class PyPtr return *this; } - bool operator==( const PyPtr & rhs ) const - { - if( m_obj == rhs.m_obj ) - return true; + bool operator==( const PyPtr & rhs ) const { return generic_compare( rhs, Py_EQ ); } - if( !m_obj || !rhs.m_obj ) - return false; + bool operator<( const PyPtr & rhs ) const { return generic_compare( rhs, Py_LT ); } - int rv = PyObject_RichCompareBool( m_obj, rhs.m_obj, Py_EQ ); - if( rv == -1 ) - CSP_THROW( PythonPassthrough, "" ); - return rv == 1; - } + bool operator>( const PyPtr & rhs ) const { return generic_compare( rhs, Py_GT ); } operator bool() const { return m_obj != nullptr; } @@ -101,6 +93,17 @@ class PyPtr PyPtr( PyObjectOwn * o ) { m_obj = ( PYOBJECT_T * ) o; } PYOBJECT_T * m_obj; + + bool generic_compare( const PyPtr & rhs, int opid ) const + { + if( !m_obj || !rhs.m_obj ) + CSP_THROW( PythonPassthrough, "" ); + + int rv = PyObject_RichCompareBool( m_obj, rhs.m_obj, opid ); + if( rv == -1 ) + CSP_THROW( PythonPassthrough, "" ); + return rv; + } }; using PyObjectPtr = PyPtr; diff --git a/cpp/csp/python/PyOutputAdapterWrapper.h b/cpp/csp/python/PyOutputAdapterWrapper.h index 29bdcf98c..f2a1b0bb8 100644 --- a/cpp/csp/python/PyOutputAdapterWrapper.h +++ b/cpp/csp/python/PyOutputAdapterWrapper.h @@ -12,7 +12,7 @@ namespace csp::python class PyEngine; -class PyOutputAdapterWrapper final: public PyObject +class CSPIMPL_EXPORT PyOutputAdapterWrapper final: public PyObject { public: PyOutputAdapterWrapper( OutputAdapter * adapter ) : m_adapter( adapter ) diff --git a/cpp/csp/python/PyPushInputAdapter.cpp b/cpp/csp/python/PyPushInputAdapter.cpp index 3b5f91b07..cd034844a 100644 --- a/cpp/csp/python/PyPushInputAdapter.cpp +++ b/cpp/csp/python/PyPushInputAdapter.cpp @@ -198,11 +198,21 @@ struct PyPushInputAdapter_PyObject CSP_RETURN_NONE; } + static PyObject * shutdown_engine( PyPushInputAdapter_PyObject * self, PyObject * pyException ) + { + CSP_BEGIN_METHOD; + + self -> adapter -> rootEngine() -> shutdown( PyEngine_shutdown_make_exception( pyException ) ); + + CSP_RETURN_NONE; + } + static PyTypeObject PyType; }; static PyMethodDef PyPushInputAdapter_PyObject_methods[] = { - { "push_tick", (PyCFunction) PyPushInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, + { "push_tick", (PyCFunction) PyPushInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, + { "shutdown_engine", (PyCFunction) PyPushInputAdapter_PyObject::shutdown_engine, METH_O, "shutdown_engine" }, {NULL} }; diff --git a/cpp/csp/python/PyPushPullInputAdapter.cpp b/cpp/csp/python/PyPushPullInputAdapter.cpp index d53ed972d..71190660c 100644 --- a/cpp/csp/python/PyPushPullInputAdapter.cpp +++ b/cpp/csp/python/PyPushPullInputAdapter.cpp @@ -119,12 +119,22 @@ struct PyPushPullInputAdapter_PyObject CSP_RETURN_NONE; } + static PyObject * shutdown_engine( PyPushPullInputAdapter_PyObject * self, PyObject * pyException ) + { + CSP_BEGIN_METHOD; + + self -> adapter -> rootEngine() -> shutdown( PyEngine_shutdown_make_exception( pyException ) ); + + CSP_RETURN_NONE; + } + static PyTypeObject PyType; }; static PyMethodDef PyPushPullInputAdapter_PyObject_methods[] = { - { "push_tick", (PyCFunction) PyPushPullInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, - { "flag_replay_complete", (PyCFunction) PyPushPullInputAdapter_PyObject::flagReplayComplete, METH_VARARGS, "finish replay ticks" }, + { "push_tick", (PyCFunction) PyPushPullInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, + { "flag_replay_complete", (PyCFunction) PyPushPullInputAdapter_PyObject::flagReplayComplete, METH_VARARGS, "finish replay ticks" }, + { "shutdown_engine", (PyCFunction) PyPushPullInputAdapter_PyObject::shutdown_engine, METH_O, "shutdown engine" }, {NULL} }; diff --git a/cpp/csp/python/PyStruct.cpp b/cpp/csp/python/PyStruct.cpp index 63e65a26d..24df48c6e 100644 --- a/cpp/csp/python/PyStruct.cpp +++ b/cpp/csp/python/PyStruct.cpp @@ -3,8 +3,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -457,8 +459,11 @@ void PyStruct::setattr( Struct * s, PyObject * attr, PyObject * value ) { switchCspType( field -> type(), [field,&struct_=s,value]( auto tag ) { - using CType = typename decltype(tag)::type; - auto *typedField = static_cast::type *>( field ); + //workaround for MS compiler bug, separate into two using lines... :/ + using TagType = decltype( tag ); + using CType = typename TagType::type; + using fieldType = typename StructField::upcast::type; + auto *typedField = static_cast( field ); if( value ) typedField -> setValue( struct_, fromPython( value, *field -> type() ) ); @@ -474,12 +479,9 @@ void PyStruct::setattr( Struct * s, PyObject * attr, PyObject * value ) // Struct printing code -// forward declarations +// forward declaration void repr_struct( const Struct * struct_, std::string & tl_repr, bool show_unset ); -template -void repr_array( const std::vector & val, const CspArrayType & arrayType, std::string & tl_repr, bool show_unset ); - // helper functions for formatting to Python standard void format_bool( const bool val, std::string & tl_repr ) { tl_repr += ( ( val ? "True" : "False" ) ); } void format_double( const double val, std::string & tl_repr ) @@ -552,12 +554,14 @@ void repr_field( const Struct * struct_, const StructFieldPtr & field, std::stri auto const * arrayType = static_cast( field -> type().get() ); const CspType * elemType = arrayType -> elemType().get(); - switchCspType( elemType, [ field, struct_, &arrayType, &tl_repr, show_unset ]( auto tag ) + switchCspType( elemType, [ field, struct_, &elemType, &tl_repr, show_unset ]( auto tag ) { - using CElemType = typename decltype( tag )::type; + //workaround for MS compiler bug, separate into two using lines... :/ + using TagType = decltype( tag ); + using CElemType = typename TagType::type; using ArrayType = typename CspType::Type::toCArrayType::type; const ArrayType & val = field -> value( struct_ ); - repr_array( val, *arrayType, tl_repr, show_unset ); + repr_array( val, *elemType, tl_repr, show_unset ); } ); break; @@ -580,7 +584,7 @@ void repr_field( const Struct * struct_, const StructFieldPtr & field, std::stri } template -void repr_array( const std::vector & val, const CspArrayType & arrayType, std::string & tl_repr, bool show_unset ) +void repr_array( const std::vector & val, const CspType & elemType, std::string & tl_repr, bool show_unset ) { using ElemT = typename CspType::Type::toCArrayElemType::type; tl_repr += "["; @@ -606,11 +610,11 @@ void repr_array( const std::vector & val, const CspArrayType & arrayTy else if constexpr( std::is_integral::value ) tl_repr += std::to_string( *it ); else if constexpr( is_vector::value ) - repr_array( *it, static_cast( arrayType.elemType() ), tl_repr, show_unset ); // recursive, allows for nested arrays! + repr_array( *it, elemType, tl_repr, show_unset ); // recursive, allows for nested arrays! else { // if the element is an enum, generic or datetime type, convert to python - PyObjectPtr attr = PyObjectPtr::own( toPython( *it, *( arrayType.elemType().get() ) ) ); + PyObjectPtr attr = PyObjectPtr::own( toPython( *it, elemType ) ); format_pyobject( attr, tl_repr ); } } @@ -933,6 +937,28 @@ PyObject * PyStruct_all_fields_set( PyStruct * self ) return toPython( self -> struct_ -> allFieldsSet() ); } +PyObject * PyStruct_to_dict( PyStruct * self, PyObject * args, PyObject * kwargs ) +{ + CSP_BEGIN_METHOD; + + // NOTE: Consider grouping customization properties into a dictionary + PyObject * callable = nullptr; + if( PyArg_ParseTuple( args, "O:to_dict", &callable ) ) + { + if( ( callable != Py_None ) && !PyCallable_Check( callable ) ) + { + CSP_THROW( TypeError, "Parameter must be callable or None got " + std::string( Py_TYPE( callable ) -> tp_name ) ); + } + } + if( callable == Py_None ) + callable = nullptr; + auto struct_ptr = self -> struct_; + auto pyobj_ptr = structToDict( struct_ptr, callable ); + return pyobj_ptr.release(); + + CSP_RETURN_NULL; +} + PyObject * PyStruct_to_json( PyStruct * self, PyObject * args, PyObject * kwargs ) { CSP_BEGIN_METHOD; @@ -966,6 +992,7 @@ static PyMethodDef PyStruct_methods[] = { { "update_from", (PyCFunction) PyStruct_update_from, METH_O, "update from struct. struct must be same type or a derived type. unset fields will be not be copied" }, { "update", (PyCFunction) PyStruct_update, METH_VARARGS | METH_KEYWORDS, "update from key=val. given fields will be set on struct. other fields will remain as is in struct" }, { "all_fields_set", (PyCFunction) PyStruct_all_fields_set, METH_NOARGS, "return true if all fields on the struct are set" }, + { "to_dict", (PyCFunction) PyStruct_to_dict, METH_VARARGS | METH_KEYWORDS, "return a python dict of the struct by recursively converting struct members into python dicts" }, { "to_json", (PyCFunction) PyStruct_to_json, METH_VARARGS | METH_KEYWORDS, "return a json string of the struct by recursively converting struct members into json format" }, { NULL} }; @@ -1017,7 +1044,6 @@ REGISTER_TYPE_INIT( &PyStructMeta::PyType, "PyStructMeta" ) REGISTER_TYPE_INIT( &PyStruct::PyType, "PyStruct" ) // Instantiate all templates for PyStructList class -template struct PyStructList; template struct PyStructList; template struct PyStructList; template struct PyStructList; @@ -1036,4 +1062,23 @@ template struct PyStructList; template struct PyStructList; template struct PyStructList; +// Instantiate all templates for PyStructFastList class +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList; +template struct PyStructFastList