Release 1.192.0

See release notes.
eruditis · May 18, 2024 · 3db0827 · 3db0827
2 parents 0eb0b56 + 380169c
commit 3db0827
Show file tree

Hide file tree

Showing 465 changed files with 18,890 additions and 7,423 deletions.
diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml
@@ -0,0 +1,48 @@
+version: '3.5'
+
+services:
+  postgres:
+    container_name: nautilus-database
+    image: postgres
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER:-postgres}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-pass}
+      POSTGRES_DATABASE: nautilus
+      PGDATA: /data/postgres
+    volumes:
+      - nautilus-database:/data/postgres
+    ports:
+      - "5432:5432"
+    networks:
+      - nautilus-network
+    restart: unless-stopped
+
+  pgadmin:
+    container_name: nautilus-pgadmin
+    image: dpage/pgadmin4
+    environment:
+      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:[email protected]}
+      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin}
+    volumes:
+      - pgadmin:/root/.pgadmin
+    ports:
+      - "${PGADMIN_PORT:-5051}:80"
+    networks:
+      - nautilus-network
+    restart: unless-stopped
+
+  redis:
+    container_name: nautilus-redis
+    image: redis
+    ports:
+      - 6379:6379
+    restart: unless-stopped
+    networks:
+      - nautilus-network
+
+networks:
+  nautilus-network:
+
+volumes:
+  nautilus-database:
+  pgadmin:
diff --git a/.docker/nautilus_trader.dockerfile b/.docker/nautilus_trader.dockerfile
@@ -4,7 +4,7 @@ ENV PYTHONUNBUFFERED=1 \
     PIP_NO_CACHE_DIR=off \
     PIP_DISABLE_PIP_VERSION_CHECK=on \
     PIP_DEFAULT_TIMEOUT=100 \
-    POETRY_VERSION=1.8.2 \
+    POETRY_VERSION=1.8.3 \
     POETRY_HOME="/opt/poetry" \
     POETRY_VIRTUALENVS_CREATE=false \
     POETRY_NO_INTERACTION=1 \

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -22,6 +22,25 @@ jobs:
     env:
       BUILD_MODE: debug
       RUST_BACKTRACE: 1
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: pass
+          POSTGRES_DB: nautilus
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
       - name: Free disk space (Ubuntu)
@@ -105,10 +124,16 @@ jobs:
           # pre-commit run --hook-stage manual gitlint-ci
           pre-commit run --all-files
 
-      - name: Install Redis (Linux)
+      - name: Install Nautilus CLI and run init postgres
         run: |
-          sudo apt-get install redis-server
-          redis-server --daemonize yes
+          make install-cli
+          nautilus database init --schema ${{ github.workspace }}/schema
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          POSTGRES_USERNAME: postgres
+          POSTGRES_PASSWORD: pass
+          POSTGRES_DATABASE: nautilus
 
       - name: Run nautilus_core cargo tests (Linux)
         run: |
@@ -209,7 +234,8 @@ jobs:
           PARALLEL_BUILD: false
 
   build-macos:
-    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/nightly'
+    if: github.ref == 'refs/heads/master'
+    # if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/nightly'
     strategy:
       fail-fast: false
       matrix:
@@ -290,11 +316,6 @@ jobs:
           # pre-commit run --hook-stage manual gitlint-ci
           pre-commit run --all-files
 
-      - name: Install Redis (macOS)
-        run: |
-          brew install redis
-          redis-server --daemonize yes
-
       - name: Run nautilus_core cargo tests (macOS)
         run: |
           cargo install cargo-nextest

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -14,6 +14,25 @@ jobs:
         python-version: ["3.10"]  # Fails on 3.11 due Cython
     name: build - Python ${{ matrix.python-version }} (${{ matrix.arch }} ${{ matrix.os }})
     runs-on: ${{ matrix.os }}
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: pass
+          POSTGRES_DB: nautilus
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
       - name: Free disk space (Ubuntu)
@@ -90,10 +109,16 @@ jobs:
           path: ${{ env.POETRY_CACHE_DIR }}
           key: ${{ runner.os }}-${{ matrix.python-version }}-poetry-${{ hashFiles('**/poetry.lock') }}
 
-      - name: Install Redis
+      - name: Install Nautilus CLI and run init postgres
         run: |
-          sudo apt-get install redis-server
-          redis-server --daemonize yes
+          make install-cli
+          nautilus database init --schema ${{ github.workspace }}/schema
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          POSTGRES_USERNAME: postgres
+          POSTGRES_PASSWORD: pass
+          POSTGRES_DATABASE: nautilus
 
       - name: Run tests with coverage
         run: make pytest-coverage

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -35,15 +35,15 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Login to GHCR
-        uses: docker/login-action@v3
+        uses: docker/login-action@v3.1.0
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.PACKAGES_TOKEN }}
 
       - name: Get branch name
         id: branch-name
-        uses: tj-actions/branch-names@v7.0.7
+        uses: tj-actions/branch-names@v8.0.1
 
       - name: Build nautilus_trader image (nightly)
         if: ${{ steps.branch-name.outputs.current_branch == 'nightly' }}

diff --git a/.gitignore b/.gitignore
@@ -17,6 +17,7 @@
 *.env
 *.tar.gz*
 *.zip
+*.iml
 
 *.dbz
 *.dbn

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -73,7 +73,7 @@ repos:
         types: [python]
 
   - repo: https://github.com/psf/black
-    rev: 24.4.0
+    rev: 24.4.2
     hooks:
       - id: black
         types_or: [python, pyi]
@@ -82,7 +82,7 @@ repos:
         exclude: "docs/_pygments/monokai.py"
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.7
+    rev: v0.4.4
     hooks:
       - id: ruff
         args: ["--fix"]
@@ -111,7 +111,7 @@ repos:
         ]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.10.0
     hooks:
       - id: mypy
         args: [

diff --git a/Makefile b/Makefile
@@ -70,7 +70,7 @@ docs-python: install-just-deps-all
 
 .PHONY: docs-rust
 docs-rust:
-	(cd nautilus_core && RUSTDOCFLAGS="--enable-index-page -Zunstable-options" cargo +nightly doc --no-deps)
+	(cd nautilus_core && RUSTDOCFLAGS="--enable-index-page -Zunstable-options" cargo +nightly doc --all-features --no-deps --workspace --exclude tokio-tungstenite)
 
 .PHONY: clippy
 clippy:
@@ -137,6 +137,14 @@ docker-build-jupyter:
 docker-push-jupyter:
 	docker push ${IMAGE}:jupyter
 
+.PHONY: start-services
+start-services:
+	docker-compose -f .docker/docker-compose.yml up -d
+
+.PHONY: stop-services
+stop-services:
+	docker-compose -f .docker/docker-compose.yml down
+
 .PHONY: pytest
 pytest:
 	bash scripts/test.sh
@@ -153,11 +161,6 @@ test-examples:
 install-talib:
 	bash scripts/install-talib.sh
 
-.PHONY: init-db
-init-db:
-	(cd nautilus_core && cargo run --bin init-db)
-
-.PHONY: drop-db
-drop-db:
-	(cd nautilus_core && cargo run --bin drop-db)
-
+.PHONY: install-cli
+install-cli:
+	(cd nautilus_core && cargo install --path cli  --bin nautilus)
diff --git a/README.md b/README.md
@@ -15,10 +15,10 @@
 
 | Platform           | Rust    | Python |
 | :----------------- | :------ | :----- |
-| `Linux (x86_64)`   | 1.77.1+ | 3.10+  |
-| `macOS (x86_64)`   | 1.77.1+ | 3.10+  |
-| `macOS (arm64)`    | 1.77.1+ | 3.10+  |
-| `Windows (x86_64)` | 1.77.1+ | 3.10+  |
+| `Linux (x86_64)`   | 1.78.0+ | 3.10+  |
+| `macOS (x86_64)`   | 1.78.0+ | 3.10+  |
+| `macOS (arm64)`    | 1.78.0+ | 3.10+  |
+| `Windows (x86_64)` | 1.78.0+ | 3.10+  |
 
 - **Website:** https://nautilustrader.io
 - **Docs:** https://docs.nautilustrader.io
@@ -44,8 +44,8 @@ including FX, Equities, Futures, Options, CFDs, Crypto and Betting - across mult
 
 ## Features
 
-- **Fast** - C-level speed through Rust and Cython. Asynchronous networking with [uvloop](https://github.com/MagicStack/uvloop)
-- **Reliable** - Type safety through Rust and Cython. Redis backed performant state persistence
+- **Fast** - Core written in Rust with asynchronous networking using [tokio](https://crates.io/crates/tokio)
+- **Reliable** - Type safety and thread safety through Rust. Redis backed performant state persistence
 - **Portable** - OS independent, runs on Linux, macOS, Windows. Deploy using Docker
 - **Flexible** - Modular adapters mean any REST, WebSocket, or FIX API can be integrated
 - **Advanced** - Time in force `IOC`, `FOK`, `GTD`, `AT_THE_OPEN`, `AT_THE_CLOSE`, advanced order types and conditional triggers. Execution instructions `post-only`, `reduce-only`, and icebergs. Contingency order lists including `OCO`, `OTO`
@@ -76,7 +76,8 @@ express the granular time and event dependent complexity of real-time trading, w
 proven to be more suitable due to their inherently higher performance, and type safety.
 
 One of the key advantages of NautilusTrader here, is that this reimplementation step is now circumvented - as the critical core components of the platform
-have all been written entirely in Rust or Cython. This means we're using the right tools for the job, where systems programming languages compile performant binaries,
+have all been written entirely in [Rust](https://www.rust-lang.org/) or [Cython](https://cython.org/).
+This means we're using the right tools for the job, where systems programming languages compile performant binaries,
 with CPython C extension modules then able to offer a Python native environment, suitable for professional quantitative traders and trading firms.
 
 ## Why Python?
@@ -91,16 +92,6 @@ implementing large performance-critical systems. Cython has addressed a lot of t
 of a statically typed language, embedded into Pythons rich ecosystem of software libraries and
 developer/user communities.
 
-## What is Cython?
-
-[Cython](https://cython.org) is a compiled programming language which aims to be a superset of the Python programming
-language, designed to give C-like performance with code that is written in Python - with
-optional C-inspired syntax.
-
-The project heavily utilizes Cython to provide static type safety and increased performance
-for Python through [C extension modules](https://docs.python.org/3/extending/extending.html). The vast majority of the production code is actually
-written in Cython, however the libraries can be accessed from both Python and Cython.
-
 ## What is Rust?
 
 [Rust](https://www.rust-lang.org/) is a multi-paradigm programming language designed for performance and safety, especially safe
@@ -111,9 +102,8 @@ Rust’s rich type system and ownership model guarantees memory-safety and threa
 eliminating many classes of bugs at compile-time.
 
 The project increasingly utilizes Rust for core performance-critical components. Python language binding is handled through
-Cython, with static libraries linked at compile-time before the wheel binaries are packaged, so a user
-does not need to have Rust installed to run NautilusTrader. In the future as more Rust code is introduced,
-[PyO3](https://pyo3.rs/latest) will be leveraged for easier Python bindings.
+Cython and [PyO3](https://pyo3.rs/latest), with static libraries linked at compile-time before the wheel binaries are packaged, so a user
+does not need to have Rust installed to run NautilusTrader.
 
 This project makes the [Soundness Pledge](https://raphlinus.github.io/rust/2020/01/18/soundness-pledge.html):
 
@@ -124,15 +114,6 @@ This project makes the [Soundness Pledge](https://raphlinus.github.io/rust/2020/
 
 ![Architecture](https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/_images/architecture-overview.png?raw=true "architecture")
 
-## Quality Attributes
-
-- Reliability
-- Performance
-- Modularity
-- Testability
-- Maintainability
-- Deployability
-
 ## Integrations
 
 NautilusTrader is designed in a modular way to work with _adapters_ which provide
@@ -238,8 +219,8 @@ A `Makefile` is provided to automate most installation and build tasks for devel
 - `make install-debug` -- Same as `make install` but with `debug` build mode
 - `make install-just-deps` -- Installs just the `main`, `dev` and `test` dependencies (does not install package)
 - `make install-just-deps-all` -- Same as `make install-just-deps` and additionally installs `docs` dependencies
-- `make build` -- Runs the Cython build script in `release` build mode (default)
-- `make build-debug` -- Runs the Cython build script in `debug` build mode
+- `make build` -- Runs the build script in `release` build mode (default)
+- `make build-debug` -- Runs the build script in `debug` build mode
 - `make build-wheel` -- Runs the Poetry build with a wheel format in `release` mode
 - `make build-wheel-debug` -- Runs the Poetry build with a wheel format in `debug` mode
 - `make clean` -- **CAUTION** Cleans all non-source artifacts from the repository

diff --git a/RELEASES.md b/RELEASES.md
@@ -1,3 +1,42 @@
+# NautilusTrader 1.192.0 Beta
+
+Released on 18th May 2024 (UTC).
+
+### Enhancements
+- Added Nautilus CLI (see [docs](https://docs.nautilustrader.io/nightly/developer_guide/index.html)) (#1602), many thanks @filipmacek
+- Added `Cfd` and `Commodity` instruments with Interactive Brokers support (#1604), thanks @DracheShiki
+- Added `OrderMatchingEngine` futures and options contract activation and expiration simulation
+- Added Sandbox example with Interactive Brokers (#1618), thanks @rsmb7z
+- Added `ParquetDataCatalog` S3 support (#1620), thanks @benjaminsingleton
+- Added `Bar.from_raw_arrays_to_list` (#1623), thanks @rsmb7z
+- Added `SandboxExecutionClientConfig.bar_execution` option (#1646), thanks @davidsblom
+- Improved venue order ID generation and assignment (it was previously possible for the `OrderMatchingEngine` to generate multiple IDs for the same order)
+- Improved `LiveTimer` robustness and flexibility by not requiring positive intervals or stop times in the future (will immediately produce a time event), thanks for reporting @davidsblom
+
+### Breaking Changes
+- Removed `allow_cash_positions` config (simplify to the most common use case, spot trading should track positions)
+- Changed `tags` param and return type from `str` to `list[str]` (more naturally expresses multiple tags)
+- Changed `Order.to_dict()` `commission` and `linked_order_id` fields to lists of strings rather than comma separated strings
+- Changed `OrderMatchingEngine` to no longer process internally aggregated bars for execution (no tests failed, but still classifying as a behavior change), thanks for reporting @davidsblom
+
+### Fixes
+- Fixed `CashAccount` PnL and balance calculations (was adjusting filled quantity based on open position quantity - causing a desync and incorrect balance values)
+- Fixed `from_str` for `Price`, `Quantity` and `Money` when input string contains underscores in Rust, thanks for reporting @filipmacek
+- Fixed `Money` string parsing where the value from `str(money)` can now be passed to `Money.from_str`
+- Fixed `TimeEvent` equality (now based on the event `id` rather than the event `name`)
+- Fixed `ParquetDataCatalog` bar queries by `instrument_id` which were no longer returning data (the intent is to use `bar_type`, however using `instrument_id` now returns all matching bars)
+- Fixed venue order ID generation and application in sandbox mode (was previously generating additional venue order IDs), thanks for reporting @rsmb7z and @davidsblom
+- Fixed multiple fills causing overfills in sandbox mode (`OrderMatchingEngine` now caching filled quantity to prevent this) (#1642), thanks @davidsblom
+- Fixed `leaves_qty` exception message underflow (now correctly displays the projected negative leaves quantity)
+- Fixed Interactive Brokers contract details parsing (#1615), thanks @rsmb7z
+- Fixed Interactive Brokers portfolio registration (#1616), thanks @rsmb7z
+- Fixed Interactive Brokers `IBOrder` attributes assignment (#1634), thanks @rsmb7z
+- Fixed IBKR reconnection after gateway/TWS disconnection (#1622), thanks @benjaminsingleton
+- Fixed Binance Futures account balance calculation (was over stating `free` balance with margin collateral, which could result in a negative `locked` balance)
+- Fixed Betfair stream reconnection and avoid multiple reconnect attempts (#1644), thanks @imemo88
+
+---
+
 # NautilusTrader 1.191.0 Beta
 
 Released on 20th April 2024 (UTC).
-Original file line number
+Diff line change
@@ Expand Up / @@ -17,6 +17,7 @@ @@
     *.env
     *.tar.gz*
     *.zip
+    *.iml
     *.dbz
     *.dbn
@@ Expand Down @@