Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DipDup version #11

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 40 additions & 47 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,81 +1,74 @@
ARG PYTHON_VERSION=3.12
ARG PYTHON_VERSION=3.12-slim-bookworm
ARG SOURCE_DIR=bridge_indexer
ARG POETRY_PATH=/opt/poetry
ARG VENV_PATH=/opt/venv
ARG APP_PATH=/opt/app
ARG APP_USER=dipdup

FROM python:${PYTHON_VERSION}-slim AS builder-base

ARG VENV_PATH
ARG POETRY_PATH
ENV PIP_NO_CACHE_DIR=off \
PIP_DISABLE_PIP_VERSION_CHECK=on \
PIP_DEFAULT_TIMEOUT=100 \
POETRY_HOME="$POETRY_PATH" \
VIRTUAL_ENV=$VENV_PATH \
PATH="$POETRY_PATH/bin:$VENV_PATH/bin:$PATH"

RUN apt-get update \
&& apt-get install --no-install-recommends -y \
# deps for installing poetry
curl \
FROM python:${PYTHON_VERSION} AS builder-base

SHELL ["/bin/bash", "-exc"]

RUN apt-get update -qy \
&& apt-get install --no-install-recommends --no-install-suggests -qyy \
# deps for building python deps
build-essential \
# pytezos deps
libsodium-dev libgmp-dev pkg-config \
git \
\
# install poetry
&& curl -sSL https://install.python-poetry.org | python - \
\
# configure poetry & make a virtualenv ahead of time since we only need one
&& python -m venv $VENV_PATH \
&& poetry config virtualenvs.create false \
\
# cleanup
# cleanup \
&& apt-get clean \
&& rm -rf /tmp/* \
&& rm -rf /var/tmp/* \
&& rm -rf /root/.cache \
&& rm -rf `find /usr/local/lib $POETRY_PATH/venv/lib $VENV_PATH/lib -name __pycache__` \
&& rm -rf /var/lib/apt/lists/*

ARG APP_PATH
ENV UV_PROJECT_ENVIRONMENT=$APP_PATH


FROM builder-base AS builder-production

COPY ["poetry.lock", "pyproject.toml", "./"]
RUN --mount=from=ghcr.io/astral-sh/uv,source=/uv,target=/bin/uv \
--mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=uv.lock,target=uv.lock \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
uv sync --frozen --exact --no-install-project --no-editable --no-dev --no-installer-metadata

RUN poetry install --only main --sync --no-root --no-interaction --no-ansi -vvv \
&& rm -rf /tmp \
&& rm -rf /root/.cache \
&& rm -rf $VIRTUAL_ENV/src \
&& rm -rf `find $VIRTUAL_ENV/lib -name __pycache__`

FROM python:${PYTHON_VERSION} AS runtime-base

FROM python:${PYTHON_VERSION}-slim AS runtime-base
SHELL ["/bin/bash", "-exc"]

ARG VENV_PATH
ENV PATH="$VENV_PATH/bin:$PATH"
RUN apt-get update -qy \
&& apt-get install --no-install-recommends --no-install-suggests -qyy \
# pytezos deps
libsodium-dev libgmp-dev pkg-config \
\
# cleanup \
&& apt-get clean \
&& rm -rf /tmp/* \
&& rm -rf /var/tmp/* \
&& rm -rf /root/.cache \
&& rm -rf /var/lib/apt/lists/*

ARG APP_PATH
ENV PATH=$APP_PATH/bin:$PATH

WORKDIR $APP_PATH

ARG APP_USER
RUN useradd -ms /bin/bash $APP_USER

FROM runtime-base AS runtime

RUN apt-get update \
&& apt-get install --no-install-recommends -y \
libsodium-dev libgmp-dev pkg-config \
# cleanup
&& rm -rf /tmp/* \
&& rm -rf /root/.cache \
&& rm -rf /var/lib/apt/lists/*

ARG VENV_PATH
COPY --from=builder-production ["$VENV_PATH", "$VENV_PATH"]
FROM runtime-base AS runtime

ARG APP_USER
ARG APP_PATH
COPY --from=builder-production --chown=$APP_USER ["$APP_PATH", "$APP_PATH"]

USER $APP_USER
ARG SOURCE_DIR
ENV DIPDUP_PACKAGE_PATH=$APP_PATH/$SOURCE_DIR
COPY --chown=$APP_USER $SOURCE_DIR ./$SOURCE_DIR

ENTRYPOINT ["dipdup"]
Expand Down
14 changes: 4 additions & 10 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
.ONESHELL:
.DEFAULT_GOAL: all

py := poetry run
py := uv run

source_dir := bridge_indexer
unit_tests_dir := tests
Expand All @@ -14,12 +14,6 @@ test:
install:
poetry install `if [ "${DEV}" = "0" ]; then echo "--only main"; fi` --sync

isort:
$(py) isort $(source_dir) $(unit_tests_dir)

ssort:
$(py) ssort $(source_dir) $(unit_tests_dir)

black:
$(py) black $(source_dir) $(unit_tests_dir)

Expand All @@ -29,7 +23,7 @@ ruff:
mypy:
$(py) mypy $(source_dir) $(unit_tests_dir)

lint: isort ssort black ruff mypy
lint: black ruff mypy

wipe:
$(py) dipdup $(dipdup_args) schema wipe --force
Expand All @@ -41,7 +35,7 @@ run:
$(py) dipdup $(dipdup_args) run

up:
docker-compose up --build --remove-orphans --force-recreate --abort-on-container-exit
docker compose up --build --remove-orphans --force-recreate --abort-on-container-exit

down:
docker-compose down --volumes
docker compose down --volumes
2 changes: 1 addition & 1 deletion bridge_indexer/dipdup.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ datasources:

advanced:
reindex:
config_modified: exception
config_modified: ignore
schema_modified: exception
migration: exception

Expand Down
103 changes: 62 additions & 41 deletions bridge_indexer/handlers/bridge_matcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ class BridgeMatcher:
async def check_pending_tezos_deposits(cls):
if not BridgeMatcherLocks.pending_tezos_deposits:
return
else:
BridgeMatcherLocks.pending_tezos_deposits = False
BridgeMatcherLocks.pending_tezos_deposits = False

qs = TezosDepositOperation.filter(bridge_deposits=None)
async for l1_deposit in qs:
Expand All @@ -45,20 +44,29 @@ async def check_pending_tezos_deposits(cls):
async def check_pending_inbox(cls):
if not BridgeMatcherLocks.pending_inbox:
return
else:
BridgeMatcherLocks.pending_inbox = False

qs = BridgeDepositOperation.filter(
inbox_message=None,
).order_by(
'l1_transaction__level', 'l1_transaction__counter', 'l1_transaction__nonce',
).prefetch_related('l1_transaction')
BridgeMatcherLocks.pending_inbox = False

qs = (
BridgeDepositOperation.filter(
inbox_message=None,
)
.order_by(
'l1_transaction__level',
'l1_transaction__counter',
'l1_transaction__nonce',
)
.prefetch_related('l1_transaction')
)
async for bridge_deposit in qs:
bridge_deposit: BridgeDepositOperation
inbox_message = await RollupInboxMessage.filter(
parameters_hash=bridge_deposit.l1_transaction.parameters_hash,
level=bridge_deposit.l1_transaction.level,
).order_by('index').first()
inbox_message = (
await RollupInboxMessage.filter(
parameters_hash=bridge_deposit.l1_transaction.parameters_hash,
level=bridge_deposit.l1_transaction.level,
)
.order_by('index')
.first()
)

if inbox_message:
bridge_deposit.inbox_message = inbox_message
Expand All @@ -72,12 +80,15 @@ async def check_pending_inbox(cls):
async def check_pending_etherlink_deposits(cls):
if not BridgeMatcherLocks.pending_etherlink_deposits:
return
else:
BridgeMatcherLocks.pending_etherlink_deposits = False
BridgeMatcherLocks.pending_etherlink_deposits = False

qs = EtherlinkDepositOperation.filter(
bridge_deposits=None,
).prefetch_related('l2_token').order_by('level', 'transaction_index', 'log_index')
qs = (
EtherlinkDepositOperation.filter(
bridge_deposits=None,
)
.prefetch_related('l2_token')
.order_by('level', 'transaction_index', 'log_index')
)

async for l2_deposit in qs:
l2_deposit: EtherlinkDepositOperation
Expand Down Expand Up @@ -114,13 +125,16 @@ async def check_pending_etherlink_deposits(cls):
async def check_pending_etherlink_xtz_deposits(cls):
if not BridgeMatcherLocks.pending_etherlink_xtz_deposits:
return
else:
BridgeMatcherLocks.pending_etherlink_xtz_deposits = False
BridgeMatcherLocks.pending_etherlink_xtz_deposits = False

qs = EtherlinkDepositOperation.filter(
bridge_deposits=None,
l2_token_id='xtz',
).order_by('level', 'transaction_index').prefetch_related('l2_token', 'l2_token__ticket')
qs = (
EtherlinkDepositOperation.filter(
bridge_deposits=None,
l2_token_id='xtz',
)
.order_by('level', 'transaction_index')
.prefetch_related('l2_token', 'l2_token__ticket')
)
async for l2_deposit in qs:
l2_deposit: EtherlinkDepositOperation
bridge_deposit = (
Expand Down Expand Up @@ -159,8 +173,7 @@ async def check_pending_etherlink_xtz_deposits(cls):
async def check_pending_etherlink_withdrawals(cls):
if not BridgeMatcherLocks.pending_etherlink_withdrawals:
return
else:
BridgeMatcherLocks.pending_etherlink_withdrawals = False
BridgeMatcherLocks.pending_etherlink_withdrawals = False

qs = EtherlinkWithdrawOperation.filter(bridge_withdrawals=None)
async for l2_withdrawal in qs:
Expand All @@ -179,20 +192,29 @@ async def check_pending_etherlink_withdrawals(cls):
async def check_pending_outbox(cls):
if not BridgeMatcherLocks.pending_outbox:
return
else:
BridgeMatcherLocks.pending_outbox = False

qs = BridgeWithdrawOperation.filter(
outbox_message=None,
).order_by(
'l2_transaction__level', 'l2_transaction__transaction_index', 'l2_transaction__log_index',
).prefetch_related('l2_transaction')
BridgeMatcherLocks.pending_outbox = False

qs = (
BridgeWithdrawOperation.filter(
outbox_message=None,
)
.order_by(
'l2_transaction__level',
'l2_transaction__transaction_index',
'l2_transaction__log_index',
)
.prefetch_related('l2_transaction')
)
async for bridge_withdrawal in qs:
bridge_withdrawal: BridgeWithdrawOperation
outbox_message = await RollupOutboxMessage.filter(
parameters_hash=bridge_withdrawal.l2_transaction.parameters_hash,
bridge_withdrawals=None,
).order_by('level', 'index').first()
outbox_message = (
await RollupOutboxMessage.filter(
parameters_hash=bridge_withdrawal.l2_transaction.parameters_hash,
bridge_withdrawals=None,
)
.order_by('level', 'index')
.first()
)

if outbox_message:
bridge_withdrawal.outbox_message = outbox_message
Expand All @@ -206,8 +228,7 @@ async def check_pending_outbox(cls):
async def check_pending_tezos_withdrawals(cls):
if not BridgeMatcherLocks.pending_tezos_withdrawals:
return
else:
BridgeMatcherLocks.pending_tezos_withdrawals = False
BridgeMatcherLocks.pending_tezos_withdrawals = False

qs = TezosWithdrawOperation.filter(bridge_withdrawals=None).order_by('level')
async for l1_withdrawal in qs:
Expand Down
2 changes: 1 addition & 1 deletion bridge_indexer/handlers/etherlink/on_deposit.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ async def on_deposit(
ctx: HandlerContext,
event: EvmEvent[DepositPayload],
) -> None:
ctx.logger.info(f'Etherlink Deposit Event found: 0x{event.data.transaction_hash}')
ctx.logger.info(f'Etherlink Deposit Event found: {event.data.transaction_hash}')

try:
await _validate_ticket(event.payload.ticket_hash)
Expand Down
2 changes: 1 addition & 1 deletion bridge_indexer/handlers/etherlink/on_transfer.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ async def on_transfer(
ctx: HandlerContext,
event: EvmEvent[TransferPayload],
) -> None:
ctx.logger.info(f'Etherlink Token Transfer Event found: 0x{event.data.transaction_hash}')
ctx.logger.info(f'Etherlink Token Transfer Event found: {event.data.transaction_hash}')

amount = event.payload.value
if not amount:
Expand Down
Loading
Loading