Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert available funds / contributions/expenditures from area to step charts #216

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
+196 −237
Open

Convert available funds / contributions/expenditures from area to step charts #216

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3da50d3
Revise contribution/expenditure chart as a true area chart
hancush Oct 1, 2024
e5796ec
Revert JavaScript formatting
hancush Oct 1, 2024
8c7b59a
Convert area chart to step chart
hancush Oct 1, 2024
26d670d
Remove unneeded option
hancush Oct 1, 2024
8a95238
Convert funds by filing into line chart
hancush Oct 2, 2024
4338d93
Use filing period end for available funds
hancush Oct 2, 2024
99fca26
Update chart title
hancush Oct 2, 2024
f21a76e
Group filings ending on same date
hancush Oct 2, 2024
16dfa3d
Update aggregate queries so that contributions and expenditures are a…
hancush Oct 3, 2024
a3f98a8
Revert to line charts
hancush Jan 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,6 @@ repos:
files: .*/templates/.*\.html$
args: [--tabwidth=2] # tabs should be 2 spaces in Django templates

- repo: https://github.com/pre-commit/mirrors-prettier
rev: v2.5.1
hooks:
- id: prettier
files: \.(js|ts|jsx|tsx|css|less|json|markdown|md|yaml|yml)$

- repo: https://github.com/pre-commit/mirrors-eslint
rev: "1f7d592"
hooks:
- id: eslint
additional_dependencies:
- [email protected]
- [email protected]
- [email protected]

- repo: https://github.com/pycqa/isort
rev: 5.13.2
hooks:
Expand Down
79 changes: 56 additions & 23 deletions camp_fin/management/commands/aggregate_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,31 @@ def makeTransactionAggregates(self, recreate_views):
)
)

continuous_interval_query = """
CREATE MATERIALIZED VIEW {transaction_type}_by_{interval} AS (
WITH {interval}_summaries AS (
{summary_query}
)
SELECT
continuous_interval.entity_id,
continuous_interval.{interval},
COALESCE({interval}_summaries.amount, 0) AS amount
FROM (
SELECT DISTINCT
entity_id,
generate_series(
MIN({interval}) OVER (PARTITION BY entity_id),
MAX({interval}) OVER (PARTITION BY entity_id),
'1 {interval}'
) AS {interval}
FROM {interval}_summaries
) continuous_interval
LEFT JOIN {interval}_summaries
USING (entity_id, {interval})
ORDER BY entity_id, {interval}
)
"""
Comment on lines +41 to +64
Copy link
Member Author

@hancush hancush Oct 3, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we have transaction info for a candidate in Jan, March, and April, then we can infer from the absence of transactions that they clocked a total of 0 in Feb. This query creates a continuous sequence of aggregates for each entity, beginning with the first date we have a transaction for and ending with the last date we have a transaction for. Left joining the summaries then fills in intervals with no transactions with 0.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is useful in particular for the chart. Given sparse data, a step chart will draw a line from point A to C, suggesting a potentially misleading value for B (which gets worse the more values are missing). This registers months without transactions appropriately as 0.


try:
self.executeTransaction(
"""
Expand All @@ -47,17 +72,16 @@ def makeTransactionAggregates(self, recreate_views):
)
)
except ProgrammingError:
view = """
CREATE MATERIALIZED VIEW contributions_by_{0} AS (
SELECT
summary_query = """
SELECT
SUM(amount) AS amount,
entity_id,
{0}
FROM (
{interval}
FROM (
SELECT
SUM(t.amount) AS amount,
f.entity_id,
MAX(date_trunc('{0}', t.received_date)) AS {0}
MAX(date_trunc('{interval}', t.received_date)) AS {interval}
FROM camp_fin_transaction AS t
JOIN camp_fin_transactiontype AS tt
ON t.transaction_type_id = tt.id
Expand All @@ -68,21 +92,26 @@ def makeTransactionAggregates(self, recreate_views):
'Monetary Contribution',
'Anonymous Contribution'
)
GROUP BY f.entity_id, date_trunc('{0}', t.received_date)
GROUP BY f.entity_id, date_trunc('{interval}', t.received_date)
UNION
SELECT
SUM(l.amount) AS amount,
f.entity_id,
MAX(date_trunc('{0}', l.received_date)) AS {0}
MAX(date_trunc('{interval}', l.received_date)) AS {interval}
FROM camp_fin_loan AS l
JOIN camp_fin_filing AS f
ON l.filing_id = f.id
GROUP BY f.entity_id, date_trunc('{0}', l.received_date)
) AS s
GROUP BY entity_id, {0}
)
GROUP BY f.entity_id, date_trunc('{interval}', l.received_date)
) contributions_and_loans
GROUP BY entity_id, {interval}
""".format(
interval
interval=interval
)

view = continuous_interval_query.format(
transaction_type="contributions",
summary_query=summary_query,
interval=interval,
)

self.executeTransaction(view)
Expand All @@ -96,17 +125,16 @@ def makeTransactionAggregates(self, recreate_views):
)
)
except ProgrammingError:
view = """
CREATE MATERIALIZED VIEW expenditures_by_{0} AS (
summary_query = """
SELECT
entity_id,
SUM(amount) AS amount,
{0}
{interval}
FROM (
SELECT
filing.entity_id,
SUM(e.amount) AS amount,
date_trunc('{0}', e.received_date) AS {0}
date_trunc('{interval}', e.received_date) AS {interval}
FROM camp_fin_transaction AS e
JOIN camp_fin_transactiontype AS tt
ON e.transaction_type_id = tt.id
Expand All @@ -116,14 +144,14 @@ def makeTransactionAggregates(self, recreate_views):
ON filing.filing_period_id = fp.id
WHERE tt.contribution = FALSE
AND filing.filed_date >= '2010-01-01'
GROUP BY filing.entity_id, date_trunc('{0}', e.received_date)
GROUP BY filing.entity_id, date_trunc('{interval}', e.received_date)

UNION

SELECT
filing.entity_id,
SUM(lt.amount) AS amount,
date_trunc('{0}', lt.transaction_date) AS {0}
date_trunc('{interval}', lt.transaction_date) AS {interval}
FROM camp_fin_loantransaction AS lt
JOIN camp_fin_loantransactiontype AS ltt
ON lt.transaction_type_id = ltt.id
Expand All @@ -133,12 +161,17 @@ def makeTransactionAggregates(self, recreate_views):
ON filing.filing_period_id = fp.id
WHERE ltt.description = 'Payment'
AND filing.filed_date >= '2010-01-01'
GROUP BY filing.entity_id, date_trunc('{0}', lt.transaction_date)
GROUP BY filing.entity_id, date_trunc('{interval}', lt.transaction_date)
) AS s
GROUP BY entity_id, {0}
)
GROUP BY entity_id, {interval}
""".format(
interval
interval=interval
)

view = continuous_interval_query.format(
transaction_type="expenditures",
summary_query=summary_query,
interval=interval,
)

self.executeTransaction(view)
Expand Down
Loading
Loading