diff --git a/.buildpacks b/.buildpacks index 9a97cd222..447307663 100644 --- a/.buildpacks +++ b/.buildpacks @@ -1,2 +1,2 @@ https://github.com/heroku/heroku-buildpack-nodejs.git#v175 -https://github.com/heroku/heroku-buildpack-python.git#v192 \ No newline at end of file +https://github.com/heroku/heroku-buildpack-python.git#v224 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a0b83eeb6..d60cf6753 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,19 +13,13 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 - uses: actions/setup-python@v2 - with: - python-version: 3.6 - + - uses: actions/setup-node@v2 with: node-version: "13.14.0" - - - name: Install test dependencies + + - name: Build frontend assets run: | - python -m pip install --upgrade pip - pip install black==19.10b0 npm install -g yarn@1.21.1 yarn yarn build @@ -38,10 +32,11 @@ jobs: - name: Run tests run: | - docker-compose run --rm test python manage.py collectstatic --no-input - docker-compose run --rm test - black --check --diff budgetportal manage.py discourse - + docker-compose run --rm app python manage.py collectstatic --no-input + docker-compose run --rm -e DJANGO_Q_SYNC=TRUE app python manage.py test + docker-compose run --rm app black --check --diff budgetportal manage.py discourse performance iym + - name: Dump dependency logs for debugging + if: ${{ failure() }} run: | - docker-compose logs db solr minio + docker-compose logs db solr minio selenium diff --git a/.gitignore b/.gitignore index 3a4ebc108..ec64d07e1 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ ghostdriver.log .idea/ .DS_Store dbdump + +.env +/*.csv diff --git a/Dockerfile-test b/Dockerfile-test deleted file mode 100644 index 2f443aa16..000000000 --- a/Dockerfile-test +++ /dev/null @@ -1,13 +0,0 @@ -FROM python:3.7 -ENV PYTHONUNBUFFERED 1 -RUN apt-get update && \ - apt-get install -y chromium-driver - -RUN mkdir /code -WORKDIR /code -COPY requirements-test.txt /code/ -COPY requirements.txt /code/ -RUN pip install -U setuptools==45.3.0 -RUN pip install -r requirements.txt -RUN pip install -r requirements-test.txt -COPY . /code/ diff --git a/README.md b/README.md index 0c3d931e6..f60b577e9 100644 --- a/README.md +++ b/README.md @@ -232,7 +232,10 @@ docker-compose run --rm app python manage.py makemigrations #### Python/Django * Get better debugging with ``python manage.py runserver_plus`` -* Format your code using Black: `budgetportal manage.py discours --exclude budgetportal/bulk_upload.py` +* Format your code using Black (See version in github actions): + + docker-compose run --rm app black budgetportal manage.py discourse performance + #### React stuff (package.json and packages/webapp/package.json) @@ -307,13 +310,13 @@ Running tests All tests ``` -docker-compose run --rm test +docker-compose run --rm app python manage.py test ``` Specific tests, e.g. ``` -docker-compose run --rm test python manage.py test budgetportal.tests.test_bulk_upload.BulkUploadTestCase +docker-compose run --rm app python manage.py test budgetportal.tests.test_bulk_upload.BulkUploadTestCase ``` Production deployment diff --git a/assets/js/components/ChartSourceController/components/BarChart/index.jsx b/assets/js/components/ChartSourceController/components/BarChart/index.jsx index 45a2622a9..d7f13d973 100644 --- a/assets/js/components/ChartSourceController/components/BarChart/index.jsx +++ b/assets/js/components/ChartSourceController/components/BarChart/index.jsx @@ -1,4 +1,4 @@ -import { h, Component } from 'preact'; +import React from 'react'; import Chart from 'chart.js'; import PropTypes from 'prop-types'; @@ -7,120 +7,124 @@ import downloadChart from './services/downloadChart/index.js'; const buildDownloadButton = downloadAction => ( -
- -
+
+ +
); -const Markup = ({ renderChart, height, downloadAction, rotated }) => { - return ( -
-
- -
- {/* {buildDownloadButton(downloadAction)} */} -
- ); +const Markup = ({renderChart, height, downloadAction, rotated}) => { + return ( +
+
+ +
+ {/* {buildDownloadButton(downloadAction)} */} +
+ ); }; Markup.propTypes = { - renderChart: PropTypes.func.isRequired, - height: PropTypes.number.isRequired, - downloadAction: PropTypes.func.isRequired, + renderChart: PropTypes.func.isRequired, + height: PropTypes.number.isRequired, + downloadAction: PropTypes.func.isRequired, }; -class BarChart extends Component { - constructor(...props) { - super(...props); - const { items, color, rotated, viewportWidth, barTypes } = this.props; +class BarChart extends React.Component { + constructor(...props) { + super(...props); + const {items, color, rotated, viewportWidth, barTypes} = this.props; - const calcHeight = (scale) => { - const config = createChartJsConfig({ items, color, rotated, viewportWidth, barTypes }); - return (config.data.datasets[0].data.length * (25 * scale)) + 55; - }; + const calcHeight = (scale) => { + const config = createChartJsConfig({items, color, rotated, viewportWidth, barTypes}); + return (config.data.datasets[0].data.length * (25 * scale)) + 55; + }; - this.values = { - node: null, - chartInstance: null, - }; + this.values = { + node: null, + chartInstance: null, + }; - this.events = { - renderChart: this.renderChart.bind(this), - downloadAction: this.downloadAction.bind(this), - calcHeight, - }; - } + this.events = { + renderChart: this.renderChart.bind(this), + downloadAction: this.downloadAction.bind(this), + calcHeight, + }; - componentDidUpdate() { - const { chartInstance } = this.values; - const { items, color, rotated, barTypes } = this.props; - - const viewportWidth = window.innerWidth; - const config = createChartJsConfig({ items, color, rotated, viewportWidth, barTypes }); + this.state = { + node: null + } + } - config.data.datasets.forEach(({ data }, index) => { - chartInstance.data.datasets[index].data = data; - }); + componentDidUpdate() { + const {chartInstance} = this.values; + const {items, color, rotated, barTypes} = this.props; - return chartInstance.update(); - } + const viewportWidth = window.innerWidth; + const config = createChartJsConfig({items, color, rotated, viewportWidth, barTypes}); - downloadAction(event) { - event.preventDefault(); - const { items, color, rotated, downloadText, barTypes, source } = this.props; - const config = createChartJsConfig({ items, color, rotated, barTypes }); - const { calcHeight } = this.events; - const height = calcHeight(2); + config.data.datasets.forEach(({data}, index) => { + chartInstance.data.datasets[index].data = data; + }); - const canvas = document.createElement('canvas'); - const container = document.createElement('div'); - container.appendChild(canvas); - document.body.appendChild(container); + return chartInstance.update(); + } - container.style.position = 'fixed'; - container.style.top = '200%'; - container.style.width = '800px'; - canvas.height = height; - canvas.style.height = `${height}px`; + downloadAction(event) { + event.preventDefault(); + const {items, color, rotated, downloadText, barTypes, source} = this.props; + const config = createChartJsConfig({items, color, rotated, barTypes}); + const {calcHeight} = this.events; + const height = calcHeight(2); + + const canvas = document.createElement('canvas'); + const container = document.createElement('div'); + container.appendChild(canvas); + document.body.appendChild(container); + + container.style.position = 'fixed'; + container.style.top = '200%'; + container.style.width = '800px'; + canvas.height = height; + canvas.style.height = `${height}px`; + + new Chart(canvas, config); + downloadChart({canvas, height, downloadText, source}); + } - new Chart(canvas, config); - downloadChart({ canvas, height, downloadText, source }); - } + renderChart(newNode) { + const {items, color, rotated, barTypes} = this.props; + const {node} = this.values; - renderChart(newNode) { - const { items, color, rotated, barTypes } = this.props; - const { node } = this.values; + const viewportWidth = window.innerWidth; + const config = createChartJsConfig({items, color, rotated, viewportWidth, barTypes}); + this.values.chartInstance = new Chart(node || newNode, config); - const viewportWidth = window.innerWidth; - const config = createChartJsConfig({ items, color, rotated, viewportWidth, barTypes }); - this.values.chartInstance = new Chart(node || newNode, config); + if (!node) { + this.values.node = newNode; + } - if (!node) { - this.values.node = newNode; + return null; } - return null; - } + render() { + const {renderChart, downloadAction} = this.events; + const {node} = this.state; + const {scale, rotated} = this.props; + const {calcHeight} = this.events; - render() { - const { renderChart, downloadAction } = this.events; - const { node } = this.state; - const { scale, rotated } = this.props; - const { calcHeight } = this.events; - - const height = calcHeight(scale); - return ; - } + const height = calcHeight(scale); + return ; + } } diff --git a/assets/js/components/ChartSourceController/index.jsx b/assets/js/components/ChartSourceController/index.jsx index 79270f8c1..0f6ca17cc 100644 --- a/assets/js/components/ChartSourceController/index.jsx +++ b/assets/js/components/ChartSourceController/index.jsx @@ -1,4 +1,4 @@ -import { h, Component } from 'preact'; +import React from 'react'; import uuid from 'uuid/v4'; import BarChart from './components/BarChart/index.jsx'; @@ -49,7 +49,7 @@ const Markup = ({ items, toggle, styling, changeSource, source, downloadText, ba }; -class ChartSourceController extends Component { +class ChartSourceController extends React.Component { constructor(...props) { super(...props); diff --git a/assets/js/components/GovernmentResources/GovernmentResources.jsx b/assets/js/components/GovernmentResources/GovernmentResources.jsx index 63bb8ee16..b0c39fd08 100644 --- a/assets/js/components/GovernmentResources/GovernmentResources.jsx +++ b/assets/js/components/GovernmentResources/GovernmentResources.jsx @@ -1,4 +1,4 @@ -import { h, Component, render } from 'preact'; +import React from 'react'; function skeletonResources() { return ( @@ -87,7 +87,7 @@ function isCollapsable(resources) { return resources.length > 3; } -export default class GovernmentResources extends Component { +export default class GovernmentResources extends React.Component { constructor(props) { super(props); diff --git a/assets/js/components/Icon/index.jsx b/assets/js/components/Icon/index.jsx index 6dfceb06a..0aa440749 100644 --- a/assets/js/components/Icon/index.jsx +++ b/assets/js/components/Icon/index.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import Close from './partials/Close.jsx'; import Download from './partials/Download.jsx'; import Facebook from './partials/Facebook.jsx'; diff --git a/assets/js/components/Icon/partials/Close.jsx b/assets/js/components/Icon/partials/Close.jsx index aaa25d114..2c6ce172d 100644 --- a/assets/js/components/Icon/partials/Close.jsx +++ b/assets/js/components/Icon/partials/Close.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/Icon/partials/Dataset.jsx b/assets/js/components/Icon/partials/Dataset.jsx index f2374ef97..af7582f29 100644 --- a/assets/js/components/Icon/partials/Dataset.jsx +++ b/assets/js/components/Icon/partials/Dataset.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/Icon/partials/Date.jsx b/assets/js/components/Icon/partials/Date.jsx index 7831a8e88..c1d4a291e 100644 --- a/assets/js/components/Icon/partials/Date.jsx +++ b/assets/js/components/Icon/partials/Date.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/Icon/partials/Download.jsx b/assets/js/components/Icon/partials/Download.jsx index a9cc2c0d0..17f657291 100644 --- a/assets/js/components/Icon/partials/Download.jsx +++ b/assets/js/components/Icon/partials/Download.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/Icon/partials/Facebook.jsx b/assets/js/components/Icon/partials/Facebook.jsx index 4e3c101cb..2a5523bd3 100644 --- a/assets/js/components/Icon/partials/Facebook.jsx +++ b/assets/js/components/Icon/partials/Facebook.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/Icon/partials/Guide.jsx b/assets/js/components/Icon/partials/Guide.jsx index fba92e77c..fd52cf733 100644 --- a/assets/js/components/Icon/partials/Guide.jsx +++ b/assets/js/components/Icon/partials/Guide.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/Icon/partials/Hamburger.jsx b/assets/js/components/Icon/partials/Hamburger.jsx index f9d7023e8..0be6fe1a0 100644 --- a/assets/js/components/Icon/partials/Hamburger.jsx +++ b/assets/js/components/Icon/partials/Hamburger.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/Icon/partials/Home.jsx b/assets/js/components/Icon/partials/Home.jsx index f2ec640b3..0a58468ac 100644 --- a/assets/js/components/Icon/partials/Home.jsx +++ b/assets/js/components/Icon/partials/Home.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/Icon/partials/Pin.jsx b/assets/js/components/Icon/partials/Pin.jsx index 6de9c0f0d..a88b4f8c1 100644 --- a/assets/js/components/Icon/partials/Pin.jsx +++ b/assets/js/components/Icon/partials/Pin.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/Icon/partials/Play.jsx b/assets/js/components/Icon/partials/Play.jsx index 8331a6a24..8e8e470d0 100644 --- a/assets/js/components/Icon/partials/Play.jsx +++ b/assets/js/components/Icon/partials/Play.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/Icon/partials/Search.jsx b/assets/js/components/Icon/partials/Search.jsx index ddcc73f3f..7e508b3ed 100644 --- a/assets/js/components/Icon/partials/Search.jsx +++ b/assets/js/components/Icon/partials/Search.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/Icon/partials/Twitter.jsx b/assets/js/components/Icon/partials/Twitter.jsx index a7eb7c85c..9dce3f3f1 100644 --- a/assets/js/components/Icon/partials/Twitter.jsx +++ b/assets/js/components/Icon/partials/Twitter.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import createSizeModifier from './createSizeModifier.js'; diff --git a/assets/js/components/LinksList/index.jsx b/assets/js/components/LinksList/index.jsx index 2cf5c98d4..3299cbb39 100644 --- a/assets/js/components/LinksList/index.jsx +++ b/assets/js/components/LinksList/index.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import PropTypes from 'prop-types'; import Icon from '../Icon/index.jsx'; diff --git a/assets/js/components/Share/index.jsx b/assets/js/components/Share/index.jsx index 36e5af277..34d9b8255 100644 --- a/assets/js/components/Share/index.jsx +++ b/assets/js/components/Share/index.jsx @@ -1,4 +1,4 @@ -import { h, Component } from 'preact'; +import React from 'react'; import { ga } from 'react-ga'; import PropTypes from 'prop-types'; @@ -87,7 +87,7 @@ Markup.defaultProps = { }; -class Share extends Component { +class Share extends React.Component { constructor(props) { super(props); diff --git a/assets/js/components/about/Contacts/index.html b/assets/js/components/about/Contacts/index.html index a7a73956b..70ce2f94e 100644 --- a/assets/js/components/about/Contacts/index.html +++ b/assets/js/components/about/Contacts/index.html @@ -41,21 +41,6 @@

Contacts

  • Public Service Accountability Monitor (IMALI YETHU representative)
    • -
    GTAC contact for the project
    -
    -
    -
    - - - - -
    -
    -
      -
    • Mr Mehleli Mpofu
      • -
    • Project Manager: Government Technical Advisory Centre
      • -
    -
    Service provider contact for the project
    @@ -67,7 +52,7 @@

    Contacts

    diff --git a/assets/js/components/budget-summary/index.html b/assets/js/components/budget-summary/index.html new file mode 100644 index 000000000..3abe0574c --- /dev/null +++ b/assets/js/components/budget-summary/index.html @@ -0,0 +1,22 @@ +{% load define_action %} +
    +
    +
    + +
    +
    + +
    +
    +
    diff --git a/assets/js/components/department-budgets/DeptControl/index.jsx b/assets/js/components/department-budgets/DeptControl/index.jsx index 2201d8e23..9f3453058 100644 --- a/assets/js/components/department-budgets/DeptControl/index.jsx +++ b/assets/js/components/department-budgets/DeptControl/index.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import PseudoSelect from './../../universal/PseudoSelect/index.jsx'; import provinces from './partials/provinces.json'; import spheres from './partials/spheres.json'; @@ -16,7 +16,7 @@ export default function DeptGroup({ changeKeywords, updateFilter, keywords, open diff --git a/assets/js/components/department-budgets/DeptGroup/index.jsx b/assets/js/components/department-budgets/DeptGroup/index.jsx index b9f9b45f2..eed39b903 100644 --- a/assets/js/components/department-budgets/DeptGroup/index.jsx +++ b/assets/js/components/department-budgets/DeptGroup/index.jsx @@ -1,48 +1,54 @@ -import { h } from 'preact'; +import React from 'react'; import GovernmentResources from './../../GovernmentResources/GovernmentResources.jsx'; import Map from './partials/Map.jsx'; function notAvailableMessage() { - return ( -
    -

    This data is not yet available. Provincial budgets are tabled after the national budget has been announced. This is because the national budget determines the amount of money each province receives. We expect to be able make provincial budget data available by April {(new Date()).getFullYear()}.

    -

    {"In the meantime you view previous financial years' data by selecting a year at the top of your screen."}

    -
    - ); + return ( +
    +

    This data is not yet available. Provincial budgets are tabled + after the national budget has been announced. This is because the national budget determines the amount + of money each province receives. We expect to be able make provincial budget data available by + April {(new Date()).getFullYear()}.

    +

    {"In the meantime you view previous financial years' data by selecting a year at the top of your screen."}

    +
    + ); } function departments(linksArray, doubleRow) { - return ( - - ) + return ( + + ) } -export default function DeptGroup({ map, linksArray, label, name, doubleRow, empty, govResourceGroups }) { - return ( -
    -
    -
    -
    -

    {label} Department Budgets

    - {empty ? notAvailableMessage() : departments(linksArray, doubleRow)} -
    -
    - {Map(map)} -
    +export default function DeptGroup({map, linksArray, label, name, doubleRow, empty, govResourceGroups}) { + return ( +
    +
    +
    +
    +

    {label} Department Budgets

    + {empty ? notAvailableMessage() : departments(linksArray, doubleRow)} +
    +
    + {Map(map)} +
    +
    + +
    - -
    -
    - ); + ); } diff --git a/assets/js/components/department-budgets/DeptGroup/partials/Map.jsx b/assets/js/components/department-budgets/DeptGroup/partials/Map.jsx index d80bb7216..541566b2e 100644 --- a/assets/js/components/department-budgets/DeptGroup/partials/Map.jsx +++ b/assets/js/components/department-budgets/DeptGroup/partials/Map.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; const southAfrica = ( diff --git a/assets/js/components/department-budgets/DeptSearch/index.jsx b/assets/js/components/department-budgets/DeptSearch/index.jsx index 03c015138..44e57e632 100644 --- a/assets/js/components/department-budgets/DeptSearch/index.jsx +++ b/assets/js/components/department-budgets/DeptSearch/index.jsx @@ -1,11 +1,14 @@ -import { h } from 'preact'; +import React from 'react'; import DeptControl from './../DeptControl/index.jsx'; import DeptGroup from './../DeptGroup/index.jsx'; -const makeGroup = (slug, departments, name, label, empty, govResourceGroups) => { +const makeGroup = (slug, departments, name, label, empty, govResourceGroups, index) => { return ( -
    +
    { } return governments.map( - ({ name, slug, departments, label }) => { + ({ name, slug, departments, label }, index) => { const empty = departments.length == 0; - return makeGroup(slug, departments, name, label, empty, resourceGroups[name]); + return makeGroup(slug, departments, name, label, empty, resourceGroups[name], index); }, ); }; diff --git a/assets/js/components/department-budgets/DeptSearch/scripts.jsx b/assets/js/components/department-budgets/DeptSearch/scripts.jsx index a35e5ae69..e3e6fed87 100644 --- a/assets/js/components/department-budgets/DeptSearch/scripts.jsx +++ b/assets/js/components/department-budgets/DeptSearch/scripts.jsx @@ -1,4 +1,5 @@ -import { h, Component, render } from 'preact'; +import React from 'react'; +import ReactDOM from 'react-dom'; import decodeHtmlEntities from './../../../utilities/js/helpers/decodeHtmlEntities.js'; import updateQs from './../../../utilities/js/helpers/updateQs.js'; import { DeptSearch, makeGroups } from './index.jsx'; @@ -6,7 +7,7 @@ import filterResults from './partials/filterResults.js'; import fetchWrapper from './../../../utilities/js/helpers/fetchWrapper.js'; import { resourcesUrl, resultsToResources, initialResourceGroups } from './../../GovernmentResources/governmentResourcesData.js'; -class DeptSearchContainer extends Component { +class DeptSearchContainer extends React.Component { constructor(props) { super(props); const filters = { @@ -109,7 +110,7 @@ function scripts() { const { sphere, province, phrase } = window.vulekamali.qs; - render( + ReactDOM.render( , component, ); diff --git a/assets/js/components/department-budgets/IntroSection/index.jsx b/assets/js/components/department-budgets/IntroSection/index.jsx index 8efd53bbe..0abc1c900 100644 --- a/assets/js/components/department-budgets/IntroSection/index.jsx +++ b/assets/js/components/department-budgets/IntroSection/index.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; export default function IntroSection({ innerHtml, open, setOpen, parentAction, triggered }) { diff --git a/assets/js/components/department-budgets/IntroSection/scripts.jsx b/assets/js/components/department-budgets/IntroSection/scripts.jsx index bed3eb408..5f991094a 100644 --- a/assets/js/components/department-budgets/IntroSection/scripts.jsx +++ b/assets/js/components/department-budgets/IntroSection/scripts.jsx @@ -1,8 +1,9 @@ -import { h, render, Component } from 'preact'; +import React from 'react'; +import ReactDOM from 'react-dom'; import IntroSection from './index.jsx'; -class IntroSectionContainer extends Component { +class IntroSectionContainer extends React.Component { constructor(props) { super(props); @@ -56,14 +57,13 @@ function scripts() { const node = nodes[i]; const innerHtml = node.getElementsByClassName('js-content')[0].innerHTML; - render( + ReactDOM.render( , - node.parentNode, node, ); } + } export default scripts(); - diff --git a/assets/js/components/department-budgets/PerformanceIndicators/indicator-card.js b/assets/js/components/department-budgets/PerformanceIndicators/indicator-card.js new file mode 100644 index 000000000..4513b3909 --- /dev/null +++ b/assets/js/components/department-budgets/PerformanceIndicators/indicator-card.js @@ -0,0 +1,521 @@ +import ReactDOM from "react-dom"; +import React, {Component} from "react"; +import {Button, Card, Grid, Tooltip} from "@material-ui/core"; +import {scaleLinear, scaleBand} from "d3-scale"; +import {create} from "d3-selection"; + +class IndicatorCard extends Component { + constructor(props) { + super(props); + + this.resizeObserver = null; + + this.state = { + indicator: props.data, + selectedQuarter: this.findLatestQuarter(props.data), + selectedPeriodType: props.data.frequency === 'annually' ? 'annual' : 'quarter', + selectedYear: props.financialYear, // current year in default + previousYearsIndicators: props.previousYearsIndicators, + financialYear: props.financialYear + } + } + + componentDidMount() { + this.handleObservers(); + + if (this.state.indicator == null) { + return; + } + + if (this.state.indicator.frequency === 'annually') { + this.handleAnnualCharts(); + } else { + this.handleQuarterlyCharts(); + } + } + + componentDidUpdate(prevProps, prevState, snapshot) { + if (prevState.selectedPeriodType !== this.state.selectedPeriodType && this.state.selectedPeriodType === 'annual') { + this.handleAnnualCharts(); + } + + if (this.props.previousYearsIndicators !== this.state.previousYearsIndicators) { + this.setState({ + ...this.state, + previousYearsIndicators: this.props.previousYearsIndicators + }); + + if (this.state.indicator.frequency === 'annually') { + this.handleAnnualCharts(); + } + } + } + + findLatestQuarter(data) { + if (data['q4_actual_output'].trim() !== '') { + return 4; + } else if (data['q3_actual_output'].trim() !== '') { + return 3; + } else if (data['q2_actual_output'].trim() !== '') { + return 2; + } else { + return 1; + } + } + + handleObservers() { + const ps = document.querySelectorAll('.output-text-container .output-text'); + const padding = 24; + if (this.resizeObserver === null) { + this.resizeObserver = new ResizeObserver(entries => { + for (let entry of entries) { + entry.target.parentElement.classList[entry.target.scrollHeight * 0.95 > entry.contentRect.height + padding ? 'add' : 'remove']('read-more-visible'); + } + }) + } + + ps.forEach(p => { + this.resizeObserver.observe(p); + }) + } + + expandOutput(event) { + const ele = event.target.parentElement.querySelectorAll('.output-text')[0] + if (ele != null) { + ele.style['-webkit-box-orient'] = 'unset'; + } + } + + handleQuarterSelection(newVal) { + this.setState({ + ...this.state, + selectedQuarter: newVal, + selectedPeriodType: 'quarter' + }); + } + + handlePeriodTypeSelection(newVal) { + this.setState({ + ...this.state, + selectedPeriodType: newVal + }); + } + + renderQuarterSelection() { + return ( + + + + + + + + ) + } + + isNumeric(str) { + if (typeof str != 'string') { + return false + } + + return !isNaN(str) && !isNaN(parseFloat(str)); + } + + getIndicatorQuarterMax() { + let valuesArr = []; + for (let i = 1; i <= 4; i++) { + const {target, actual} = this.getQuarterTargetAndActual(i); + valuesArr.push(this.isNumeric(target) ? parseFloat(target) : 0); + valuesArr.push(this.isNumeric(actual) ? parseFloat(actual) : 0); + } + + return Math.max(...valuesArr); + } + + getIndicatorAnnualMax(financialYear) { + let valuesArr = []; + for (let i = 3; i >= 0; i--) { + const {target, actual} = this.getAnnualTargetAndActual(financialYear); + valuesArr.push(this.isNumeric(target) ? parseFloat(target) : 0); + valuesArr.push(this.isNumeric(actual) ? parseFloat(actual) : 0); + } + + return Math.max(...valuesArr); + } + + getQuarterKeyValue(appix, annualAppix, quarter) { + const prefix = this.state.selectedPeriodType === 'annual' ? '' : 'q'; + const finalAppix = this.state.selectedPeriodType === 'annual' ? annualAppix : appix; + const key = `${prefix}${quarter}_${finalAppix}`; + if (this.state.selectedPeriodType === 'annual' && this.state.selectedYear !== this.state.financialYear) { + const indicator = this.state.previousYearsIndicators.filter(x => x.financialYear === this.state.selectedYear)[0].indicator; + return indicator == null ? null : indicator[key]; + } else { + return this.state.indicator[key]; + } + } + + renderQuarterKeyValue(appix, annualAppix, quarter = this.state.selectedQuarter) { + const val = this.getQuarterKeyValue(appix, annualAppix, quarter); + const elem = (val == null || val.trim() === '') ? + Data not yet available : {val} + + return elem; + } + + getQuarterKeyText(appix, annualAppix) { + const prefix = this.state.selectedPeriodType === 'annual' ? `${this.state.selectedYear} ANNUAL` : 'QUARTER'; + const finalAppix = this.state.selectedPeriodType === 'annual' ? annualAppix : appix; + const quarter = this.state.selectedPeriodType === 'annual' ? '' : this.state.selectedQuarter; + return `${prefix} ${quarter} ${finalAppix}`; + } + + createChart(data, indicatorMax) { + const width = 400; + const height = 400; + const margin = {top: 0, right: 0, bottom: 0, left: 0}; + + const x = scaleBand() + .domain(data.map((d) => d.quarter)) + .rangeRound([margin.left, width - margin.right]) + .padding(0); + + const y = scaleLinear() + .domain([0, indicatorMax]) //max value + .range([height - margin.bottom, margin.top]); + + const svg = create("svg").attr("viewBox", [0, 0, width, height]); + + // bar + svg + .append("g") + .attr("fill", "#f59e46") + .selectAll("rect") + .data(data) + .join("rect") + .attr("x", (d) => x(d.quarter)) + .attr("y", (d) => y(d.actual)) + .attr("height", (d) => y(0) - y(d.actual)) + .attr("width", x.bandwidth()); + + // dashed line + svg.append('line') + .data(data) + .attr('x1', margin.left) + .attr('x2', width) + .attr('y1', (d) => y(d.target)) + .attr('y2', (d) => y(d.target)) + .attr('stroke-width', 10) + .style('stroke-dasharray', '8') + .style('stroke', 'rgba(0, 0, 0, 0.7)') + + return svg.node(); + } + + createUnavailableChartIndicator(quarter, nonNumeric) { + const svgElement = + + + + return ( +
    + + {svgElement} + +
    + ) + } + + getQuarterTargetAndActual(quarter) { + const target_init = this.getQuarterKeyValue('target', 'target', quarter); + const actual_init = this.getQuarterKeyValue('actual_output', 'audited_output', quarter); + const target = target_init == null ? 0 : target_init.replace('%', '').trim(); + const actual = actual_init == null ? 0 : actual_init.replace('%', '').trim(); + + return {target, actual}; + } + + handleQuarterChart(quarter) { + const ctx = document.getElementById(`chart-quarter-${this.state.indicator.id}-${quarter}`); + const {target, actual} = this.getQuarterTargetAndActual(quarter); + const bothNumeric = this.isNumeric(target) && this.isNumeric(actual); + + if (bothNumeric) { + // show chart + let values = [{ + quarter: `Q${quarter}`, + actual: parseFloat(actual), + target: parseFloat(target) + }]; + let indicatorMax = this.getIndicatorQuarterMax(); + + const chart = this.createChart(values, indicatorMax); + ctx.appendChild(chart) + } else { + // chart is not available + const nonNumeric = !this.isNumeric(actual) ? 'actual output' : 'target'; + const parentDiv = this.createUnavailableChartIndicator(`Q${quarter}`, nonNumeric); + + ReactDOM.render(parentDiv, ctx); + } + } + + getAnnualTargetAndActual(financialYear) { + const indicator = this.state.previousYearsIndicators.filter(x => x.financialYear === financialYear)[0].indicator; + const target = indicator == null ? 0 : indicator['annual_target'].replace('%', '').trim(); + const actual = indicator == null ? 0 : indicator['annual_audited_output'].replace('%', '').trim(); + + return {target, actual}; + } + + handleAnnualCharts() { + for (let i = 1; i <= 4; i++) { + if (this.state.previousYearsIndicators[i - 1] != null) { + const financialYear = this.state.previousYearsIndicators[i - 1].financialYear; + const ctx = document.getElementById(`chart-annual-${this.state.indicator.id}-${i}`); + if (!ctx.hasChildNodes()) { + const {target, actual} = this.getAnnualTargetAndActual(financialYear); + const bothNumeric = this.isNumeric(target) && this.isNumeric(actual); + + if (bothNumeric) { + // show chart + let values = [{ + quarter: financialYear, + actual: parseFloat(actual), + target: parseFloat(target) + }] + + let indicatorMax = this.getIndicatorAnnualMax(financialYear); + + const chart = this.createChart(values, indicatorMax); + ctx.appendChild(chart) + } else { + // chart is not available + const nonNumeric = !this.isNumeric(actual) ? 'actual output' : 'target'; + const parentDiv = this.createUnavailableChartIndicator(financialYear, nonNumeric); + + ReactDOM.render(parentDiv, ctx); + } + } + } + } + } + + handleQuarterlyCharts() { + for (let i = 1; i <= 4; i++) { + this.handleQuarterChart(i); + } + } + + getQuarterChartContainer(q) { + return ( + { + this.setState({ + ...this.state, + selectedQuarter: q + }) + }} + > +
    + + ) + } + + getAnnualChartContainer(q) { + if (this.state.previousYearsIndicators.length <= q - 1 || this.state.previousYearsIndicators[q - 1] == null) { + return; + } + + const selectedYear = this.state.previousYearsIndicators[q - 1].financialYear; + return ( + { + this.setState({ + ...this.state, + selectedYear: selectedYear + }) + }} + > +
    + + ) + } + + renderChartContainerColumns() { + return ( + [1, 2, 3, 4].map(q => { + return ([ + this.getQuarterChartContainer(q), + this.getAnnualChartContainer(q) + ]) + }) + ) + } + + renderChartContainers() { + return ( + + {this.renderChartContainerColumns()} + + {this.state.selectedPeriodType === 'annual' && this.state.previousYearsIndicators[0] !== undefined ? this.state.previousYearsIndicators[0].financialYear : 'Q1'} + + + {this.state.selectedPeriodType === 'annual' && this.state.previousYearsIndicators[1] !== undefined ? this.state.previousYearsIndicators[1].financialYear : 'Q2'} + + + {this.state.selectedPeriodType === 'annual' && this.state.previousYearsIndicators[2] !== undefined ? this.state.previousYearsIndicators[2].financialYear : 'Q3'} + + + {this.state.selectedPeriodType === 'annual' && this.state.previousYearsIndicators[3] !== undefined ? this.state.previousYearsIndicators[3].financialYear : 'Q4'} + + + ) + } + + renderCard() { + return ( + + +

    TYPE: {this.state.indicator.type}

    +

    {this.state.indicator.indicator_name}

    + {this.renderQuarterSelection()} + +

    {this.getQuarterKeyText('TARGET', 'TARGET')}:

    +
    +
    + {this.renderQuarterKeyValue('target', 'target', this.state.selectedPeriodType === 'annual' ? 'annual' : this.state.selectedQuarter)} +
    +
    + + + +
    +
    +     + +

    {this.getQuarterKeyText('ACTUAL OUTPUT', 'AUDITED PERFORMANCE')}:

    +
    +
    + {this.renderQuarterKeyValue('actual_output', 'audited_output', this.state.selectedPeriodType === 'annual' ? 'annual' : this.state.selectedQuarter)} +
    + +
    +     + +

    + {this.getQuarterKeyText('PERFORMANCE', 'PERFORMANCE')}: + + + + + + + Actual + + + + + + + Target + +

    +
    + {this.renderChartContainers()} +
    +     +     +     + ) + } + + render() { + return this.renderCard() + } +} + +export default IndicatorCard; \ No newline at end of file diff --git a/assets/js/components/department-budgets/PerformanceIndicators/programme.js b/assets/js/components/department-budgets/PerformanceIndicators/programme.js new file mode 100644 index 000000000..b60f08802 --- /dev/null +++ b/assets/js/components/department-budgets/PerformanceIndicators/programme.js @@ -0,0 +1,104 @@ +import React, {Component} from "react"; +import {Button, Grid, Paper} from "@material-ui/core"; +import IndicatorCard from "./indicator-card"; + +class Programme extends Component { + constructor(props) { + super(props); + + this.state = { + open: false, + triggered: false, + programme: props.data, + previousYearsProgrammes: props.previousYearsProgrammes, + financialYear: props.financialYear + } + } + + componentDidUpdate(prevProps, prevState, snapshot) { + if (this.props.previousYearsProgrammes !== this.state.previousYearsProgrammes) { + this.setState({ + ...this.state, + previousYearsProgrammes: this.props.previousYearsProgrammes + }); + } + } + + setOpen() { + this.setState({ + ...this.state, open: !this.state.open + }); + } + + renderIndicatorCards(programme) { + return programme.visibleIndicators.map((indicator) => { + let prevArr = this.state.previousYearsProgrammes.map(item => { + return { + financialYear: item.financialYear, + indicator: item.programme == null ? null : item.programme.allIndicators.filter(p => p.indicator_name === indicator.indicator_name)[0] + }; + }) + + return () + }) + } + + renderReadMoreButton() { + if (this.state.open) { + return; + } + + return ( +
    +
    +
    + +
    +
    + ) + } + + renderProgramme() { + return ( +
    +

    {this.state.programme.name}

    + + {this.renderIndicatorCards(this.state.programme)} + + + + +
    + {this.renderReadMoreButton()} +     ) + } + + render() { + return (
    {this.renderProgramme()}
    ); + } +} + +export default Programme; \ No newline at end of file diff --git a/assets/js/components/department-budgets/PerformanceIndicators/scripts.jsx b/assets/js/components/department-budgets/PerformanceIndicators/scripts.jsx new file mode 100644 index 000000000..2e2784b96 --- /dev/null +++ b/assets/js/components/department-budgets/PerformanceIndicators/scripts.jsx @@ -0,0 +1,316 @@ +import ReactDOM from "react-dom"; +import React, {Component} from "react"; +import Programme from "./programme"; +import fetchWrapper from "../../../utilities/js/helpers/fetchWrapper"; +import decodeHtmlEntities from "../../../utilities/js/helpers/decodeHtmlEntities"; +import {Button, CircularProgress, Dialog, Grid} from "@material-ui/core"; + +class PerformanceIndicators extends Component { + constructor(props) { + super(props); + + this.state = { + department: props.department, + financialYear: props.year, + sphere: props.sphere, + government: props.government, + previousYears: props.previousYears, + programmes: [], + previousYearsProgrammes: [], + pageCount: 3, + isLoading: true + }; + } + + componentDidMount() { + this.fetchAPIData(); + } + + fetchPreviousYearsAPIData() { + this.state.previousYears.forEach((fy, index) => { + this.fetchAPIDataRecursive(1, [], fy) + .then((items) => { + let arr = this.state.previousYearsProgrammes; + arr[index] = { + financialYear: fy, + programmes: this.extractProgrammeData(items) + } + + this.setState({ + ...this.state, + previousYearsProgrammes: arr + }); + }) + }) + } + + fetchAPIData() { + this.fetchAPIDataRecursive(1, [], this.state.financialYear) + .then((items) => { + this.setState({ + ...this.state, + isLoading: false, + programmes: this.extractProgrammeData(items) + }); + + this.fetchPreviousYearsAPIData(); + }) + } + + extractProgrammeData(items) { + let programmes = [...new Set(items.map(x => x.programme_name))]; + let data = []; + programmes.forEach(p => { + const allIndicators = items.filter(x => x.programme_name === p); + data.push({ + name: p, + visibleIndicators: allIndicators.slice(0, this.state.pageCount), + allIndicators: allIndicators + }) + }) + + return data; + } + + fetchAPIDataRecursive(page = 1, allItems = [], financialYear) { + return new Promise((resolve, reject) => { + const pageQuery = `page=${page}`; + const departmentQuery = `department__name=${encodeURI(this.state.department)}`; + const financialYearQuery = `department__government__sphere__financial_year__slug=${financialYear}`; + const baseUrl = `../../../../../performance/api/v1/eqprs/`; + let url = `${baseUrl}?${pageQuery}&${departmentQuery}&${financialYearQuery}`; + + fetchWrapper(url) + .then((response) => { + let newArr = allItems.concat(response.results.items); + if (response.next === null) { + resolve(newArr); + } else { + this.fetchAPIDataRecursive(page + 1, newArr, financialYear) + .then((items) => { + resolve(items); + }); + } + }) + .catch((errorResult) => console.warn(errorResult)); + }) + } + + handleShowMore(currentProgramme) { + let programmes = this.state.programmes.map(programme => { + if (programme.name === currentProgramme.name) { + programme.visibleIndicators = programme.allIndicators.slice(0, programme.visibleIndicators.length + this.state.pageCount); + } + return programme; + }) + + this.setState({ + ...this.state, + programmes: programmes + }) + } + + renderProgrammes() { + return this.state.programmes.map((programme, index) => { + let prevArr = this.state.previousYearsProgrammes.map(item => { + return { + financialYear: item.financialYear, + programme: item.programmes.filter(p => p.name === programme.name)[0] + }; + }) + + return ( + this.handleShowMore(programme)} + previousYearsProgrammes={prevArr} + financialYear={this.state.financialYear} + /> + ) + }) + } + + renderLoadingState() { + if (!this.state.isLoading) { + return + } + const tableContainer = document.getElementsByClassName('js-initYearSelect')[0]; + const gifWidth = 40; + const marginLeftVal = (tableContainer.clientWidth - gifWidth) / 2; + + return ( +
    + +
    + ) + } + + render() { + return (
    + {this.renderProgrammes()} + {this.renderLoadingState()} +
    ); + } +} + +class PerformanceIndicatorsContainer extends Component { + constructor(props) { + super(props); + + this.state = { + dataDisclaimerAcknowledged: false, + modalOpen: false, + department: props.department, + year: props.year, + sphere: props.sphere, + government: props.government, + previousYears: props.previousYears + } + } + + componentDidMount() { + this.checkForLocalStorage(); + } + + checkForLocalStorage() { + const ack = localStorage.getItem('data-disclaimer-acknowledged'); + this.setState({ + ...this.state, + dataDisclaimerAcknowledged: ack === 'true', + modalOpen: ack !== 'true' + }) + } + + handleStorage() { + localStorage.setItem('data-disclaimer-acknowledged', 'true'); + this.setState({ + ...this.state, + dataDisclaimerAcknowledged: true, + modalOpen: false + }) + } + + renderNavigateButtons() { + const baseUrl = '../../../../../performance'; + const sphereQuery = `department__government__sphere__name=${encodeURI(this.state.sphere)}`; + const governmentQuery = `department__government__name=${encodeURI(this.state.government)}`; + const yearQuery = `department__government__sphere__financial_year__slug=${this.state.year}`; + const departmentQuery = `department__name=${this.state.department}`; + return ( + + + + + ) + } + + renderModal() { + return ( + document.getElementById('js-initPerformanceIndicators')} + style={{position: 'absolute'}} + BackdropProps={{ + style: {position: 'absolute'} + }} + disableAutoFocus={true} + disableEnforceFocus={true} + className={'performance-modal'} + > + + Data disclaimer + + + The Quarterly Performance Reporting (QPR) data (other than the Annual audited output field) + is pre-audited non financial data. This data is approved by the accounting officer of the + relevant organ of state before publication. + + +     Learn more about these performance indicators. + + + + + + ) + } + + render() { + return (
    +

    Indicators of performance

    + + {this.renderNavigateButtons()} + {this.renderModal()} +
    ); + } +} + +function scripts() { + const nodes = document.getElementsByClassName('js-initYearSelect'); + const nodesArray = [...nodes]; + let previousYears = []; + + if (nodesArray.length > 0) { + const jsonData = JSON.parse(decodeHtmlEntities(nodes[0].getAttribute('data-json'))).data; + jsonData.forEach((d) => { + previousYears.push(d.id) + }) + } + + const parent = document.getElementById('js-initPerformanceIndicators'); + if (parent) { + const departmentName = parent.getAttribute('data-department'); + const financialYear = parent.getAttribute('data-year'); + const sphere = parent.getAttribute('data-sphere'); + const government = parent.getAttribute('data-government'); + ReactDOM.render(, parent) + } +} + + +export default scripts(); \ No newline at end of file diff --git a/assets/js/components/header-and-footer/Footer/index.html b/assets/js/components/header-and-footer/Footer/index.html index 873238ec3..c82103c46 100644 --- a/assets/js/components/header-and-footer/Footer/index.html +++ b/assets/js/components/header-and-footer/Footer/index.html @@ -54,17 +54,11 @@

    Connect with us

  • -
    Subscribe to our newsletter
    -
    -
    - -
    -
    - -
    -
    + + Subscribe to our newsletter +
    • -
  • Newsletter archive
    • +
  • Newsletter archive
    • diff --git a/assets/js/components/header-and-footer/Modals/index.jsx b/assets/js/components/header-and-footer/Modals/index.jsx index 16a3ea188..573af8aa7 100644 --- a/assets/js/components/header-and-footer/Modals/index.jsx +++ b/assets/js/components/header-and-footer/Modals/index.jsx @@ -1,5 +1,4 @@ -import { h } from 'preact'; -import CssTransitionGroup from 'preact-css-transition-group'; +import React from 'react'; import Icon from './../../Icon/index.jsx'; @@ -28,13 +27,7 @@ export default function Modal({ markup, title, closeModal }) { return (
    - - {buildModal()} - + {buildModal()}
    ); } diff --git a/assets/js/components/header-and-footer/Modals/scripts.jsx b/assets/js/components/header-and-footer/Modals/scripts.jsx index f7674638b..4c90ac33f 100644 --- a/assets/js/components/header-and-footer/Modals/scripts.jsx +++ b/assets/js/components/header-and-footer/Modals/scripts.jsx @@ -1,10 +1,11 @@ -import { h, render, Component } from 'preact'; +import React from 'react'; +import ReactDOM from 'react-dom'; import { getState, subscribe } from './../../../reduxStore.js'; import Modals from './index.jsx'; import createComponents from './../../../utilities/js/helpers/createComponents.js'; -class ModalsContainer extends Component { +class ModalsContainer extends React.Component { constructor(props) { super(props); @@ -60,7 +61,7 @@ class ModalsContainer extends Component { function scripts() { - const createInstance = node => render(, node); + const createInstance = node => ReactDOM.render(, node); createComponents('Modals', createInstance); } diff --git a/assets/js/components/header-and-footer/Search/index.jsx b/assets/js/components/header-and-footer/Search/index.jsx index 119242347..976b40b68 100644 --- a/assets/js/components/header-and-footer/Search/index.jsx +++ b/assets/js/components/header-and-footer/Search/index.jsx @@ -1,8 +1,7 @@ -import { h } from 'preact'; import PropTypes from 'prop-types'; import FormArea from './partials/FormArea.jsx'; import ResultsArea from './partials/ResultsArea.jsx'; - +import React from 'react'; export default function SearchMarkup(props) { const { @@ -61,7 +60,6 @@ export default function SearchMarkup(props) { } SearchMarkup.propTypes = { - count: PropTypes.string.isRequired, currentKeywords: PropTypes.string.isRequired, error: PropTypes.bool.isRequired, focus: PropTypes.bool.isRequired, diff --git a/assets/js/components/header-and-footer/Search/partials/FormArea.jsx b/assets/js/components/header-and-footer/Search/partials/FormArea.jsx index 9c813ff45..a0897d0ba 100644 --- a/assets/js/components/header-and-footer/Search/partials/FormArea.jsx +++ b/assets/js/components/header-and-footer/Search/partials/FormArea.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import PropTypes from 'prop-types'; import Icon from './Icon.jsx'; @@ -18,7 +18,7 @@ export default function FormArea({ setFocus, currentKeywords, selectedYear }) { name="search" onFocus={addFocus} placeholder="Search vulekamali" - value={currentKeywords} + defaultValue={currentKeywords} />
    diff --git a/assets/js/components/header-and-footer/Search/partials/Icon.jsx b/assets/js/components/header-and-footer/Search/partials/Icon.jsx index b9e7580a1..d47c98de7 100644 --- a/assets/js/components/header-and-footer/Search/partials/Icon.jsx +++ b/assets/js/components/header-and-footer/Search/partials/Icon.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; export default function Icon() { diff --git a/assets/js/components/header-and-footer/Search/partials/List.jsx b/assets/js/components/header-and-footer/Search/partials/List.jsx index ea9f16e53..27aba15ff 100644 --- a/assets/js/components/header-and-footer/Search/partials/List.jsx +++ b/assets/js/components/header-and-footer/Search/partials/List.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import PropTypes from 'prop-types'; diff --git a/assets/js/components/header-and-footer/Search/partials/ResultsArea.jsx b/assets/js/components/header-and-footer/Search/partials/ResultsArea.jsx index 38fd29893..1e21b85c6 100644 --- a/assets/js/components/header-and-footer/Search/partials/ResultsArea.jsx +++ b/assets/js/components/header-and-footer/Search/partials/ResultsArea.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import PropTypes from 'prop-types'; import List from './List.jsx'; diff --git a/assets/js/components/header-and-footer/Search/scripts.jsx b/assets/js/components/header-and-footer/Search/scripts.jsx index 719123e50..b1d92dbe7 100644 --- a/assets/js/components/header-and-footer/Search/scripts.jsx +++ b/assets/js/components/header-and-footer/Search/scripts.jsx @@ -1,12 +1,13 @@ import { ga } from 'react-ga'; -import { h, render, Component } from 'preact'; +import ReactDOM from 'react-dom'; +import React from 'react'; import PropTypes from 'prop-types'; import queryString from 'query-string'; import Search from './index.jsx'; import removePunctuation from '../../../utilities/js/helpers/removePunctuation.js'; -class SearchContainer extends Component { +class SearchContainer extends React.Component { constructor(props) { super(props); @@ -94,7 +95,7 @@ function scripts() { } // Initialise Search Preact App - render( + ReactDOM.render( , component, ); diff --git a/assets/js/components/header-and-footer/YearSelect/index.jsx b/assets/js/components/header-and-footer/YearSelect/index.jsx index 1b9d4e5ad..bcbb10f26 100644 --- a/assets/js/components/header-and-footer/YearSelect/index.jsx +++ b/assets/js/components/header-and-footer/YearSelect/index.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import queryString from 'query-string'; import Tooltip from './../../universal/Tooltip/index.jsx'; @@ -10,7 +10,7 @@ const navToYearPage = (event, page) => { export default function YearSelectMarkup({ sticky, jsonData, updateNode, tooltip, open, updateItem, search, loading, year, newYear }) { - const items = jsonData.map((data) => { + const items = jsonData.map((data, index) => { const Tag = data.active || data.direct === false ? 'span' : 'a'; const toggleOpen = () => updateItem('open', !open); @@ -19,6 +19,7 @@ export default function YearSelectMarkup({ sticky, jsonData, updateNode, tooltip
  • { const jsonData = JSON.parse(decodeHtmlEntities(nodes[i].getAttribute('data-json'))).data; - render( + ReactDOM.render( , - nodes[i].parentNode, nodes[i], ); }); diff --git a/assets/js/components/homepage/HomeChart/index.jsx b/assets/js/components/homepage/HomeChart/index.jsx index 7e58b36da..6859fde2a 100644 --- a/assets/js/components/homepage/HomeChart/index.jsx +++ b/assets/js/components/homepage/HomeChart/index.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import ResponsiveChart from './../../universal/ResponsiveChart/index.jsx'; import ValueBlocks from './../ValueBlocks/index.jsx'; diff --git a/assets/js/components/homepage/HomeChart/scripts.jsx b/assets/js/components/homepage/HomeChart/scripts.jsx index c68638346..5d529ef17 100644 --- a/assets/js/components/homepage/HomeChart/scripts.jsx +++ b/assets/js/components/homepage/HomeChart/scripts.jsx @@ -1,10 +1,11 @@ -import { h, render, Component } from 'preact'; +import React from 'react'; +import ReactDOM from 'react-dom'; import DebounceFunction from './../../../utilities/js/helpers/DebounceFunction.js'; import getProp from './../../../utilities/js/helpers/getProp.js'; import HomeChart from './index.jsx'; -class HomeChartContainer extends Component { +class HomeChartContainer extends React.Component { constructor(props) { super(props); @@ -111,7 +112,7 @@ function scripts() { const hasNull = type === 'revenue' ? false : calcIfHasNullTotalBudget(rawValues.data); const items = Object.assign(...normaliseData(rawValues.data, hasNull, type, yearString)); - render( + ReactDOM.render( , node, ); diff --git a/assets/js/components/homepage/Revenue/index.jsx b/assets/js/components/homepage/Revenue/index.jsx index a6c332a13..6b6f845de 100644 --- a/assets/js/components/homepage/Revenue/index.jsx +++ b/assets/js/components/homepage/Revenue/index.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import ValueBlocks from './../ValueBlocks/index.jsx'; diff --git a/assets/js/components/homepage/Revenue/scripts.jsx b/assets/js/components/homepage/Revenue/scripts.jsx index 374bdfb06..db9ef9628 100644 --- a/assets/js/components/homepage/Revenue/scripts.jsx +++ b/assets/js/components/homepage/Revenue/scripts.jsx @@ -1,4 +1,5 @@ -import { h, render } from 'preact'; +import React from 'react'; +import ReactDOM from 'react-dom'; import Revenue from './index.jsx'; import decodeHtmlEntities from './../../../utilities/js/helpers/decodeHtmlEntities.js'; @@ -12,7 +13,7 @@ function scripts() { const values = JSON.parse(decodeHtmlEntities(component.getAttribute('data-values'))); const year = component.getAttribute('data-year'); - render( + ReactDOM.render( , component, ); diff --git a/assets/js/components/homepage/Showcase/scripts.jsx b/assets/js/components/homepage/Showcase/scripts.jsx new file mode 100644 index 000000000..9b49d2906 --- /dev/null +++ b/assets/js/components/homepage/Showcase/scripts.jsx @@ -0,0 +1,104 @@ +import ReactDOM from "react-dom"; +import React, {Component} from "react"; +import {Card, CardContent, CardMedia, Grid} from "@material-ui/core"; +import ForwardArrow from "@material-ui/icons/ArrowForward"; + +class Showcase extends Component { + constructor(props) { + super(props); + + this.state = { + features: JSON.parse(document.getElementById('showcase-items-data').textContent) + } + } + + renderCTA(type, text, link, enabled) { + if (!enabled) { + return; + } + + if (type === "primary") { + return ( +

    + + {text} + + + + +

    + ) + } else if (type === "secondary") { + return ( +

    {text}

    + ) + } + } + + render() { + return ( + {this.state.features.map((feature, index) => { + return ( + + + + + + + + + {feature.name} +

    {feature.description}

    + {this.renderCTA('primary', feature.cta_text_1, feature.cta_link_1, true)} + {this.renderCTA(feature.second_cta_type, + feature.cta_text_2, + feature.cta_link_2, + (feature.cta_text_2 != null && feature.cta_text_2.trim() != ""))} +     +     +     +     +     + ) + })} +     ) + } +} + +function scripts() { + const parent = document.getElementById('js-initShowcase'); + if (parent) { + ReactDOM.render(, parent) + } +} + +export default scripts(); diff --git a/assets/js/components/homepage/SignUpBox/index.html b/assets/js/components/homepage/SignUpBox/index.html deleted file mode 100644 index 330216889..000000000 --- a/assets/js/components/homepage/SignUpBox/index.html +++ /dev/null @@ -1,9 +0,0 @@ -
    -
    Subscribe to our newsletter
    - -
    \ No newline at end of file diff --git a/assets/js/components/homepage/ValueBlocks/index.jsx b/assets/js/components/homepage/ValueBlocks/index.jsx index be9a85f95..e94fb88c1 100644 --- a/assets/js/components/homepage/ValueBlocks/index.jsx +++ b/assets/js/components/homepage/ValueBlocks/index.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import trimValues from './../../../utilities/js/helpers/trimValues.js'; diff --git a/assets/js/components/homepage/ValueBlocks/scripts.jsx b/assets/js/components/homepage/ValueBlocks/scripts.jsx index c1f1c9f72..d23a31553 100644 --- a/assets/js/components/homepage/ValueBlocks/scripts.jsx +++ b/assets/js/components/homepage/ValueBlocks/scripts.jsx @@ -1,4 +1,5 @@ -import { h, render } from 'preact'; +import React from 'react'; +import ReactDOM from 'react-dom'; import ValueBlocks from './index.jsx'; import decodeHtmlEntities from './../../../utilities/js/helpers/decodeHtmlEntities.js'; @@ -10,7 +11,7 @@ function scripts() { const component = componentList[i]; const items = JSON.parse(decodeHtmlEntities(component.getAttribute('data-values'))); - render( + ReactDOM.render( , component, ); diff --git a/assets/js/components/learning-centre/Glossary/index.jsx b/assets/js/components/learning-centre/Glossary/index.jsx index a4037ed2b..6971adef0 100644 --- a/assets/js/components/learning-centre/Glossary/index.jsx +++ b/assets/js/components/learning-centre/Glossary/index.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import Controls from './partials/Controls.jsx'; import List from './partials/List.jsx'; diff --git a/assets/js/components/learning-centre/Glossary/partials/Controls.jsx b/assets/js/components/learning-centre/Glossary/partials/Controls.jsx index 21cc94bdc..deca91eb0 100644 --- a/assets/js/components/learning-centre/Glossary/partials/Controls.jsx +++ b/assets/js/components/learning-centre/Glossary/partials/Controls.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; export default function Controls({ currentPhrase, currentItems, changePhrase }) { diff --git a/assets/js/components/learning-centre/Glossary/partials/List.jsx b/assets/js/components/learning-centre/Glossary/partials/List.jsx index 1564f4ad7..9f88703b0 100644 --- a/assets/js/components/learning-centre/Glossary/partials/List.jsx +++ b/assets/js/components/learning-centre/Glossary/partials/List.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; export default function List({ currentPhrase, currentItems }) { diff --git a/assets/js/components/learning-centre/Glossary/scripts.jsx b/assets/js/components/learning-centre/Glossary/scripts.jsx index 5d70ed4ec..7d60c2bef 100644 --- a/assets/js/components/learning-centre/Glossary/scripts.jsx +++ b/assets/js/components/learning-centre/Glossary/scripts.jsx @@ -1,4 +1,5 @@ -import { h, render, Component } from 'preact'; +import React from 'react'; +import ReactDOM from 'react-dom'; import { parse } from 'query-string'; import Glossary from './index.jsx'; import glossary from './../../../../../_data/glossary.json'; @@ -10,7 +11,7 @@ import wrapStringPhrases from './../../../utilities/js/helpers/wrapStringPhrases const { items: glossaryObject } = glossary; -class GlossaryContainer extends Component { +class GlossaryContainer extends React.Component { constructor(props) { super(props); @@ -100,7 +101,7 @@ function scripts() { if (nodes.length > 0) { for (let i = 0; i < nodes.length; i++) { - render(, nodes[i]); + ReactDOM.render(, nodes[i]); } } } diff --git a/assets/js/components/performance/Table/index.html b/assets/js/components/performance/Table/index.html new file mode 100644 index 000000000..2c01d0374 --- /dev/null +++ b/assets/js/components/performance/Table/index.html @@ -0,0 +1,5 @@ +{% load define_action %} +
    +
    diff --git a/assets/js/components/performance/Table/scripts.jsx b/assets/js/components/performance/Table/scripts.jsx new file mode 100644 index 000000000..cab205f07 --- /dev/null +++ b/assets/js/components/performance/Table/scripts.jsx @@ -0,0 +1,711 @@ +import React, {Component} from 'react'; +import ReactDOM from 'react-dom'; +import { + FormControl, + Grid, + InputLabel, + TextField, + Paper, + Select, + Table, + TableBody, + TableCell, + TableContainer, + TableFooter, + TableHead, + TablePagination, + TableRow, + Chip, + CircularProgress, + Dialog, + MenuItem, + Button, +} from "@material-ui/core"; +import {ThemeProvider} from "@material-ui/styles"; +import {createTheme} from '@material-ui/core/styles'; +import fetchWrapper from "../../../utilities/js/helpers/fetchWrapper"; +import debounce from "lodash.debounce"; + +class TabularView extends Component { + constructor(props) { + super(props); + + this.resizeObserver = null; + this.observedElements = []; + this.abortController = null; + + this.state = { + dataDisclaimerAcknowledged: false, + modalOpen: false, + rows: null, + departments: null, + financialYears: null, + frequencies: null, + governments: null, + mtsfOutcomes: null, + sectors: null, + spheres: null, + totalCount: 0, + rowsPerPage: 20, + currentPage: 0, + selectedFilters: {}, + isLoading: false, + downloadUrl: '', + excludeColumns: new Set(['id', 'department']), + titleMappings: { + 'indicator_name': 'Indicator name', + 'q1_target': 'Quarter 1 target', + 'q1_actual_output': 'Quarter 1 actual output', + 'q1_deviation_reason': 'Quarter 1 deviation reason', + 'q1_corrective_action': 'Quarter 1 corrective action', + 'q2_target': 'Quarter 2 target', + 'q2_actual_output': 'Quarter 2 actual output', + 'q2_deviation_reason': 'Quarter 2 deviation reason', + 'q2_corrective_action': 'Quarter 2 corrective action', + 'q3_target': 'Quarter 3 target', + 'q3_actual_output': 'Quarter 3 actual output', + 'q3_deviation_reason': 'Quarter 3 deviation reason', + 'q3_corrective_action': 'Quarter 3 corrective action', + 'q4_target': 'Quarter 4 target', + 'q4_actual_output': 'Quarter 4 actual output', + 'q4_deviation_reason': 'Quarter 4 deviation reason', + 'q4_corrective_action': 'Quarter 4 corrective action', + 'annual_target': 'Annual target', + 'annual_aggregate_output': 'Annual aggregate output', + 'annual_pre_audit_output': 'Annual pre-audit output', + 'annual_deviation_reason': 'Annual deviation reason', + 'annual_corrective_action': 'Annual corrective action', + 'annual_audited_output': 'Annual audited output', + 'sector': 'Sector', + 'programme_name': 'Programme name', + 'subprogramme_name': 'Subprogramme name', + 'frequency': 'Frequency', + 'type': 'Type', + 'subtype': 'Subtype', + 'mtsf_outcome': 'Mtsf outcome', + 'cluster': 'Cluster', + 'financial_year': 'Financial year', + 'department_name': 'Department name', + 'government_name': 'Government name', + 'sphere_name': 'Sphere name' + } + } + } + + componentDidMount() { + this.fetchAPIData(0); + window.addEventListener('popstate', (event) => { + this.setSelectedFiltersAndFetchAPIData(); + }) + + this.setSelectedFiltersAndFetchAPIData(); + this.checkForLocalStorage(); + } + + checkForLocalStorage() { + const ack = localStorage.getItem('data-disclaimer-acknowledged'); + this.setState({ + dataDisclaimerAcknowledged: ack === 'true', + modalOpen: ack !== 'true' + }) + } + + + setSelectedFiltersAndFetchAPIData() { + let selectedFilters = {}; + let params = new URLSearchParams(window.location.search); + for (const key of params.keys()) { + selectedFilters[key] = params.get(key); + } + + document.getElementById('frm-textSearch').value = selectedFilters['q'] === undefined ? '' : selectedFilters['q']; + + this.setState({ + ...this.state, selectedFilters: selectedFilters + }, () => { + this.fetchAPIData(0); + }) + } + + handleFilterChange(event) { + const name = event.target.name; + const value = event.target.value; + + let selectedFilters = this.state.selectedFilters; + selectedFilters[name] = value; + + let url = ''; + Object.keys(selectedFilters) + .filter(key => selectedFilters[key] !== null) + .forEach((key, i) => { + const value = selectedFilters[key]; + url += `${i === 0 ? '?' : '&'}${key}=${encodeURI(value)}`; + }) + + history.pushState(null, '', url === '' ? location.pathname : url) + + this.setState({ + ...this.state, selectedFilters: selectedFilters + }, () => { + this.fetchAPIData(0); + }) + } + + cancelAndInitAbortController() { + if (this.abortController !== null) { + // this.abortController is null on the first request + this.abortController.abort(); + } + this.abortController = new AbortController(); + + } + + fetchAPIData(pageToCall) { + this.setState({ + ...this.state, + isLoading: true + }, () => { + this.setDownloadUrl(); + this.cancelAndInitAbortController(); + + this.unobserveElements(); + + let url = `api/v1/eqprs/?page=${pageToCall + 1}`; + + // append filters + Object.keys(this.state.selectedFilters).forEach((key) => { + let value = this.state.selectedFilters[key]; + if (value !== null) { + url += `&${key}=${encodeURI(value)}`; + } + }) + + fetchWrapper(url, this.abortController) + .then((response) => { + this.setState({ + ...this.state, + currentPage: pageToCall, + rows: response.results.items, + departments: response.results.facets['department_name'], + financialYears: response.results.facets['financial_year_slug'], + frequencies: response.results.facets['frequency'], + governments: response.results.facets['government_name'], + mtsfOutcomes: response.results.facets['mtsf_outcome'], + sectors: response.results.facets['sector'], + spheres: response.results.facets['sphere_name'], + totalCount: response.count, + isLoading: false + }); + }) + .then(() => { + this.handleObservers(); + }) + .catch((errorResult) => console.warn(errorResult)); + }) + } + + setDownloadUrl() { + let url = 'performance-indicators.xlsx/'; + + // append filters + Object.keys(this.state.selectedFilters).forEach((key, index) => { + let value = this.state.selectedFilters[key]; + if (value !== null) { + let prefix = index === 0 ? '?' : '&'; + url += `${prefix}${key}=${encodeURI(value)}`; + } + }) + + this.setState({ + ...this.state, + downloadUrl: url + }); + } + + renderTableHead() { + if (this.state.rows.length > 0) { + return ( + {Object.keys(this.state.rows[0]).map((key, index) => { + if (!this.state.excludeColumns.has(key)) { + if (key === 'indicator_name') { + return ( +
    + {this.getTitleMapping(key)} +
    +     ) + } else { + return ( +
    + {this.getTitleMapping(key)} +
    +     ) + } + } + })} +     ) + } else { + return
    No matching indicators found.
    ; + } + } + + renderTableCells(row, index) { + const isAlternating = index % 2 !== 0; + return ( + {Object.keys(row).map((key, i) => { + if (!this.state.excludeColumns.has(key)) { + if (key === 'indicator_name') { + return ( +
    + {this.renderIndicatorColumn(row, index)} +
    +     ) + } else { + return ( +
    + + {row[key]} + +
    +     ) + } + } + })} +     ) + } + + handleReadMoreClick(e, i, index, text) { + if (e.target.className === 'link-button') { + const cellId = `cell_${this.state.currentPage}_${index}_${i}`; + let element = document.getElementById(cellId); + + element.innerText = text; + } + } + + getTitleMapping(key) { + const mapping = this.state.titleMappings[key]; + + return mapping === undefined ? key : mapping; + } + + renderIndicatorColumn(row, index) { + const chips = [{ + key: "financial_year", + value: row.department.government.sphere.financial_year.slug + }, { + key: "government_name", + value: row.department.government.name + }, { + key: "department_name", + value: row.department.name + }]; + return ( +
    +
    + {row['indicator_name']} +
    + { + chips.map((chip, i) => { + return ( + + ) + }) + } +
    + ) + } + + handlePageChange(event, newPage) { + this.fetchAPIData(newPage); + } + + renderPaginationAndTools() { + return ( + + + {this.renderPagination()} + + + + + + ); + } + + renderPagination() { + if (this.state.rows === null) { + // empty pagination row + return
    + } + + return ( + this.handlePageChange(event, newPage)} + SelectProps={{ + inputProps: {'aria-label': 'rows per page'}, native: true, + }} + component="div" + /> + ); + } + + renderTableContainer() { + if (this.state.rows === null) { + return
    + } + + return ( + + + + {this.renderTableHead()} + + + {this.state.rows.map((row, index) => this.renderTableCells(row, index))} + + + + + +
    +     + ) + } + + renderTable() { + const tableTheme = createTheme({ + overrides: { + MuiTablePagination: { + spacer: { + flex: 'none' + }, toolbar: { + "padding-left": "16px" + } + } + } + }); + return ( + + {this.renderPaginationAndTools()} + + {this.renderLoadingState()} + {this.renderTableContainer()} + + {this.renderPagination()} + + ); + } + + renderLoadingState() { + if (!this.state.isLoading) { + return; + } + + const tableContainer = document.getElementById('js-initTabularView'); + const gifWidth = 40; + const marginLeftVal = (tableContainer.clientWidth - gifWidth) / 2; + + return ( +
    + +
    + ) + } + + handleObservers() { + const ps = document.querySelectorAll('.performance-table-cell span'); + if (this.resizeObserver === null) { + this.resizeObserver = new ResizeObserver(entries => { + for (let entry of entries) { + entry.target.classList[entry.target.scrollHeight * 0.95 > entry.contentRect.height ? 'add' : 'remove']('truncated'); + } + }) + } + + ps.forEach(p => { + this.observedElements.push(p); + this.resizeObserver.observe(p); + }) + } + + unobserveElements() { + if (this.resizeObserver === null) { + return; + } + + this.observedElements.forEach(ele => { + this.resizeObserver.unobserve(ele); + }) + this.observedElements = []; + } + + renderSearchField() { + const debouncedHandleFilterChange = debounce((event) => this.handleFilterChange(event), 300); + const persistedEventDeboundedHandler = (event) => { + // https://reactjs.org/docs/legacy-event-pooling.html + event.persist(); + debouncedHandleFilterChange(event); + }; + + return ( + + ) + } + + renderFilter(id, apiField, stateField, fieldLabel, blankLabel) { + if (this.state[stateField] === null) { + return
    + } else { + return ( + + {fieldLabel} + + + ) + } + } + + renderMenuItemText(text) { + if (text == null || text.trim() === '') { + return ( + + Blank + + ) + } else { + return ( + + {text} + + ) + } + } + + renderFilters() { + return ( + {this.renderSearchField()} + {this.renderFilter('financialYears', 'department__government__sphere__financial_year__slug', 'financialYears', 'Financial year', 'All financial years')} + {this.renderFilter('sphere', 'department__government__sphere__name', 'spheres', 'Sphere', 'All spheres')} + {this.renderFilter('government', 'department__government__name', 'governments', 'Government', 'All governments')} + {this.renderFilter('department', 'department__name', 'departments', 'Department', 'All departments')} + {this.renderFilter('frequency', 'frequency', 'frequencies', 'Frequency', 'All frequencies')} + {this.renderFilter('sector', 'sector', 'sectors', 'Sectors', 'All sectors')} + {this.renderFilter('mtsfOutcome', 'mtsf_outcome', 'mtsfOutcomes', 'MTSF Outcome', 'All outcomes')} + ) + } + + handleStorage() { + localStorage.setItem('data-disclaimer-acknowledged', 'true'); + this.setState({ + ...this.state, + dataDisclaimerAcknowledged: true, + modalOpen: false + }) + } + + renderLearnMoreButton() { + return ( + + Learn more about Quarterly Performance Reporting + + ) + } + + renderDataSourceModal() { + return ( + document.getElementById('performance-table-container')} + style={{position: 'absolute'}} + BackdropProps={{ + style: {position: 'absolute'} + }} + disableAutoFocus={true} + disableEnforceFocus={true} + className={'performance-modal'} + > + + Data disclaimer + + + The Quarterly Performance Reporting (QPR) data (other than the Annual audited output field) + is pre-audited non financial data. This data is approved by the accounting officer of the + relevant organ of state before publication. + + + Learn more about these performance indicators. + + + + + + ) + } + + renderLearnMore() { + return ( + + {this.renderLearnMoreButton()} + {this.renderDataSourceModal()} + + ) + } + + render() { + return (
    + {this.renderLearnMore()} + {this.renderFilters()} + {this.renderTable()} +
    ); + } +} + +function scripts() { + const parent = document.getElementById('js-initTabularView'); + if (parent) { + ReactDOM.render(, parent) + } +} + + +export default scripts(); \ No newline at end of file diff --git a/assets/js/components/search-results/SearchResult/presentation/FacetLayout.jsx b/assets/js/components/search-results/SearchResult/presentation/FacetLayout.jsx index 12db1d924..838d7a511 100644 --- a/assets/js/components/search-results/SearchResult/presentation/FacetLayout.jsx +++ b/assets/js/components/search-results/SearchResult/presentation/FacetLayout.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; const buildSnippet = (snippet, tab) => { diff --git a/assets/js/components/search-results/SearchResult/presentation/LandingLayout.jsx b/assets/js/components/search-results/SearchResult/presentation/LandingLayout.jsx index 6ffbc79d7..f34e11372 100644 --- a/assets/js/components/search-results/SearchResult/presentation/LandingLayout.jsx +++ b/assets/js/components/search-results/SearchResult/presentation/LandingLayout.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import tabOptions from './../data/tabOptions.json'; import StaticContent from './StaticContent.jsx'; @@ -112,9 +112,12 @@ const createOtherYears = (otherYears, color) => {
    See more results
    • { - otherYears.map(({ name, url, count: innerCount }) => { + otherYears.map(({ name, url, count: innerCount }, index) => { return ( -
    +
    {name}  ({innerCount} results) @@ -139,8 +142,11 @@ function Section({ type, items, tab, otherYears, error }) {
    { - items.map(({ title, url, snippet, source, contributor }) => { - return
    ; + items.map(({ title, url, snippet, source, contributor }, index) => { + return
    ; }) }
    diff --git a/assets/js/components/search-results/SearchResult/presentation/SearchPage.jsx b/assets/js/components/search-results/SearchResult/presentation/SearchPage.jsx index 7f7d4116a..c10aa0005 100644 --- a/assets/js/components/search-results/SearchResult/presentation/SearchPage.jsx +++ b/assets/js/components/search-results/SearchResult/presentation/SearchPage.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import PropTypes from 'prop-types'; import TabSelection from './TabSelection.jsx'; import LandingLayout from './LandingLayout.jsx'; diff --git a/assets/js/components/search-results/SearchResult/presentation/StaticContent.jsx b/assets/js/components/search-results/SearchResult/presentation/StaticContent.jsx index 27ba96a8e..3dd458f9a 100644 --- a/assets/js/components/search-results/SearchResult/presentation/StaticContent.jsx +++ b/assets/js/components/search-results/SearchResult/presentation/StaticContent.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import { compact } from 'lodash'; diff --git a/assets/js/components/search-results/SearchResult/presentation/TabSelection.jsx b/assets/js/components/search-results/SearchResult/presentation/TabSelection.jsx index 0cbca2745..64ac36a9a 100644 --- a/assets/js/components/search-results/SearchResult/presentation/TabSelection.jsx +++ b/assets/js/components/search-results/SearchResult/presentation/TabSelection.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; export default function TabSelection({ tab, updateTab, tabOptions }) { diff --git a/assets/js/components/search-results/SearchResult/services/SearchResult.js b/assets/js/components/search-results/SearchResult/services/SearchResult.js index 029c27090..841d37d45 100644 --- a/assets/js/components/search-results/SearchResult/services/SearchResult.js +++ b/assets/js/components/search-results/SearchResult/services/SearchResult.js @@ -1,11 +1,11 @@ -import { createElement, Component } from 'preact'; +import React from 'react'; import getLandingResults from './getLandingResults.js'; import getFacetResults from './getFacetResults.js'; import SearchPage from './../presentation/SearchPage.jsx'; import getCkanUrl from './../../../../utilities/config/siteConfig.js'; -export default class SearchPageContainer extends Component { +export default class SearchPageContainer extends React.Component { constructor(props) { super(props); const { view } = this.props; @@ -140,7 +140,7 @@ export default class SearchPageContainer extends Component { const { updateTab, addPage } = this.events; - return createElement( + return React.createElement( SearchPage, { phrase, diff --git a/assets/js/components/universal/BarChart/examples/pattern-barchart.jsx b/assets/js/components/universal/BarChart/examples/pattern-barchart.jsx index c3d053cfd..64017f682 100644 --- a/assets/js/components/universal/BarChart/examples/pattern-barchart.jsx +++ b/assets/js/components/universal/BarChart/examples/pattern-barchart.jsx @@ -1,4 +1,5 @@ -import { h, render } from 'preact'; +import React from 'react'; +import ReactDOM from 'react-dom'; import BarChart from './../index.jsx'; @@ -9,7 +10,7 @@ function pattern() { const no = document.getElementById('pattern-barchart-no'); if (basic) { - render( + ReactDOM.render( { - const activeState = selected === month ? ' is-active' : ''; + const selectors = Object.keys(months).map((month, index) => { + const activeState = selected === month ? ' is-active' : ''; - return ( - - ); - }); + return ( + + ); + }); - const mobileSelectors = ( - setMobileMonth(month)} - name="participate-select" - open={open} - /> - ); + const mobileSelectors = ( + setMobileMonth(month)} + name="participate-select" + open={open} + /> + ); - return ( -
    - { mobile ? mobileSelectors : selectors } -
    createTooltips([node])} - /> -
    - ); + return ( +
    + {mobile ? mobileSelectors : selectors} +
    createTooltips([node])} + /> +
    + ); } diff --git a/assets/js/components/universal/Participate/scripts.jsx b/assets/js/components/universal/Participate/scripts.jsx index b8c842b50..3c77f75cf 100644 --- a/assets/js/components/universal/Participate/scripts.jsx +++ b/assets/js/components/universal/Participate/scripts.jsx @@ -1,94 +1,97 @@ -import { h, Component, render } from 'preact'; +import React from 'react'; +import ReactDOM from 'react-dom'; import Participate from './index.jsx'; import DebounceFunction from './../../../utilities/js/helpers/DebounceFunction.js'; -class ParticipateContainer extends Component { - constructor(props) { - super(props); - - this.months = { - January: 'January', - February: 'February', - March: 'March', - April: 'April', - May: 'May', - June: 'June', - July: 'July', - August: 'August', - September: 'September', - October: 'October', - November: 'November', - December: 'December', - }; - - this.state = { - selected: this.months[Object.keys(this.months)[this.props.currentMonthIndex]], - open: false, - mobile: true, - }; - - this.setMonth = this.setMonth.bind(this); - this.setMobileMonth = this.setMobileMonth.bind(this); - - const func = () => { - if (this.state.mobile && window.innerWidth >= 600) { - this.setState({ mobile: false }); - } else if (!this.state.mobile && window.innerWidth < 600) { - this.setState({ mobile: true }); - } - }; - - func(); - const viewportDebounce = new DebounceFunction(100); - const updateViewport = () => viewportDebounce.update(func); - - window.addEventListener( - 'resize', - updateViewport, - ); - } - - setMonth(selected) { - this.setState({ selected }); - } - - setMobileMonth(selected) { - if (this.state.open) { - return this.setState({ - ...this.state, - open: false, - selected, - }); +class ParticipateContainer extends React.Component { + constructor(props) { + super(props); + + this.months = { + January: 'January', + February: 'February', + March: 'March', + April: 'April', + May: 'May', + June: 'June', + July: 'July', + August: 'August', + September: 'September', + October: 'October', + November: 'November', + December: 'December', + }; + + this.state = { + selected: this.months[Object.keys(this.months)[this.props.currentMonthIndex]], + open: false, + mobile: true, + }; } - return this.setState({ open: true }); - } - - render() { - return ( - - ); - } + componentDidMount() { + this.setMonth = this.setMonth.bind(this); + this.setMobileMonth = this.setMobileMonth.bind(this); + + const func = () => { + if (this.state.mobile && window.innerWidth >= 600) { + this.setState({mobile: false}); + } else if (!this.state.mobile && window.innerWidth < 600) { + this.setState({mobile: true}); + } + }; + + func(); + const viewportDebounce = new DebounceFunction(100); + const updateViewport = () => viewportDebounce.update(func); + + window.addEventListener( + 'resize', + updateViewport, + ); + } + + setMonth(selected) { + this.setState({selected}); + } + + setMobileMonth(selected) { + if (this.state.open) { + return this.setState({ + ...this.state, + open: false, + selected, + }); + } + + return this.setState({open: true}); + } + + render() { + return ( + + ); + } } const nodes = document.getElementsByClassName('js-initParticipate'); for (let i = 0; i < nodes.length; i++) { - const node = nodes[i]; + const node = nodes[i]; - const currentMonthIndex = new Date().getMonth(); + const currentMonthIndex = new Date().getMonth(); - render( - , - node, - ); + ReactDOM.render( + , + node, + ); } diff --git a/assets/js/components/universal/PseudoSelect/examples/basic-script.jsx b/assets/js/components/universal/PseudoSelect/examples/basic-script.jsx index ba500bb55..d367ca748 100644 --- a/assets/js/components/universal/PseudoSelect/examples/basic-script.jsx +++ b/assets/js/components/universal/PseudoSelect/examples/basic-script.jsx @@ -1,9 +1,10 @@ -import { h, Component, render } from 'preact'; +import React from 'react'; +import ReactDOM from 'react-dom'; import PseudoSelect from './../index.jsx'; function basicScript() { - class PseudoSelectBasicExample extends Component { + class PseudoSelectBasicExample extends React.Component { constructor(props) { super(props); @@ -40,7 +41,7 @@ function basicScript() { const node = document.getElementById('example-pseudoselect-basic-07-03'); - render( + ReactDOM.render( , node, ); diff --git a/assets/js/components/universal/PseudoSelect/index.jsx b/assets/js/components/universal/PseudoSelect/index.jsx index 4feafde15..4a6ba9bf3 100644 --- a/assets/js/components/universal/PseudoSelect/index.jsx +++ b/assets/js/components/universal/PseudoSelect/index.jsx @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; export default function PseudoSelect(props) { const { @@ -17,13 +17,13 @@ export default function PseudoSelect(props) { const id = `pseudo-select-${name}-${index}`; return ( -
  • +
    • - + {% if subprogramme_viz_data.get_dataset %}
    Budget (Main appropriation) {{ selected_financial_year }}
    + {% else %} +
    Data not available.
    + {% endif %}
    diff --git a/assets/js/scenes/homepage/Hero/scripts.js b/assets/js/scenes/homepage/Hero/scripts.js index 44c4a2f8f..86fc38bab 100644 --- a/assets/js/scenes/homepage/Hero/scripts.js +++ b/assets/js/scenes/homepage/Hero/scripts.js @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import { jsConnect as connect } from '../../../utilities/js/helpers/connector.js'; import VideoEmbed from '../../../components/universal/VideoEmbed/index.jsx'; @@ -26,7 +26,7 @@ const videoConfig = { const Hero = ({ node, image, button }) => { const playVideo = () => createModal( videoConfig.title, - h(VideoEmbed, videoConfig, null), + React.createElement(VideoEmbed, videoConfig, null), ); button.addEventListener('click', playVideo); diff --git a/assets/js/scripts.js b/assets/js/scripts.js index eac4ad1c0..04d3e8b26 100644 --- a/assets/js/scripts.js +++ b/assets/js/scripts.js @@ -3,7 +3,6 @@ import '../scss/styles.scss'; import 'classlist-polyfill'; import 'whatwg-fetch'; import 'url-search-params-polyfill'; -import 'preact/devtools'; import 'jquery'; import 'bootstrap/js/dist/scrollspy.js'; @@ -20,6 +19,7 @@ import './components/learning-centre/Glossary/scripts.jsx'; import './components/homepage/ValueBlocks/scripts.jsx'; import './components/homepage/Revenue/scripts.jsx'; import './components/homepage/HomeChart/scripts.jsx'; +import './components/homepage/Showcase/scripts.jsx'; import './components/header-and-footer/Search/scripts.jsx'; import './components/header-and-footer/Search/index.jsx'; @@ -31,6 +31,9 @@ import './components/header-and-footer/Modals/scripts.jsx'; import './components/department-budgets/DeptSearch/scripts.jsx'; import './components/department-budgets/IntroSection/scripts.jsx'; import './components/department-budgets/ArrowButtons/scripts.js'; +import './components/department-budgets/PerformanceIndicators/scripts.jsx'; + +import './components/performance/Table/scripts.jsx'; import './components/contributed-data/CsoMeta/scripts.js'; diff --git a/assets/js/services/chartAdaptor/data.json b/assets/js/services/chartAdaptor/data.json deleted file mode 100644 index 864a8df18..000000000 --- a/assets/js/services/chartAdaptor/data.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "toggleValues": { - "nominal": { - "title": "Not adjusted for inflation" - }, - "real": { - "title": "Adjusted for inflation", - "description": "The Rand values in this chart are adjusted for CPI inflation and are the effective value in 2018 Rands. CPI is used as the deflator, with the 2018-19 financial year as the base." - } - } -} diff --git a/assets/js/services/chartAdaptor/scripts.js b/assets/js/services/chartAdaptor/scripts.js index 05b8d11cd..211dbcd21 100644 --- a/assets/js/services/chartAdaptor/scripts.js +++ b/assets/js/services/chartAdaptor/scripts.js @@ -1,4 +1,4 @@ -import { h } from 'preact'; +import React from 'react'; import { preactConnect as connect } from '../../utilities/js/helpers/connector.js'; import normaliseProgrammes from './services/normaliseProgrammes/index.js'; @@ -9,7 +9,7 @@ import normaliseAdjusted from './services/normaliseAdjusted/index.js'; import normaliseExpenditureMultiples from './services/normaliseExpenditureMultiples/index.js'; import ChartSourceController from '../../components/ChartSourceController/index.jsx'; -import { toggleValues } from './data.json'; + const normaliseData = ({ type, rawItems }) => { @@ -34,6 +34,19 @@ const ChartAdaptor = (props) => { const needToggle = type === 'expenditurePhase' || type === 'expenditure'; const items = normaliseData({ type, rawItems, rotated }); const color = expenditure ? '#ad3c64' : '#73b23e'; + + const toggleValues = { + "nominal": { + "title": "Not adjusted for inflation" + }, + "real": { + "title": "Adjusted for inflation", + "description": `The Rand values in this chart are adjusted for CPI inflation and are the effective \ + value in ${rawItems.base_financial_year.slice(0, 4)} Rands. CPI is used as the deflator, with the ${rawItems.base_financial_year} \ + financial year as the base.` + } + } + const toggle = needToggle ? toggleValues : null; const downloadText = { @@ -43,7 +56,7 @@ const ChartAdaptor = (props) => { }; const styling = { scale, color, rotated }; - return h(ChartSourceController, { items, toggle, barTypes, styling, downloadText }); + return React.createElement(ChartSourceController, { items, toggle, barTypes, styling, downloadText }); }; @@ -61,4 +74,3 @@ const query = { export default connect(ChartAdaptor, 'ChartAdaptor', query); - diff --git a/assets/js/utilities/js/helpers/connector.js b/assets/js/utilities/js/helpers/connector.js index 67d65278d..cdf2813c0 100644 --- a/assets/js/utilities/js/helpers/connector.js +++ b/assets/js/utilities/js/helpers/connector.js @@ -1,7 +1,8 @@ -import { h, render } from 'preact'; +import React from 'react'; +import ReactDOM from 'react-dom'; import ReactHtmlConnector from 'react-html-connector'; const { connect: jsConnect } = new ReactHtmlConnector(null, null, { library: 'none' }); -const { connect: preactConnect } = new ReactHtmlConnector(h, render, { library: 'preact' }); +const { connect: preactConnect } = new ReactHtmlConnector(React.createElement, ReactDOM.render); export { jsConnect, preactConnect }; diff --git a/assets/js/utilities/js/helpers/fetchWrapper.js b/assets/js/utilities/js/helpers/fetchWrapper.js index e82044a77..3939e0add 100644 --- a/assets/js/utilities/js/helpers/fetchWrapper.js +++ b/assets/js/utilities/js/helpers/fetchWrapper.js @@ -1,6 +1,8 @@ -export default function fetchWrapper(url) { +export default function fetchWrapper(url,abortController=null) { return new Promise((resolve, reject) => { - fetch(url) + fetch(url, { + signal: abortController === null ? null : abortController.signal + }) .then((response) => { if (!response.ok) { reject(response); diff --git a/assets/scss/components/department-budgets/PerformanceIndicators/styles.scss b/assets/scss/components/department-budgets/PerformanceIndicators/styles.scss new file mode 100644 index 000000000..666ee492f --- /dev/null +++ b/assets/scss/components/department-budgets/PerformanceIndicators/styles.scss @@ -0,0 +1,262 @@ +#js-initPerformanceIndicators { + position: relative; +} + +.performance-indicators-container { + font-size: 16px; + line-height: 18px; + background-color: #e2e3e5 !important; + border-radius: 20px !important; + font-family: Lato, sans-serif !important; + padding: 30px 30px 0px; + box-shadow: none !important; + margin-bottom: 30px; + max-height: 300px; + overflow: hidden; + position: relative; + @media screen and (max-width: 400px) { + padding: 20px 10px 10px; + } + + .programme-name { + font-size: 18px; + font-weight: 800; + line-height: 120%; + margin-top: 0px; + color: #3f3f3f; + @media screen and (max-width: 400px) { + padding-right: 20px; + padding-left: 20px; + } + } + + .programme-card { + padding: 30px; + border-radius: 20px !important; + @media screen and (max-width: 400px) { + padding: 20px; + } + + .indicator-type { + font-family: Roboto, sans-serif; + font-weight: 500; + font-size: 14px; + line-height: 20px; + letter-spacing: 0.06em; + text-transform: uppercase; + color: #3F3F3F; + opacity: 0.5; + margin: 0 0 5px; + } + + .indicator-name { + font-weight: 700; + font-size: 18px; + line-height: 120%; + color: #3F3F3F; + margin: 0; + } + + .quarter-selection-container { + margin-top: 15px; + border-bottom: 1px solid #f2f2f2; + padding-bottom: 15px; + + .quarter-selection { + background-color: rgba(63, 63, 63, 0.05); + border-radius: 100px; + padding: 9px 12px; + box-shadow: none; + font-weight: 700; + font-size: 16px; + line-height: 100%; + color: #3F3F3F; + margin-right: 5px; + min-width: unset; + text-transform: none; + @media screen and (max-width: 400px) { + padding: 9px 7px; + } + } + + .quarter-selection.selected { + background-color: #3F3F3F; + color: #fff; + } + + .hidden-quarter { + visibility: hidden; + } + } + + .indicator-section { + p.section-head { + font-family: Roboto, sans-serif; + font-weight: 600; + font-size: 14px; + line-height: 20px; + letter-spacing: 0.06em; + text-transform: uppercase; + color: #3f3f3f; + + .chart-legend { + float: right; + text-transform: none; + font-weight: 400; + font-size: 14px; + line-height: 20px; + color: #3f3f3f; + } + } + + p.section-text { + border-radius: 16px; + background-color: rgba(0, 0, 0, 0.05); + -webkit-box-orient: vertical; + -webkit-line-clamp: 2; + display: -webkit-box; + overflow: hidden; + font-family: Roboto, sans-serif; + font-weight: 400; + font-size: 14px; + line-height: 145%; + color: #333; + padding: 12px 12px 4px; + } + + .output-text-container { + border-radius: 16px; + background-color: rgba(0, 0, 0, 0.05); + display: flex; + width: 100%; + min-height: 44px; + + .output-text { + width: 90%; + -webkit-line-clamp: 2; + display: -webkit-box; + overflow: hidden; + font-family: Roboto, sans-serif; + font-weight: 400; + font-size: 14px; + line-height: 145%; + color: #333; + padding: 12px 12px; + + .data-not-available { + opacity: 0.5; + } + } + + .read-more-output { + width: 10%; + align-self: center; + text-align: center; + display: none; + cursor: pointer; + border: none; + + svg { + pointer-events: none; + } + } + } + + .output-chart-container { + border-radius: 16px; + background-color: rgba(0, 0, 0, 0.05); + display: flex; + width: 100%; + padding: 12px; + + .active-chart { + border: 1px solid #3f3f3f; + border-radius: 6px; + } + + .bar-text { + text-align: center; + font-size: 12px; + } + + .unavailable-chart-indicator { + height: 100%; + min-height: 100px; + align-items: center; + display: flex; + @media screen and (max-width: 400px) { + min-height: 50px; + } + + svg { + margin: 0 auto; + } + } + } + + .output-text-container.read-more-visible { + .read-more-output { + display: block; + } + + .output-text { + padding: 12px 12px 4px; + } + } + } + } + + .float-right { + float: right; + } + + .IntroSection-fade { + background: linear-gradient(180deg, rgba(0, 0, 0, 0) 0%, rgba(0, 0, 0, 0.34) 100%); + } + + .IntroSection-button { + bottom: 40px; + + button { + border: none; + } + } +} + +.performance-indicators-container.is-open { + max-height: unset; + padding-bottom: 30px; +} + +.programme-btn { + text-transform: none !important; + font-weight: 700 !important; + font-size: 16px !important; + line-height: 100% !important; + font-family: Lato, sans-serif !important; + border-radius: 100px !important; + border: 1px solid #3f3f3f !important; + background-color: #fff !important; + padding: 9px 12px !important; + letter-spacing: unset !important; + font-stretch: normal !important; +} + +.MuiTooltip-tooltip { + font-family: Roboto, sans-serif !important; + font-weight: 400 !important; + font-size: 14px !important; + line-height: 145% !important; + color: #333 !important; + background-color: #fff !important; + border-radius: 20px !important; + white-space: nowrap; + max-width: unset !important; +} + +.Page-contentInner--department { + @media screen and (max-width: 400px) { + padding-right: 1rem; + padding-left: 1rem; + } +} \ No newline at end of file diff --git a/assets/scss/components/homepage/SignUpBox/styles.scss b/assets/scss/components/homepage/SignUpBox/styles.scss deleted file mode 100644 index eeec1edd3..000000000 --- a/assets/scss/components/homepage/SignUpBox/styles.scss +++ /dev/null @@ -1,40 +0,0 @@ -.SignUpBox { - box-shadow: 0 0 3px 0 rgba(0, 0, 0, 0.25); - border-radius: 20px; - padding: 20px; - background-color: #7d7d7d; - color: white; -} - -.SignUpBox-title { - font-size: 18px; - font-weight: bold; - margin-bottom: 16px; -} - -.SignUp-content { - width: 100%; - - @media screen and (min-width: 450px) { - display: table; - } -} - -.SignUpBox-input { - @media screen and (min-width: 450px) { - display: table; - width: 100%; - } -} - -.SignUpBox-button { - padding-top: 20px; - - @media screen and (min-width: 450px) { - vertical-align: top; - display: table-cell; - width: 110px; - padding-left: 10px; - padding-top: 0; - } -} diff --git a/assets/scss/components/performance/styles.scss b/assets/scss/components/performance/styles.scss new file mode 100644 index 000000000..889c8e536 --- /dev/null +++ b/assets/scss/components/performance/styles.scss @@ -0,0 +1,345 @@ +.performance-table-paper { + padding: 20px; + border-radius: 20px !important; + position: relative; + + $page_height: 100vh; + $padding_top: 12px; + $padding_bottom: 12px; + height: calc(#{$page_height} - #{$padding_top} - #{ $padding_bottom}); +} + +.performance-table-container { + $page_height: 100vh; + $pagination_row: 50px; + $pagination_row_bottom: 50px; + $padding_top: 12px; + $padding_bottom: 12px; + max-height: calc(#{$page_height} - #{$pagination_row } - #{ $pagination_row_bottom } - #{$padding_top} - #{ $padding_bottom}); +} + +.performance-table { + height: 1px; /* ignored but needed */ +} + +.performance-table .performance-table-head { + white-space: nowrap; +} + +.performance-table .indicator-detail-chip { + display: inline-block; + padding: 4px; + background-color: rgba(0, 0, 0, 0.1); + border-radius: 4px; + margin-right: 5px; + font-weight: 400; + font-size: 12px; + height: unset; + margin-bottom: 5px; + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; + line-height: 1; +} + +.performance-table .indicator-detail-chip span { + padding: 0; +} + +.performance-table .performance-table-cell { + background-color: #fff; + height: 100%; + padding: 16px 0px 16px 5px; +} + +.performance-table .performance-indicator-cell { + background-color: #fff; + height: 100%; + padding: 16px 0px 16px 5px; +} + +.performance-table .performance-table-cell .cell-content { + max-width: 200px; + position: relative; +} + +.performance-table .performance-indicator-cell .cell-content { + font-weight: 700; + margin-right: 5px; + margin-left: 5px; + width: 25vw; +} + +.performance-table .performance-table-cell .cell-content, .performance-table .performance-indicator-cell .cell-content { + font-family: Lato, sans-serif; + border-right: 1px solid #c6c6c6; + border-bottom: none; + height: 100%; + padding-left: 10px; + padding-right: 10px; +} + +.performance-table .performance-table-cell.alternate, .performance-table .performance-indicator-cell.alternate { + background-color: #f5f5f5; +} + +.performance-table .performance-table-head-cell { + background-color: #fff; + border-radius: 5px; + border-bottom: none; + font-weight: 700; + padding: 0 5px 0 0; +} + +.performance-table .performance-table-head-cell .cell-content { + font-family: Lato, sans-serif; + height: 100%; + background-color: #f5f5f5; + padding: 5px 5px 5px 10px; + margin-right: 5px; + margin-left: 5px; + min-width: 150px; +} + +.performance-table .performance-table-head-cell.indicator-column-head .cell-content { + width: 25vw; + min-width: unset; +} + +.performance-table .performance-indicator-cell .cell-content .indicator-name { + overflow: hidden; + text-overflow: ellipsis; + margin-bottom: 5px; +} + +.performance-table .link-button { + text-decoration: underline; + cursor: pointer; +} + +.table-loading-state { + position: absolute; + top: 0; + left: 0; + bottom: 0; + right: 0; + background-color: rgba(255, 255, 255, 0.9); + z-index: 9; + + .table-circular-progress { + $page_height: 100vh; + $pagination_row: 50px; + $pagination_row_bottom: 50px; + $padding_top: 12px; + $padding_bottom: 12px; + $gif_height: 40px; + margin-top: calc((#{$page_height} - #{$pagination_row } - #{ $pagination_row_bottom } - #{$padding_top} - #{ $padding_bottom} - #{ $gif_height}) / 2); + } +} + +/* read more */ +.performance-table-cell { + input { + opacity: 0; + position: absolute; + pointer-events: none; + } + + span { + display: -webkit-box; + -webkit-line-clamp: 4; + overflow: hidden; + } + + label { + text-decoration: underline; + cursor: pointer; + } + + input:focus ~ label { + outline: -webkit-focus-ring-color auto 5px; + } + + input:checked + span { + -webkit-line-clamp: unset; + } + + input:checked ~ label, span:not(.truncated) ~ label { + display: none; + } +} + +/* read more */ + +.filter-search, .filter-search label, .filter-search select, .filter-search option, .filter-search .filter-menu-item, .filter-search ul li { + font-family: Lato, sans-serif; +} + +.filter-menu-item { + font-size: 14px !important; + font-family: Lato, sans-serif !important; + padding-left: 12px !important; + line-height: 1 !important; +} + +.filter-menu-item .option-text { + font-family: Lato, sans-serif !important; + width: 100%; + margin-right: 16px; + white-space: break-spaces; +} + +.filter-menu-item .option-text.blank-label { + padding-bottom: 5.5px; + padding-top: 5.5px; +} + +.filter-menu-item .option-facet { + font-family: Lato, sans-serif !important; + padding: 6px; + height: unset; + margin: auto; +} + +.filter-menu-item .option-facet span { + padding: 0px; +} + +.filter-search .option-facet { + display: none; +} + +.performance-modal .MuiDialog-paper { + margin: 0 auto; + position: absolute; + top: 200px; + padding: 20px; + border-radius: 20px !important; + + .performance-modal-title { + font-family: Lato, sans-serif !important; + font-weight: 700; + font-size: 24px; + line-height: 145%; + color: #000; + } + + .performance-modal-content { + font-family: Lato, sans-serif !important; + font-weight: 400; + font-size: 14px; + line-height: 145%; + color: #000; + margin-top: 15px; + } + + .performance-modal-link { + margin-top: 15px; + margin-bottom: 15px; + + a { + font-family: Lato, sans-serif !important; + font-weight: 400; + font-size: 14px; + line-height: 145%; + color: #000; + } + } + + .performance-modal-full { + width: 100%; + } +} + +.performance-modal-button { + font-family: Lato, sans-serif !important; + line-height: 100%; + font-size: 16px; +} + +#performance-table-container { + position: relative; +} + +.Page-head { + z-index: 9999; +} + +.download-btn { + text-transform: none !important; + font-weight: 700 !important; + font-size: 16px !important; + line-height: 100% !important; + font-family: Lato, sans-serif !important; + border-radius: 100px !important; + border: 1px solid #3f3f3f !important; + background-color: #fff !important; + padding: 9px 12px !important; + letter-spacing: unset !important; + font-stretch: normal !important; +} + +.performance-modal .MuiDialog-paper { + margin: 0 auto; + position: relative; + padding: 20px; + border-radius: 20px !important; + + .performance-modal-title { + font-family: Lato, sans-serif !important; + font-weight: 700; + font-size: 24px; + line-height: 145%; + color: #000; + } + + .performance-modal-content { + font-family: Lato, sans-serif !important; + font-weight: 400; + font-size: 14px; + line-height: 145%; + color: #000; + margin-top: 15px; + } + + .performance-modal-link { + margin-top: 15px; + margin-bottom: 15px; + + a { + font-family: Lato, sans-serif !important; + font-weight: 400; + font-size: 14px; + line-height: 145%; + color: #000; + } + } + + .performance-modal-full { + width: 100%; + } +} + +.performance-modal-button { + font-family: Lato, sans-serif !important; + line-height: 100%; + font-size: 16px; +} + +#performance-table-container { + position: relative; +} + +.Page-head { + z-index: 9999; +} + +@media (max-width: 480px) { + .performance-table .performance-indicator-cell .cell-content { + width: 80vw; + } + + .performance-modal .MuiDialog-paper { + width: 87%; + } +} diff --git a/assets/scss/scenes/department/ExpenditureMultiplesSection/styles.scss b/assets/scss/scenes/department/ExpenditureMultiplesSection/styles.scss index 3960ee2c3..c13146aa5 100644 --- a/assets/scss/scenes/department/ExpenditureMultiplesSection/styles.scss +++ b/assets/scss/scenes/department/ExpenditureMultiplesSection/styles.scss @@ -33,6 +33,9 @@ .ExpenditureMultiplesSection-label { display: flex; + @media screen and (max-width: 400px) { + display: block; + } } .ExpenditureMultiplesSection-labelItem { diff --git a/assets/scss/scenes/department/ExpenditurePhaseSection/styles.scss b/assets/scss/scenes/department/ExpenditurePhaseSection/styles.scss index 051a5c40e..a7e210d9a 100644 --- a/assets/scss/scenes/department/ExpenditurePhaseSection/styles.scss +++ b/assets/scss/scenes/department/ExpenditurePhaseSection/styles.scss @@ -29,9 +29,13 @@ .ExpenditurePhaseSection-label { display: flex; + @media screen and (max-width: 400px) { + display: block; + } } .ExpenditurePhaseSection-labelItem { margin-top: 1rem; margin-right: 3rem; } + diff --git a/assets/scss/styles.scss b/assets/scss/styles.scss index f0ae253a0..6cf370f07 100644 --- a/assets/scss/styles.scss +++ b/assets/scss/styles.scss @@ -83,7 +83,6 @@ @import 'components/homepage/Links/styles'; @import 'components/homepage/NewIntro/styles'; @import 'components/homepage/Related/styles'; -@import 'components/homepage/SignUpBox/styles'; @import 'components/homepage/ValueBlocks/styles'; @import 'components/header-and-footer/ExtraFooter/styles'; @import 'components/header-and-footer/Footer/styles'; @@ -104,6 +103,7 @@ @import 'components/department-budgets/DeptSectionHead/styles'; @import 'components/department-budgets/InfraProjectList/styles'; @import 'components/department-budgets/SectionIndicator/styles'; +@import 'components/department-budgets/PerformanceIndicators/styles'; @import 'components/contributed-data/CsoMeta/styles'; @import 'components/contributed-data/Screenshots/styles'; @import 'components/learning-centre/Glossary/styles'; @@ -114,6 +114,7 @@ @import 'components/DescriptionEmbedSection/styles'; @import 'components/universal/SiteNotice/styles'; @import 'components/GovernmentResources/GovernmentResources.scss'; +@import 'components/performance/styles'; @import 'scenes/homepage/Hero/styles'; @import 'scenes/homepage/Features/styles'; diff --git a/budgetportal/admin.py b/budgetportal/admin.py index 4ede61057..5823e9131 100644 --- a/budgetportal/admin.py +++ b/budgetportal/admin.py @@ -60,8 +60,8 @@ def get_resource_kwargs(self, request, *args, **kwargs): Get the kwargs to send on to the department resource when we import departments. """ - if u"sphere" in request.POST: - return {"sphere": request.POST[u"sphere"]} + if "sphere" in request.POST: + return {"sphere": request.POST["sphere"]} return {} list_display = ( @@ -219,6 +219,11 @@ class MainMenuItemAdmin(SortableAdmin): model = models.MainMenuItem +class ShowcaseItemAdmin(SortableAdmin): + list_display = ("name", "description", "created_at") + model = models.ShowcaseItem + + admin.site.register(models.FinancialYear, FinancialYearAdmin) admin.site.register(models.Sphere, SphereAdmin) admin.site.register(models.Government, GovernmentAdmin) @@ -238,3 +243,4 @@ class MainMenuItemAdmin(SortableAdmin): admin.site.register(models.CategoryGuide) admin.site.register(models.MainMenuItem, MainMenuItemAdmin) admin.site.register(models.Notice, SortableAdmin) +admin.site.register(models.ShowcaseItem, ShowcaseItemAdmin) diff --git a/budgetportal/bulk_upload.py b/budgetportal/bulk_upload.py index 32926a357..72780f5eb 100644 --- a/budgetportal/bulk_upload.py +++ b/budgetportal/bulk_upload.py @@ -31,20 +31,38 @@ logger = logging.getLogger(__name__) HEADINGS = [ - {"label": "government", "comment": None,}, - {"label": "group_id", "comment": None,}, - {"label": "department_name", "comment": None,}, + { + "label": "government", + "comment": None, + }, + { + "label": "group_id", + "comment": None, + }, + { + "label": "department_name", + "comment": None, + }, { "label": "dataset_name", "comment": 'This will be "sluggified" and must then be unique to this dataset in the entire system. For example, Gauteng Provincial Legislature EPRE for 2017-19 is prov-dept-gt-gauteng-provincial-legislature-2017-18', }, - {"label": "dataset_title", "comment": None,}, - {"label": "resource_name", "comment": None,}, + { + "label": "dataset_title", + "comment": None, + }, + { + "label": "resource_name", + "comment": None, + }, { "label": "resource_format", "comment": "You can usually make this the capitalised extension of the file. We use the combination of the format and resource title to ensure we do not duplicate resources. So we assume a file has only one resource with the same title and format combination.", }, - {"label": "resource_url", "comment": None,}, + { + "label": "resource_url", + "comment": None, + }, ] @@ -153,7 +171,7 @@ def __init__(self, sphere, metadata_file): heading_index[cell.value] = i else: government_name = ws_row[heading_index["government"]].value - department_name = ws_row[heading_index[u"department_name"]].value + department_name = ws_row[heading_index["department_name"]].value group_name = max_length_slugify( ws_row[heading_index["group_id"]].value ) diff --git a/budgetportal/context_processors.py b/budgetportal/context_processors.py index dd75fbe23..584b00274 100644 --- a/budgetportal/context_processors.py +++ b/budgetportal/context_processors.py @@ -28,7 +28,7 @@ def tag_manager_id(request): return { "TAG_MANAGER_ID": settings.TAG_MANAGER_ID, "TAG_MANAGER_SCRIPT_NONCE": b64encode( - str(randint(10 ** 10, 10 ** 11)).encode() + str(randint(10**10, 10**11)).encode() ).decode(), } diff --git a/budgetportal/fixtures/test-guides-pages.json b/budgetportal/fixtures/test-guides-pages.json index dbaf04bbf..ec53d4992 100644 --- a/budgetportal/fixtures/test-guides-pages.json +++ b/budgetportal/fixtures/test-guides-pages.json @@ -102,7 +102,10 @@ "title": "Root", "draft_title": "Root", "slug": "root", - "content_type": 1, + "content_type": [ + "wagtailcore", + "page" + ], "live": true, "has_unpublished_changes": false, "url_path": "/", @@ -130,7 +133,10 @@ "title": "Welcome to your new Wagtail site!", "draft_title": "Welcome to your new Wagtail site!", "slug": "home", - "content_type": 1, + "content_type": [ + "wagtailcore", + "page" + ], "live": true, "has_unpublished_changes": false, "url_path": "/home/", @@ -158,7 +164,10 @@ "title": "Learning index page", "draft_title": "Learning index page", "slug": "learning-index-page", - "content_type": 22, + "content_type": [ + "budgetportal", + "learningindexpage" + ], "live": true, "has_unpublished_changes": false, "url_path": "/learning-index-page/", @@ -186,7 +195,10 @@ "title": "Guide index page", "draft_title": "Guide index page", "slug": "guide-index-page", - "content_type": 21, + "content_type": [ + "budgetportal", + "guideindexpage" + ], "live": true, "has_unpublished_changes": false, "url_path": "/learning-index-page/guide-index-page/", @@ -214,7 +226,10 @@ "title": "Guide page", "draft_title": "Guide page", "slug": "guide-page", - "content_type": 26, + "content_type": [ + "budgetportal", + "guidepage" + ], "live": true, "has_unpublished_changes": false, "url_path": "/learning-index-page/guide-index-page/guide-page/", diff --git a/budgetportal/fixtures/test-posts-pages.json b/budgetportal/fixtures/test-posts-pages.json index ca0dc25ef..757f70d69 100644 --- a/budgetportal/fixtures/test-posts-pages.json +++ b/budgetportal/fixtures/test-posts-pages.json @@ -90,7 +90,10 @@ "title": "Root", "draft_title": "Root", "slug": "root", - "content_type": 1, + "content_type": [ + "wagtailcore", + "page" + ], "live": true, "has_unpublished_changes": false, "url_path": "/", @@ -118,7 +121,10 @@ "title": "Welcome to your new Wagtail site!", "draft_title": "Welcome to your new Wagtail site!", "slug": "home", - "content_type": 1, + "content_type": [ + "wagtailcore", + "page" + ], "live": true, "has_unpublished_changes": false, "url_path": "/home/", @@ -146,7 +152,10 @@ "title": "Post Index Page", "draft_title": "Post Index Page", "slug": "post-index-page", - "content_type": 23, + "content_type": [ + "budgetportal", + "postindexpage" + ], "live": true, "has_unpublished_changes": false, "url_path": "/post-index-page/", @@ -174,7 +183,10 @@ "title": "Post page", "draft_title": "Post page", "slug": "post-page", - "content_type": 25, + "content_type": [ + "budgetportal", + "postpage" + ], "live": true, "has_unpublished_changes": false, "url_path": "/post-index-page/post-page/", diff --git a/budgetportal/forms.py b/budgetportal/forms.py index d0d7c2183..c2a631d60 100644 --- a/budgetportal/forms.py +++ b/budgetportal/forms.py @@ -11,5 +11,5 @@ class AllauthSignupForm(forms.Form): captcha = ReCaptchaField() def signup(self, request, user): - """ Required, or else it throws deprecation warnings """ + """Required, or else it throws deprecation warnings""" pass diff --git a/budgetportal/infra_projects/__init__.py b/budgetportal/infra_projects/__init__.py index b247af7b9..8cd1adf13 100644 --- a/budgetportal/infra_projects/__init__.py +++ b/budgetportal/infra_projects/__init__.py @@ -114,7 +114,9 @@ class InfraProjectSnapshotResource(resources.ModelResource): column_name="irm_snapshot", widget=ForeignKeyWidget(models.IRMSnapshot), ) - sphere_slug = Field(column_name="sphere_slug",) + sphere_slug = Field( + column_name="sphere_slug", + ) project_number = Field(attribute="project_number", column_name="Project No") name = Field(attribute="name", column_name="Project Name") department = Field(attribute="department", column_name="Department") diff --git a/budgetportal/infra_projects/irm_preprocessor.py b/budgetportal/infra_projects/irm_preprocessor.py index 054c2d088..b882936c5 100644 --- a/budgetportal/infra_projects/irm_preprocessor.py +++ b/budgetportal/infra_projects/irm_preprocessor.py @@ -40,7 +40,7 @@ "Nature of Investment", "Funding Status", ] -REPEATED_IMPLEMENTOR_HEADER = u"Project Contractor" +REPEATED_IMPLEMENTOR_HEADER = "Project Contractor" EXTRA_IMPLEMENTOR_HEADER = "Other parties" IMPLEMENTORS = ["Program Implementing Agent", "Principal Agent", "Main Contractor"] IMPLEMENTOR_HEADERS = IMPLEMENTORS + [EXTRA_IMPLEMENTOR_HEADER] diff --git a/budgetportal/migrations/0042_auto_20200131_1500.py b/budgetportal/migrations/0042_auto_20200131_1500.py index 4dcc7be41..55a52d46f 100644 --- a/budgetportal/migrations/0042_auto_20200131_1500.py +++ b/budgetportal/migrations/0042_auto_20200131_1500.py @@ -13,6 +13,7 @@ class Migration(migrations.Migration): operations = [ migrations.AlterUniqueTogether( - name="irmsnapshot", unique_together=set([("financial_year", "quarter")]), + name="irmsnapshot", + unique_together=set([("financial_year", "quarter")]), ), ] diff --git a/budgetportal/migrations/0047_auto_20200313_0626.py b/budgetportal/migrations/0047_auto_20200313_0626.py index 3f437f2fa..240256996 100644 --- a/budgetportal/migrations/0047_auto_20200313_0626.py +++ b/budgetportal/migrations/0047_auto_20200313_0626.py @@ -26,12 +26,17 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RenameModel(old_name="ProvInfraProject", new_name="InfraProject",), migrations.RenameModel( - old_name="ProvInfraProjectSnapshot", new_name="InfraProjectSnapshot", + old_name="ProvInfraProject", + new_name="InfraProject", + ), + migrations.RenameModel( + old_name="ProvInfraProjectSnapshot", + new_name="InfraProjectSnapshot", ), migrations.AlterModelOptions( - name="infraproject", options={"verbose_name": "Infrastructure project"}, + name="infraproject", + options={"verbose_name": "Infrastructure project"}, ), migrations.AlterModelOptions( name="infraprojectsnapshot", @@ -69,7 +74,11 @@ class Migration(migrations.Migration): ), ), migrations.AlterUniqueTogether( - name="irmsnapshot", unique_together={("sphere", "quarter")}, + name="irmsnapshot", + unique_together={("sphere", "quarter")}, + ), + migrations.RemoveField( + model_name="irmsnapshot", + name="financial_year", ), - migrations.RemoveField(model_name="irmsnapshot", name="financial_year",), ] diff --git a/budgetportal/migrations/0048_auto_20200313_2011.py b/budgetportal/migrations/0048_auto_20200313_2011.py index 026eacc0b..122acd734 100644 --- a/budgetportal/migrations/0048_auto_20200313_2011.py +++ b/budgetportal/migrations/0048_auto_20200313_2011.py @@ -31,7 +31,9 @@ class Migration(migrations.Migration): ), ("intro", wagtail.core.fields.RichTextField(blank=True)), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, bases=("wagtailcore.page",), ), migrations.CreateModel( @@ -49,7 +51,9 @@ class Migration(migrations.Migration): ), ), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, bases=("wagtailcore.page",), ), migrations.CreateModel( @@ -68,7 +72,9 @@ class Migration(migrations.Migration): ), ("intro", wagtail.core.fields.RichTextField(blank=True)), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, bases=("wagtailcore.page",), ), migrations.CreateModel( @@ -89,7 +95,9 @@ class Migration(migrations.Migration): bases=("wagtailcore.page",), ), migrations.AlterField( - model_name="faq", name="content", field=wagtail.core.fields.RichTextField(), + model_name="faq", + name="content", + field=wagtail.core.fields.RichTextField(), ), migrations.CreateModel( name="PostPage", @@ -151,7 +159,9 @@ class Migration(migrations.Migration): ), ), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, bases=("wagtailcore.page",), ), migrations.CreateModel( @@ -214,7 +224,9 @@ class Migration(migrations.Migration): ), ), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, bases=("wagtailcore.page",), ), migrations.CreateModel( diff --git a/budgetportal/migrations/0051_auto_20200314_1522.py b/budgetportal/migrations/0051_auto_20200314_1522.py index 0e764f58c..1a39b805e 100644 --- a/budgetportal/migrations/0051_auto_20200314_1522.py +++ b/budgetportal/migrations/0051_auto_20200314_1522.py @@ -16,6 +16,7 @@ class Migration(migrations.Migration): field=models.IntegerField(), ), migrations.AlterUniqueTogether( - name="infraproject", unique_together={("sphere_slug", "IRM_project_id")}, + name="infraproject", + unique_together={("sphere_slug", "IRM_project_id")}, ), ] diff --git a/budgetportal/migrations/0053_custompage.py b/budgetportal/migrations/0053_custompage.py index 9bbe7db95..c74c46374 100644 --- a/budgetportal/migrations/0053_custompage.py +++ b/budgetportal/migrations/0053_custompage.py @@ -28,7 +28,9 @@ class Migration(migrations.Migration): ), ), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, bases=("wagtailcore.page", budgetportal.models.NavContextMixin), ), ] diff --git a/budgetportal/migrations/0057_mainmenuitem_submenuitem.py b/budgetportal/migrations/0057_mainmenuitem_submenuitem.py index 8604ff2c7..dc394f52f 100644 --- a/budgetportal/migrations/0057_mainmenuitem_submenuitem.py +++ b/budgetportal/migrations/0057_mainmenuitem_submenuitem.py @@ -36,7 +36,9 @@ class Migration(migrations.Migration): ), ("align_right", models.BooleanField()), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, ), migrations.CreateModel( name="SubMenuItem", @@ -68,6 +70,8 @@ class Migration(migrations.Migration): ), ), ], - options={"abstract": False,}, + options={ + "abstract": False, + }, ), ] diff --git a/budgetportal/migrations/0058_auto_20200327_1414.py b/budgetportal/migrations/0058_auto_20200327_1414.py index 86e56f730..bf02085f5 100644 --- a/budgetportal/migrations/0058_auto_20200327_1414.py +++ b/budgetportal/migrations/0058_auto_20200327_1414.py @@ -11,10 +11,12 @@ class Migration(migrations.Migration): operations = [ migrations.AlterModelOptions( - name="mainmenuitem", options={"ordering": ["main_menu_item_order"]}, + name="mainmenuitem", + options={"ordering": ["main_menu_item_order"]}, ), migrations.AlterModelOptions( - name="submenuitem", options={"ordering": ["sub_menu_item_order"]}, + name="submenuitem", + options={"ordering": ["sub_menu_item_order"]}, ), migrations.AddField( model_name="mainmenuitem", diff --git a/budgetportal/migrations/0059_notice.py b/budgetportal/migrations/0059_notice.py index 2f3582871..6b58862a9 100644 --- a/budgetportal/migrations/0059_notice.py +++ b/budgetportal/migrations/0059_notice.py @@ -32,6 +32,8 @@ class Migration(migrations.Migration): ), ), ], - options={"ordering": ["notice_order"],}, + options={ + "ordering": ["notice_order"], + }, ), ] diff --git a/budgetportal/migrations/0060_auto_20200416_1218.py b/budgetportal/migrations/0060_auto_20200416_1218.py index 28ec5e523..b6f7fd3c6 100644 --- a/budgetportal/migrations/0060_auto_20200416_1218.py +++ b/budgetportal/migrations/0060_auto_20200416_1218.py @@ -12,9 +12,13 @@ class Migration(migrations.Migration): operations = [ migrations.AlterField( - model_name="faq", name="content", field=ckeditor.fields.RichTextField(), + model_name="faq", + name="content", + field=ckeditor.fields.RichTextField(), ), migrations.AlterField( - model_name="notice", name="content", field=ckeditor.fields.RichTextField(), + model_name="notice", + name="content", + field=ckeditor.fields.RichTextField(), ), ] diff --git a/budgetportal/migrations/0065_showcaseitem.py b/budgetportal/migrations/0065_showcaseitem.py new file mode 100644 index 000000000..4739d0df1 --- /dev/null +++ b/budgetportal/migrations/0065_showcaseitem.py @@ -0,0 +1,42 @@ +# Generated by Django 2.2.28 on 2023-02-17 14:16 + +import budgetportal.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("budgetportal", "0064_auto_20200728_1054"), + ] + + operations = [ + migrations.CreateModel( + name="ShowcaseItem", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=200)), + ("description", models.CharField(max_length=400)), + ("button_text_1", models.CharField(max_length=200)), + ("button_link_1", models.URLField(blank=True, null=True)), + ("button_text_2", models.CharField(max_length=200)), + ("button_link_2", models.URLField(blank=True, null=True)), + ( + "file", + models.FileField( + upload_to=budgetportal.models.irm_snapshot_file_path + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True, null=True)), + ("updated_at", models.DateTimeField(auto_now=True, null=True)), + ], + ), + ] diff --git a/budgetportal/migrations/0066_auto_20230217_1433.py b/budgetportal/migrations/0066_auto_20230217_1433.py new file mode 100644 index 000000000..07688008d --- /dev/null +++ b/budgetportal/migrations/0066_auto_20230217_1433.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.28 on 2023-02-17 14:33 + +import budgetportal.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("budgetportal", "0065_showcaseitem"), + ] + + operations = [ + migrations.AlterField( + model_name="showcaseitem", + name="file", + field=models.FileField( + upload_to=budgetportal.models.showcase_item_file_path + ), + ), + ] diff --git a/budgetportal/migrations/0067_auto_20230219_1148.py b/budgetportal/migrations/0067_auto_20230219_1148.py new file mode 100644 index 000000000..0ff244e08 --- /dev/null +++ b/budgetportal/migrations/0067_auto_20230219_1148.py @@ -0,0 +1,70 @@ +# Generated by Django 2.2.28 on 2023-02-19 11:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("budgetportal", "0066_auto_20230217_1433"), + ] + + operations = [ + migrations.RemoveField( + model_name="showcaseitem", + name="button_link_1", + ), + migrations.RemoveField( + model_name="showcaseitem", + name="button_link_2", + ), + migrations.RemoveField( + model_name="showcaseitem", + name="button_text_1", + ), + migrations.RemoveField( + model_name="showcaseitem", + name="button_text_2", + ), + migrations.AddField( + model_name="showcaseitem", + name="cta_link_1", + field=models.URLField( + blank=True, null=True, verbose_name="Call to action link 1" + ), + ), + migrations.AddField( + model_name="showcaseitem", + name="cta_link_2", + field=models.URLField( + blank=True, null=True, verbose_name="Call to action link 2" + ), + ), + migrations.AddField( + model_name="showcaseitem", + name="cta_text_1", + field=models.CharField( + default="", max_length=200, verbose_name="Call to action text 1" + ), + preserve_default=False, + ), + migrations.AddField( + model_name="showcaseitem", + name="cta_text_2", + field=models.CharField( + default="", max_length=200, verbose_name="Call to action text 2" + ), + preserve_default=False, + ), + migrations.AddField( + model_name="showcaseitem", + name="second_cta_type", + field=models.CharField( + choices=[("primary", "Primary"), ("secondary", "Secondary")], + default="primary", + max_length=255, + verbose_name="Second call to action type", + ), + preserve_default=False, + ), + ] diff --git a/budgetportal/migrations/0068_auto_20230220_0910.py b/budgetportal/migrations/0068_auto_20230220_0910.py new file mode 100644 index 000000000..1ef59a7fc --- /dev/null +++ b/budgetportal/migrations/0068_auto_20230220_0910.py @@ -0,0 +1,22 @@ +# Generated by Django 2.2.28 on 2023-02-20 09:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("budgetportal", "0067_auto_20230219_1148"), + ] + + operations = [ + migrations.AlterModelOptions( + name="showcaseitem", + options={"ordering": ["item_order"]}, + ), + migrations.AddField( + model_name="showcaseitem", + name="item_order", + field=models.PositiveIntegerField(db_index=True, default=0, editable=False), + ), + ] diff --git a/budgetportal/migrations/0069_showcaseitem_cta_enabled_2.py b/budgetportal/migrations/0069_showcaseitem_cta_enabled_2.py new file mode 100644 index 000000000..17afd5c63 --- /dev/null +++ b/budgetportal/migrations/0069_showcaseitem_cta_enabled_2.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.28 on 2023-02-22 18:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("budgetportal", "0068_auto_20230220_0910"), + ] + + operations = [ + migrations.AddField( + model_name="showcaseitem", + name="cta_enabled_2", + field=models.BooleanField( + default=True, verbose_name="Enable call to action 2" + ), + preserve_default=False, + ), + ] diff --git a/budgetportal/migrations/0070_auto_20230228_0756.py b/budgetportal/migrations/0070_auto_20230228_0756.py new file mode 100644 index 000000000..27d5ae90c --- /dev/null +++ b/budgetportal/migrations/0070_auto_20230228_0756.py @@ -0,0 +1,27 @@ +# Generated by Django 2.2.28 on 2023-02-28 07:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("budgetportal", "0069_showcaseitem_cta_enabled_2"), + ] + + operations = [ + migrations.RemoveField( + model_name="showcaseitem", + name="cta_enabled_2", + ), + migrations.AlterField( + model_name="showcaseitem", + name="cta_text_2", + field=models.CharField( + blank=True, + max_length=200, + null=True, + verbose_name="Call to action text 2", + ), + ), + ] diff --git a/budgetportal/migrations/0071_auto_20230605_1521.py b/budgetportal/migrations/0071_auto_20230605_1521.py new file mode 100644 index 000000000..8eea8b100 --- /dev/null +++ b/budgetportal/migrations/0071_auto_20230605_1521.py @@ -0,0 +1,22 @@ +# Generated by Django 2.2.28 on 2023-06-05 15:21 + +import budgetportal.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("budgetportal", "0070_auto_20230228_0756"), + ] + + operations = [ + migrations.AlterField( + model_name="showcaseitem", + name="file", + field=models.FileField( + help_text="
    • Thumbnail aspect ratio should be 1.91:1.
    • Recommended resolution is 1200 x 630 px.
    • Main focus of image should be centered occupying 630 x 630 px in the middle of the image.
    ", + upload_to=budgetportal.models.showcase_item_file_path, + ), + ), + ] diff --git a/budgetportal/models/__init__.py b/budgetportal/models/__init__.py new file mode 100644 index 000000000..40ff87c96 --- /dev/null +++ b/budgetportal/models/__init__.py @@ -0,0 +1,858 @@ +import logging +import re +import uuid +from collections import OrderedDict +from datetime import datetime + +import requests +from slugify import slugify + +import ckeditor.fields as ckeditor_fields +from adminsortable.models import SortableMixin +from budgetportal.blocks import DescriptionEmbedBlock, SectionBlock +from budgetportal.datasets import Dataset +from django.conf import settings +from django.core.cache import cache +from django.core.exceptions import MultipleObjectsReturned, ValidationError +from django.db import models +from django.urls import reverse +from django.utils.safestring import mark_safe +from wagtail.admin.edit_handlers import FieldPanel, MultiFieldPanel, StreamFieldPanel +from wagtail.core import blocks as wagtail_blocks +from wagtail.core.fields import RichTextField, StreamField +from wagtail.core.models import Page +from wagtail.images.edit_handlers import ImageChooserPanel +from wagtail.snippets.models import register_snippet + +from .government import ( + FinancialYear, + Sphere, + Government, + GovtFunction, + Department, + Programme, + SPHERE_SLUG_CHOICES, + NATIONAL_SLUG, + PROVINCIAL_SLUG, + EXPENDITURE_TIME_SERIES_PHASE_MAPPING, +) + +logger = logging.getLogger(__name__) +ckan = settings.CKAN + +MAPIT_POINT_API_URL = "https://mapit.code4sa.org/point/4326/{},{}" + +prov_abbrev = { + "Eastern Cape": "EC", + "Free State": "FS", + "Gauteng": "GT", + "KwaZulu-Natal": "NL", + "Limpopo": "LP", + "Mpumalanga": "MP", + "Northern Cape": "NC", + "North West": "NW", + "Western Cape": "WC", +} + + +class Homepage(models.Model): + main_heading = models.CharField(max_length=1000, blank=True) + sub_heading = models.CharField(max_length=1000, blank=True) + primary_button_label = models.CharField(max_length=1000, blank=True) + primary_button_url = models.CharField(max_length=1000, blank=True) + primary_button_target = models.CharField(max_length=1000, blank=True) + secondary_button_label = models.CharField(max_length=1000, blank=True) + secondary_button_url = models.CharField(max_length=1000, blank=True) + secondary_button_target = models.CharField(max_length=1000, blank=True) + call_to_action_sub_heading = models.CharField(max_length=1000, blank=True) + call_to_action_heading = models.CharField(max_length=1000, blank=True) + call_to_action_link_label = models.CharField(max_length=1000, blank=True) + call_to_action_link_url = models.CharField(max_length=1000, blank=True) + call_to_action_link_target = models.CharField(max_length=1000, blank=True) + + +class InfrastructureProjectPart(models.Model): + administration_type = models.CharField(max_length=255) + government_institution = models.CharField(max_length=255) + sector = models.CharField(max_length=255) + project_name = models.CharField(max_length=255) + project_description = models.TextField() + nature_of_investment = models.CharField(max_length=255) + infrastructure_type = models.CharField(max_length=255) + current_project_stage = models.CharField(max_length=255) + sip_category = models.CharField(max_length=255) + br_featured = models.CharField(max_length=255) + featured = models.BooleanField() + budget_phase = models.CharField(max_length=255) + project_slug = models.CharField(max_length=255) + amount_rands = models.BigIntegerField(blank=True, null=True, default=None) + financial_year = models.CharField(max_length=4) + project_value_rands = models.BigIntegerField(default=0) + provinces = models.CharField(max_length=510, default="") + gps_code = models.CharField(max_length=255, default="") + + # PPP fields + partnership_type = models.CharField(max_length=255, null=True, blank=True) + date_of_close = models.CharField(max_length=255, null=True, blank=True) + duration = models.CharField(max_length=255, null=True, blank=True) + financing_structure = models.CharField(max_length=255, null=True, blank=True) + project_value_rand_million = models.CharField(max_length=255, null=True, blank=True) + form_of_payment = models.CharField(max_length=255, null=True, blank=True) + + class Meta: + verbose_name = "National infrastructure project part" + + def __str__(self): + return "{} ({} {})".format( + self.project_slug, self.budget_phase, self.financial_year + ) + + def get_url_path(self): + return "/infrastructure-projects/{}".format(self.project_slug) + + def get_absolute_url(self): + return reverse("infrastructure-projects", args=[self.project_slug]) + + def calculate_projected_expenditure(self): + """Calculate sum of predicted amounts from a list of records""" + projected_records_for_project = InfrastructureProjectPart.objects.filter( + budget_phase="MTEF", project_slug=self.project_slug + ) + projected_expenditure = 0 + for project in projected_records_for_project: + projected_expenditure += float(project.amount_rands or 0.0) + return projected_expenditure + + @staticmethod + def _parse_coordinate(coordinate): + """Expects a single set of coordinates (lat, long) split by a comma""" + if not isinstance(coordinate, str): + raise TypeError("Invalid type for coordinate parsing") + lat_long = [float(x) for x in coordinate.split(",")] + cleaned_coordinate = {"latitude": lat_long[0], "longitude": lat_long[1]} + return cleaned_coordinate + + @classmethod + def clean_coordinates(cls, raw_coordinate_string): + cleaned_coordinates = [] + try: + if "and" in raw_coordinate_string: + list_of_coordinates = raw_coordinate_string.split("and") + for coordinate in list_of_coordinates: + cleaned_coordinates.append(cls._parse_coordinate(coordinate)) + elif "," in raw_coordinate_string: + cleaned_coordinates.append(cls._parse_coordinate(raw_coordinate_string)) + else: + logger.warning("Invalid co-ordinates: {}".format(raw_coordinate_string)) + except Exception as e: + logger.warning( + "Caught Exception '{}' for co-ordinates {}".format( + e, raw_coordinate_string + ) + ) + return cleaned_coordinates + + @staticmethod + def _get_province_from_coord(coordinate): + """Expects a cleaned coordinate""" + key = f"coordinate province {coordinate['latitude']}, {coordinate['longitude']}" + province_name = cache.get(key, default="cache-miss") + if province_name == "cache-miss": + logger.info(f"Coordinate Province Cache MISS for coordinate {key}") + params = {"type": "PR"} + province_result = requests.get( + MAPIT_POINT_API_URL.format( + coordinate["longitude"], coordinate["latitude"] + ), + params=params, + ) + province_result.raise_for_status() + r = province_result.json() + list_of_objects_returned = list(r.values()) + if len(list_of_objects_returned) > 0: + province_name = list_of_objects_returned[0]["name"] + else: + province_name = None + cache.set(key, province_name) + else: + logger.info(f"Coordinate Province Cache HIT for coordinate {key}") + return province_name + + @staticmethod + def _get_province_from_project_name(project_name): + """Searches name of project for province name or abbreviation""" + project_name_slug = slugify(project_name) + new_dict = {} + for prov_name in prov_abbrev.keys(): + new_dict[prov_name] = slugify(prov_name) + for name, slug in new_dict.items(): + if slug in project_name_slug: + return name + return None + + @classmethod + def get_provinces(cls, cleaned_coordinates=None, project_name=""): + """Returns a list of provinces based on values in self.coordinates""" + provinces = set() + if cleaned_coordinates: + for c in cleaned_coordinates: + province = cls._get_province_from_coord(c) + if province: + provinces.add(province) + else: + logger.warning( + "Couldn't find GPS co-ordinates for infrastructure project '{}' on MapIt: {}".format( + project_name, c + ) + ) + else: + province = cls._get_province_from_project_name(project_name) + if province: + logger.info("Found province {} in project name".format(province)) + provinces.add(province) + return list(provinces) + + @staticmethod + def _build_expenditure_item(project): + return { + "year": project.financial_year, + "amount": project.amount_rands, + "budget_phase": project.budget_phase, + } + + def build_complete_expenditure(self): + complete_expenditure = [] + projects = InfrastructureProjectPart.objects.filter( + project_slug=self.project_slug + ) + for project in projects: + complete_expenditure.append(self._build_expenditure_item(project)) + return complete_expenditure + + +prov_keys = prov_abbrev.keys() +prov_choices = tuple([(prov_key, prov_key) for prov_key in prov_keys]) + + +class Event(models.Model): + start_date = models.DateField(default=datetime.now) + date = models.CharField(max_length=255) + type = models.CharField( + max_length=255, + choices=( + ("hackathon", "hackathon"), + ("dataquest", "dataquest"), + ("cid", "cid"), + ("gift-dataquest", "gift-dataquest"), + ), + ) + province = models.CharField(max_length=255, choices=prov_choices) + where = models.CharField(max_length=255) + url = models.URLField(blank=True, null=True) + notes_url = models.URLField(blank=True, null=True) + video_url = models.URLField(blank=True, null=True) + rsvp_url = models.URLField(blank=True, null=True) + presentation_url = models.URLField(blank=True, null=True) + status = models.CharField( + max_length=255, + default="upcoming", + choices=(("upcoming", "upcoming"), ("past", "past")), + ) + + class Meta: + ordering = ["-start_date"] + + def __str__(self): + return "{} {} ({} {})".format(self.type, self.date, self.where, self.province) + + def get_absolute_url(self): + return reverse("events") + + +class VideoLanguage(SortableMixin): + label = models.CharField(max_length=255) + youtube_id = models.CharField(max_length=255, null=True, blank=True) + video = models.ForeignKey("Video", null=True, blank=True, on_delete=models.SET_NULL) + video_language_order = models.PositiveIntegerField( + default=0, editable=False, db_index=True + ) + + class Meta: + ordering = ["video_language_order"] + + def __str__(self): + return self.label + + +class Video(SortableMixin): + title_id = models.CharField(max_length=255) + title = models.CharField(max_length=255) + description = models.TextField(max_length=510) + video_order = models.PositiveIntegerField(default=0, editable=False, db_index=True) + + class Meta: + ordering = ["video_order"] + + def __str__(self): + return self.title + + def get_absolute_url(self): + return reverse("videos") + + +class FAQ(SortableMixin): + title = models.CharField(max_length=1024) + content = ckeditor_fields.RichTextField() + the_order = models.PositiveIntegerField(default=0, editable=False, db_index=True) + + def __str__(self): + return self.title + + class Meta: + verbose_name = "FAQ" + verbose_name_plural = "FAQs" + ordering = ["the_order"] + + +class Quarter(models.Model): + number = models.IntegerField(unique=True) + + class Meta: + ordering = ["number"] + + def __str__(self): + return "Quarter %d" % self.number + + +def irm_snapshot_file_path(instance, filename): + extension = filename.split(".")[-1] + return ( + f"irm-snapshots/{uuid.uuid4()}/" + f"{instance.sphere.financial_year.slug}-Q{instance.quarter.number}-" + f"{instance.sphere.slug}-taken-{instance.date_taken.isoformat()[:18]}.{extension}" + ) + + +class IRMSnapshot(models.Model): + """ + This represents a particular snapshot from the Infrastructure Reporting Model + (IRM) database + """ + + sphere = models.ForeignKey(Sphere, on_delete=models.CASCADE) + quarter = models.ForeignKey(Quarter, on_delete=models.CASCADE) + date_taken = models.DateTimeField() + file = models.FileField(upload_to=irm_snapshot_file_path) + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) + updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) + + class Meta: + ordering = ["sphere__financial_year__slug", "quarter__number"] + verbose_name = "IRM Snapshot" + unique_together = ["sphere", "quarter"] + + def __str__(self): + return ( + f"{self.sphere.name} " + f"{self.sphere.financial_year.slug} Q{self.quarter.number} " + f"taken {self.date_taken.isoformat()[:18]}" + ) + + +class InfraProject(models.Model): + """ + This represents a project, grouping its snapshots by project's ID in IRM. + This assumes the same project will have the same ID in IRM across financial years. + + The internal ID of these instances is used as the ID in the URL, since we don't + want to expose IRM IDs publicly but we want to have a consistent URL for projects. + + We don't delete these even when there's no snapshots associated with them + so that the URL based on this id remains consistent in case projects with this + IRM ID are uploaded after snapshots are deleted. + """ + + IRM_project_id = models.IntegerField() + sphere_slug = models.CharField(max_length=100) + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) + updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) + + class Meta: + verbose_name = "Infrastructure project" + unique_together = ["sphere_slug", "IRM_project_id"] + + def __str__(self): + if self.project_snapshots.count(): + return self.project_snapshots.latest().name + else: + return f"{self.sphere_slug} IRM project ID {self.IRM_project_id} (no snapshots loaded)" + + def get_slug(self): + return slugify( + "{0} {1}".format( + self.project_snapshots.latest().name, + self.project_snapshots.latest().province, + ) + ) + + def get_absolute_url(self): + args = [self.pk, self.get_slug()] + return reverse("infra-project-detail", args=args) + + @property + def csv_download_url(self): + return reverse( + "infra-project-detail-csv-download", + args=(self.id, self.get_slug()), + ) + + +class InfraProjectSnapshot(models.Model): + """This represents a snapshot of a project, as it occurred in an IRM snapshot""" + + irm_snapshot = models.ForeignKey( + IRMSnapshot, on_delete=models.CASCADE, related_name="project_snapshots" + ) + project = models.ForeignKey( + InfraProject, on_delete=models.CASCADE, related_name="project_snapshots" + ) + project_number = models.CharField(max_length=1024, blank=True, null=True) + name = models.CharField(max_length=1024, blank=True, null=True) + department = models.CharField(max_length=1024, blank=True, null=True) + sector = models.CharField(max_length=1024, blank=True, null=True) + province = models.CharField(max_length=1024, blank=True, null=True) + local_municipality = models.CharField(max_length=1024, blank=True, null=True) + district_municipality = models.CharField(max_length=1024, blank=True, null=True) + latitude = models.CharField(max_length=20, blank=True, null=True) + longitude = models.CharField(max_length=20, blank=True, null=True) + status = models.CharField(max_length=1024, blank=True, null=True) + budget_programme = models.CharField(max_length=1024, blank=True, null=True) + primary_funding_source = models.CharField(max_length=1024, blank=True, null=True) + nature_of_investment = models.CharField(max_length=1024, blank=True, null=True) + funding_status = models.CharField(max_length=1024, blank=True, null=True) + program_implementing_agent = models.CharField( + max_length=1024, blank=True, null=True + ) + principle_agent = models.CharField(max_length=1024, blank=True, null=True) + main_contractor = models.CharField(max_length=1024, blank=True, null=True) + other_parties = models.TextField(blank=True, null=True) + + # Dates + start_date = models.DateField(blank=True, null=True) + estimated_construction_start_date = models.DateField(blank=True, null=True) + estimated_completion_date = models.DateField(blank=True, null=True) + contracted_construction_end_date = models.DateField(blank=True, null=True) + estimated_construction_end_date = models.DateField(blank=True, null=True) + + # Budgets and spending + total_professional_fees = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + total_construction_costs = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + variation_orders = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + estimated_total_project_cost = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + expenditure_from_previous_years_professional_fees = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + expenditure_from_previous_years_construction_costs = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + expenditure_from_previous_years_total = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + project_expenditure_total = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + main_appropriation_professional_fees = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + adjusted_appropriation_professional_fees = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + main_appropriation_construction_costs = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + adjusted_appropriation_construction_costs = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + main_appropriation_total = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + adjusted_appropriation_total = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + actual_expenditure_q1 = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + actual_expenditure_q2 = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + actual_expenditure_q3 = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + actual_expenditure_q4 = models.DecimalField( + max_digits=20, decimal_places=2, blank=True, null=True + ) + + # Metadata + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) + updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) + + class Meta: + ordering = [ + "irm_snapshot__sphere__financial_year__slug", + "irm_snapshot__quarter__number", + ] + get_latest_by = "irm_snapshot" + verbose_name = "Infrastructure project snapshot" + unique_together = ["irm_snapshot", "project"] + + @property + def government(self): + if self.irm_snapshot.sphere.slug == NATIONAL_SLUG: + return "South Africa" + elif self.irm_snapshot.sphere.slug == PROVINCIAL_SLUG: + return self.province + else: + raise Exception(f"Unexpected sphere {self.irm_snapshot.sphere}") + + @property + def government_label(self): + if self.irm_snapshot.sphere.slug == NATIONAL_SLUG: + return "National" + elif self.irm_snapshot.sphere.slug == PROVINCIAL_SLUG: + return self.province + else: + raise Exception(f"Unexpected sphere {self.irm_snapshot.sphere}") + + def __str__(self): + return self.name + + +class NavContextMixin: + def get_context(self, request): + context = super().get_context(request) + nav = MainMenuItem.objects.prefetch_related("children").all() + + context["navbar"] = nav + + for item in nav: + if item.name and item.url and request.path.startswith(item.url): + context["selected_tab"] = item.name + + return context + + +class WagtailHomePage(NavContextMixin, Page): + max_count = 1 + + +class CustomPage(NavContextMixin, Page): + body = StreamField( + [ + ("section", SectionBlock()), + ("html", wagtail_blocks.RawHTMLBlock()), + ("chart_embed", DescriptionEmbedBlock()), + ] + ) + + content_panels = Page.content_panels + [ + StreamFieldPanel("body"), + ] + + +class LearningIndexPage(NavContextMixin, Page): + parent_page_types = ["budgetportal.WagtailHomePage"] + subpage_types = ["budgetportal.GuideIndexPage"] + max_count = 1 + + +class PostIndexPage(NavContextMixin, Page): + parent_page_types = ["budgetportal.WagtailHomePage"] + subpage_types = ["budgetportal.PostPage"] + max_count = 1 + intro = RichTextField(blank=True) + + content_panels = Page.content_panels + [FieldPanel("intro", classname="full")] + + +class GuideIndexPage(NavContextMixin, Page): + max_count = 1 + parent_page_types = ["budgetportal.LearningIndexPage"] + subpage_types = ["budgetportal.GuidePage"] + intro = RichTextField(blank=True) + + content_panels = Page.content_panels + [FieldPanel("intro", classname="full")] + + def get_context(self, request, *args, **kwargs): + context = super().get_context(request, *args, **kwargs) + guides = {p.title: p for p in self.get_children().live()} + for external in CategoryGuide.objects.filter(external_url__isnull=False): + guides[external.external_url_title] = external + + context["guides"] = OrderedDict(sorted(guides.items())).values() + return context + + +class GuidePage(NavContextMixin, Page): + parent_page_types = ["budgetportal.GuideIndexPage"] + body = StreamField( + [ + ("section", SectionBlock()), + ("html", wagtail_blocks.RawHTMLBlock()), + ("chart_embed", DescriptionEmbedBlock()), + ] + ) + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) + updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) + image = models.ForeignKey( + "wagtailimages.Image", + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="+", + ) + + content_panels = Page.content_panels + [ + StreamFieldPanel("body"), + ImageChooserPanel("image"), + ] + + +class CategoryGuide(models.Model): + """Link GuidePages or external URLs to dataset category slugs""" + + category_slug = models.SlugField(max_length=200, unique=True) + guide_page = models.ForeignKey( + GuidePage, null=True, blank=True, on_delete=models.CASCADE + ) + external_url = models.URLField(null=True, blank=True) + external_url_title = models.CharField( + max_length=200, + null=True, + blank=True, + help_text=( + "Only shown if External URL is used. This may be truncated so " + "use a short description that will also be seen on the external page." + ), + ) + external_url_description = models.TextField( + null=True, + blank=True, + help_text=( + "Only shown if External URL is used. This may be truncated so " + "use a short description that will also be seen on the external page." + ), + ) + + def __str__(self): + return "{} - {}".format( + self.category_slug, self.guide_page or self.external_url + ) + + def clean(self): + if self.external_url is None and self.guide_page is None: + raise ValidationError("Either Guide Page or External URL must be set.") + if self.external_url is not None and self.guide_page is not None: + raise ValidationError("Only one of Guide Page or External URL may be set.") + if self.external_url is not None and self.external_url_title is None: + raise ValidationError("Title is required when using External URL.") + + return super().clean() + + +class PostPage(NavContextMixin, Page): + parent_page_types = ["budgetportal.PostIndexPage"] + body = StreamField( + [ + ("section", SectionBlock()), + ("html", wagtail_blocks.RawHTMLBlock()), + ] + ) + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) + updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) + image = models.ForeignKey( + "wagtailimages.Image", + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="+", + ) + + content_panels = Page.content_panels + [ + StreamFieldPanel("body"), + ] + + +class MainMenuItem(SortableMixin): + name = models.CharField(max_length=100) + label = models.CharField(max_length=200) + url = models.CharField( + max_length=1000, + help_text="Use URLs relative to the site root (e.g. /about) for urls on this site.", + blank=True, + null=True, + ) + align_right = models.BooleanField() + main_menu_item_order = models.PositiveIntegerField( + default=0, editable=False, db_index=True + ) + + class Meta: + ordering = ["main_menu_item_order"] + + def __str__(self): + return f"{self.label} ({self.url}) ({self.children.count()} children)" + + +class SubMenuItem(SortableMixin): + parent = models.ForeignKey( + MainMenuItem, on_delete=models.CASCADE, related_name="children" + ) + name = models.CharField(max_length=100) + label = models.CharField(max_length=200) + url = models.CharField( + max_length=1000, + help_text="Use URLs relative to the site root (e.g. /about) for urls on this site.", + ) + sub_menu_item_order = models.PositiveIntegerField( + default=0, editable=False, db_index=True + ) + + class Meta: + ordering = ["sub_menu_item_order"] + + def __str__(self): + return f"{self.parent.label} > {self.label} ({self.url})" + + +class Notice(SortableMixin): + """ + Any number of notices shown at the top of the site. Intended e.g. to configure + the staging site to make it clear to users that that instance is not necessarily + correct, but for testing only. + """ + + description = models.CharField(max_length=1024) + content = ckeditor_fields.RichTextField() + + notice_order = models.PositiveIntegerField(default=0, editable=False, db_index=True) + + class Meta: + ordering = ["notice_order"] + + def __str__(self): + return self.description + + +class ResourceLink(models.Model): + title = models.CharField(max_length=150) + url = models.URLField(null=True, blank=True) + description = models.CharField(max_length=300) + resource_link_order = models.PositiveIntegerField( + default=0, db_index=True, help_text="Links are shown in this order." + ) + sphere_slug = models.CharField( + max_length=100, + choices=[ + ("all", "All"), + ] + + SPHERE_SLUG_CHOICES, + default="all", + verbose_name="Sphere", + help_text="Only show on pages for this sphere or all spheres.", + ) + + panels = [ + MultiFieldPanel( + [ + FieldPanel("title"), + FieldPanel("url"), + FieldPanel("description"), + ], + heading="Content", + ), + MultiFieldPanel( + [ + FieldPanel("resource_link_order"), + FieldPanel("sphere_slug"), + ], + heading="Display options", + ), + ] + + def __str__(self): + return self.title + + class Meta: + abstract = True + ordering = ["resource_link_order"] + + +def showcase_item_file_path(instance, filename): + extension = filename.split(".")[-1] + return f"showcase-items/{uuid.uuid4()}.{extension}" + + +class ShowcaseItem(SortableMixin): + name = models.CharField(max_length=200) + description = models.CharField(max_length=400) + cta_text_1 = models.CharField(max_length=200, verbose_name="Call to action text 1") + cta_link_1 = models.URLField( + null=True, blank=True, verbose_name="Call to action link 1" + ) + cta_text_2 = models.CharField( + max_length=200, verbose_name="Call to action text 2", null=True, blank=True + ) + cta_link_2 = models.URLField( + null=True, blank=True, verbose_name="Call to action link 2" + ) + second_cta_type = models.CharField( + max_length=255, + choices=(("primary", "Primary"), ("secondary", "Secondary")), + verbose_name="Second call to action type", + ) + file = models.FileField( + upload_to=showcase_item_file_path, + help_text=mark_safe( + "
    • Thumbnail aspect ratio should be 1.91:1.
      • " + "
    • Recommended resolution is 1200 x 630 px.
      • " + "
    • Main focus of image should be centered occupying 630 x 630 px in the middle of the image.
    " + ), + ) + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) + updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) + item_order = models.PositiveIntegerField(default=0, editable=False, db_index=True) + + class Meta: + ordering = ["item_order"] + + def __str__(self): + return self.name + + +@register_snippet +class ProcurementResourceLink(ResourceLink): + class Meta: + verbose_name = "Procurement resource link" + verbose_name_plural = "Procurement resource links" + + +@register_snippet +class PerformanceResourceLink(ResourceLink): + class Meta: + verbose_name = "Performance resource link" + verbose_name_plural = "Performance resource links" + + +@register_snippet +class InYearMonitoringResourceLink(ResourceLink): + class Meta: + verbose_name = "In-year monitoring resource link" + verbose_name_plural = "In-year monitoring resource links" diff --git a/budgetportal/models.py b/budgetportal/models/government.py similarity index 68% rename from budgetportal/models.py rename to budgetportal/models/government.py index 8fcb7f0f4..b5bfd3e0b 100644 --- a/budgetportal/models.py +++ b/budgetportal/models/government.py @@ -1,55 +1,32 @@ -import logging -import re -import uuid -from collections import OrderedDict -from datetime import datetime -from decimal import Decimal -from pprint import pformat -from urllib.parse import quote - -import requests -from slugify import slugify - -import ckeditor.fields as ckeditor_fields -from adminsortable.models import SortableMixin from autoslug import AutoSlugField -from budgetportal.blocks import DescriptionEmbedBlock, SectionBlock from budgetportal.datasets import Dataset, get_expenditure_time_series_dataset +from collections import OrderedDict +from decimal import Decimal from django.conf import settings -from django.core.cache import cache -from django.core.exceptions import MultipleObjectsReturned, ValidationError from django.db import models -from django.urls import reverse from partial_index import PartialIndex -from wagtail.admin.edit_handlers import FieldPanel, MultiFieldPanel, StreamFieldPanel -from wagtail.core import blocks as wagtail_blocks -from wagtail.core.fields import RichTextField, StreamField -from wagtail.core.models import Page -from wagtail.images.edit_handlers import ImageChooserPanel -from wagtail.snippets.models import register_snippet +from pprint import pformat +from slugify import slugify +from urllib.parse import quote +import logging +from django.urls import reverse +import requests logger = logging.getLogger(__name__) ckan = settings.CKAN -URL_LENGTH_LIMIT = 2000 - -CKAN_DATASTORE_URL = settings.CKAN_URL + "/api/3/action" "/datastore_search_sql" - -MAPIT_POINT_API_URL = "https://mapit.code4sa.org/point/4326/{},{}" - -DIRECT_CHARGE_NRF = "Direct charge against the National Revenue Fund" - -prov_abbrev = { - "Eastern Cape": "EC", - "Free State": "FS", - "Gauteng": "GT", - "KwaZulu-Natal": "NL", - "Limpopo": "LP", - "Mpumalanga": "MP", - "Northern Cape": "NC", - "North West": "NW", - "Western Cape": "WC", -} +NATIONAL_SLUG = "national" +PROVINCIAL_SLUG = "provincial" +SPHERE_SLUG_CHOICES = [ + ( + NATIONAL_SLUG, + "National", + ), + ( + PROVINCIAL_SLUG, + "Provincial", + ), +] # Budget Phase IDs for the 7-year overview period TRENDS_AND_ESTIMATES_PHASES = [ @@ -75,28 +52,11 @@ "actual": "Audit Outcome", } -NATIONAL_SLUG = "national" -PROVINCIAL_SLUG = "provincial" -SPHERE_SLUG_CHOICES = [ - (NATIONAL_SLUG, "National",), - (PROVINCIAL_SLUG, "Provincial",), -] +DIRECT_CHARGE_NRF = "Direct charge against the National Revenue Fund" +URL_LENGTH_LIMIT = 2000 -class Homepage(models.Model): - main_heading = models.CharField(max_length=1000, blank=True) - sub_heading = models.CharField(max_length=1000, blank=True) - primary_button_label = models.CharField(max_length=1000, blank=True) - primary_button_url = models.CharField(max_length=1000, blank=True) - primary_button_target = models.CharField(max_length=1000, blank=True) - secondary_button_label = models.CharField(max_length=1000, blank=True) - secondary_button_url = models.CharField(max_length=1000, blank=True) - secondary_button_target = models.CharField(max_length=1000, blank=True) - call_to_action_sub_heading = models.CharField(max_length=1000, blank=True) - call_to_action_heading = models.CharField(max_length=1000, blank=True) - call_to_action_link_label = models.CharField(max_length=1000, blank=True) - call_to_action_link_url = models.CharField(max_length=1000, blank=True) - call_to_action_link_target = models.CharField(max_length=1000, blank=True) +CKAN_DATASTORE_URL = settings.CKAN_URL + "/api/3/action" "/datastore_search_sql" class FinancialYear(models.Model): @@ -158,7 +118,11 @@ def __str__(self): class Sphere(models.Model): organisational_unit = "sphere" name = models.CharField(max_length=200) - slug = AutoSlugField(populate_from="name", max_length=200, always_update=True,) + slug = AutoSlugField( + populate_from="name", + max_length=200, + always_update=True, + ) financial_year = models.ForeignKey( FinancialYear, on_delete=models.CASCADE, related_name="spheres" ) @@ -208,7 +172,21 @@ def __str__(self): return "<%s %s>" % (self.__class__.__name__, self.get_url_path()) +class DepartmentManager(models.Manager): + def get_by_natural_key( + self, financial_year, sphere_slug, government_slug, department_slug + ): + return self.get( + slug=department_slug, + government__slug=government_slug, + government__sphere__slug=sphere_slug, + government__sphere__financial_year__slug=financial_year, + ) + + class Department(models.Model): + objects = DepartmentManager() + organisational_unit = "department" government = models.ForeignKey( Government, on_delete=models.CASCADE, related_name="departments" @@ -317,7 +295,7 @@ def create_dataset(self, name, title, group_name): return Dataset.from_package(ckan.action.package_create(**dataset_fields)) def get_latest_website_url(self): - """ Always return the latest available non-null URL, even for old departments. """ + """Always return the latest available non-null URL, even for old departments.""" newer_departments = Department.objects.filter( government__slug=self.government.slug, government__sphere__slug=self.government.sphere.slug, @@ -327,11 +305,11 @@ def get_latest_website_url(self): return newer_departments.first().website_url if newer_departments else None def get_url_path(self): - """ e.g. 2018-19/national/departments/military-veterans """ + """e.g. 2018-19/national/departments/military-veterans""" return "%s/departments/%s" % (self.government.get_url_path(), self.slug) def get_preview_url_path(self): - """ e.g. 2018-19/previews/national/south-africa/agriculture-and-fisheries """ + """e.g. 2018-19/previews/national/south-africa/agriculture-and-fisheries""" return "%s/previews/%s/%s/%s" % ( self.government.sphere.financial_year.slug, self.government.sphere.slug, @@ -346,8 +324,8 @@ def get_financial_year(self): return self.government.sphere.financial_year def get_latest_department_instance(self): - """ Try to find the department in the most recent year with the same slug. - Continue traversing backwards in time until found, or until the original year has been reached. """ + """Try to find the department in the most recent year with the same slug. + Continue traversing backwards in time until found, or until the original year has been reached.""" newer_departments = Department.objects.filter( government__slug=self.government.slug, government__sphere__slug=self.government.sphere.slug, @@ -1006,7 +984,7 @@ def _get_budget_direct_charges(self, openspending_api): return subprog_dict.values() if subprog_dict else None def get_all_budget_totals_by_year_and_phase(self): - """ Returns the total for each year:phase combination from the expenditure time series dataset. """ + """Returns the total for each year:phase combination from the expenditure time series dataset.""" dataset = get_expenditure_time_series_dataset(self.government.sphere.slug) if not dataset: return None @@ -1048,8 +1026,8 @@ def get_all_budget_totals_by_year_and_phase(self): return total_budgets def get_national_expenditure_treemap(self, financial_year_id, budget_phase): - """ Returns a data object for each department, year and phase. Adds additional data required for the Treemap. - From the Expenditure Time Series dataset. """ + """Returns a data object for each department, year and phase. Adds additional data required for the Treemap. + From the Expenditure Time Series dataset.""" # Take budget sphere, year and phase as positional arguments from URL # Output expenditure specific to sphere:year:phase scope, simple list of objects try: @@ -1157,7 +1135,7 @@ def get_national_expenditure_treemap(self, financial_year_id, budget_phase): ) def get_provincial_expenditure_treemap(self, financial_year_id, budget_phase): - """ Returns a list of department objects nested in provinces. """ + """Returns a list of department objects nested in provinces.""" # Take budget sphere, year and phase as positional arguments from URL # Output expenditure specific to sphere:year:phase scope, simple list of objects try: @@ -1590,469 +1568,6 @@ def __str__(self): return "<%s %s>" % (self.__class__.__name__, self.get_url_path()) -class InfrastructureProjectPart(models.Model): - administration_type = models.CharField(max_length=255) - government_institution = models.CharField(max_length=255) - sector = models.CharField(max_length=255) - project_name = models.CharField(max_length=255) - project_description = models.TextField() - nature_of_investment = models.CharField(max_length=255) - infrastructure_type = models.CharField(max_length=255) - current_project_stage = models.CharField(max_length=255) - sip_category = models.CharField(max_length=255) - br_featured = models.CharField(max_length=255) - featured = models.BooleanField() - budget_phase = models.CharField(max_length=255) - project_slug = models.CharField(max_length=255) - amount_rands = models.BigIntegerField(blank=True, null=True, default=None) - financial_year = models.CharField(max_length=4) - project_value_rands = models.BigIntegerField(default=0) - provinces = models.CharField(max_length=510, default="") - gps_code = models.CharField(max_length=255, default="") - - # PPP fields - partnership_type = models.CharField(max_length=255, null=True, blank=True) - date_of_close = models.CharField(max_length=255, null=True, blank=True) - duration = models.CharField(max_length=255, null=True, blank=True) - financing_structure = models.CharField(max_length=255, null=True, blank=True) - project_value_rand_million = models.CharField(max_length=255, null=True, blank=True) - form_of_payment = models.CharField(max_length=255, null=True, blank=True) - - class Meta: - verbose_name = "National infrastructure project part" - - def __str__(self): - return "{} ({} {})".format( - self.project_slug, self.budget_phase, self.financial_year - ) - - def get_url_path(self): - return "/infrastructure-projects/{}".format(self.project_slug) - - def get_absolute_url(self): - return reverse("infrastructure-projects", args=[self.project_slug]) - - def calculate_projected_expenditure(self): - """ Calculate sum of predicted amounts from a list of records """ - projected_records_for_project = InfrastructureProjectPart.objects.filter( - budget_phase="MTEF", project_slug=self.project_slug - ) - projected_expenditure = 0 - for project in projected_records_for_project: - projected_expenditure += float(project.amount_rands or 0.0) - return projected_expenditure - - @staticmethod - def _parse_coordinate(coordinate): - """ Expects a single set of coordinates (lat, long) split by a comma """ - if not isinstance(coordinate, str): - raise TypeError("Invalid type for coordinate parsing") - lat_long = [float(x) for x in coordinate.split(",")] - cleaned_coordinate = {"latitude": lat_long[0], "longitude": lat_long[1]} - return cleaned_coordinate - - @classmethod - def clean_coordinates(cls, raw_coordinate_string): - cleaned_coordinates = [] - try: - if "and" in raw_coordinate_string: - list_of_coordinates = raw_coordinate_string.split("and") - for coordinate in list_of_coordinates: - cleaned_coordinates.append(cls._parse_coordinate(coordinate)) - elif "," in raw_coordinate_string: - cleaned_coordinates.append(cls._parse_coordinate(raw_coordinate_string)) - else: - logger.warning("Invalid co-ordinates: {}".format(raw_coordinate_string)) - except Exception as e: - logger.warning( - "Caught Exception '{}' for co-ordinates {}".format( - e, raw_coordinate_string - ) - ) - return cleaned_coordinates - - @staticmethod - def _get_province_from_coord(coordinate): - """ Expects a cleaned coordinate """ - key = f"coordinate province {coordinate['latitude']}, {coordinate['longitude']}" - province_name = cache.get(key, default="cache-miss") - if province_name == "cache-miss": - logger.info(f"Coordinate Province Cache MISS for coordinate {key}") - params = {"type": "PR"} - province_result = requests.get( - MAPIT_POINT_API_URL.format( - coordinate["longitude"], coordinate["latitude"] - ), - params=params, - ) - province_result.raise_for_status() - r = province_result.json() - list_of_objects_returned = list(r.values()) - if len(list_of_objects_returned) > 0: - province_name = list_of_objects_returned[0]["name"] - else: - province_name = None - cache.set(key, province_name) - else: - logger.info(f"Coordinate Province Cache HIT for coordinate {key}") - return province_name - - @staticmethod - def _get_province_from_project_name(project_name): - """ Searches name of project for province name or abbreviation """ - project_name_slug = slugify(project_name) - new_dict = {} - for prov_name in prov_abbrev.keys(): - new_dict[prov_name] = slugify(prov_name) - for name, slug in new_dict.items(): - if slug in project_name_slug: - return name - return None - - @classmethod - def get_provinces(cls, cleaned_coordinates=None, project_name=""): - """ Returns a list of provinces based on values in self.coordinates """ - provinces = set() - if cleaned_coordinates: - for c in cleaned_coordinates: - province = cls._get_province_from_coord(c) - if province: - provinces.add(province) - else: - logger.warning( - "Couldn't find GPS co-ordinates for infrastructure project '{}' on MapIt: {}".format( - project_name, c - ) - ) - else: - province = cls._get_province_from_project_name(project_name) - if province: - logger.info("Found province {} in project name".format(province)) - provinces.add(province) - return list(provinces) - - @staticmethod - def _build_expenditure_item(project): - return { - "year": project.financial_year, - "amount": project.amount_rands, - "budget_phase": project.budget_phase, - } - - def build_complete_expenditure(self): - complete_expenditure = [] - projects = InfrastructureProjectPart.objects.filter( - project_slug=self.project_slug - ) - for project in projects: - complete_expenditure.append(self._build_expenditure_item(project)) - return complete_expenditure - - -prov_keys = prov_abbrev.keys() -prov_choices = tuple([(prov_key, prov_key) for prov_key in prov_keys]) - - -class Event(models.Model): - start_date = models.DateField(default=datetime.now) - date = models.CharField(max_length=255) - type = models.CharField( - max_length=255, - choices=( - ("hackathon", "hackathon"), - ("dataquest", "dataquest"), - ("cid", "cid"), - ("gift-dataquest", "gift-dataquest"), - ), - ) - province = models.CharField(max_length=255, choices=prov_choices) - where = models.CharField(max_length=255) - url = models.URLField(blank=True, null=True) - notes_url = models.URLField(blank=True, null=True) - video_url = models.URLField(blank=True, null=True) - rsvp_url = models.URLField(blank=True, null=True) - presentation_url = models.URLField(blank=True, null=True) - status = models.CharField( - max_length=255, - default="upcoming", - choices=(("upcoming", "upcoming"), ("past", "past")), - ) - - class Meta: - ordering = ["-start_date"] - - def __str__(self): - return "{} {} ({} {})".format(self.type, self.date, self.where, self.province) - - def get_absolute_url(self): - return reverse("events") - - -class VideoLanguage(SortableMixin): - label = models.CharField(max_length=255) - youtube_id = models.CharField(max_length=255, null=True, blank=True) - video = models.ForeignKey("Video", null=True, blank=True, on_delete=models.SET_NULL) - video_language_order = models.PositiveIntegerField( - default=0, editable=False, db_index=True - ) - - class Meta: - ordering = ["video_language_order"] - - def __str__(self): - return self.label - - -class Video(SortableMixin): - title_id = models.CharField(max_length=255) - title = models.CharField(max_length=255) - description = models.TextField(max_length=510) - video_order = models.PositiveIntegerField(default=0, editable=False, db_index=True) - - class Meta: - ordering = ["video_order"] - - def __str__(self): - return self.title - - def get_absolute_url(self): - return reverse("videos") - - -class FAQ(SortableMixin): - title = models.CharField(max_length=1024) - content = ckeditor_fields.RichTextField() - the_order = models.PositiveIntegerField(default=0, editable=False, db_index=True) - - def __str__(self): - return self.title - - class Meta: - verbose_name = "FAQ" - verbose_name_plural = "FAQs" - ordering = ["the_order"] - - -class Quarter(models.Model): - number = models.IntegerField(unique=True) - - class Meta: - ordering = ["number"] - - def __str__(self): - return "Quarter %d" % self.number - - -def irm_snapshot_file_path(instance, filename): - extension = filename.split(".")[-1] - return ( - f"irm-snapshots/{uuid.uuid4()}/" - f"{instance.sphere.financial_year.slug}-Q{instance.quarter.number}-" - f"{instance.sphere.slug}-taken-{instance.date_taken.isoformat()[:18]}.{extension}" - ) - - -class IRMSnapshot(models.Model): - """ - This represents a particular snapshot from the Infrastructure Reporting Model - (IRM) database - """ - - sphere = models.ForeignKey(Sphere, on_delete=models.CASCADE) - quarter = models.ForeignKey(Quarter, on_delete=models.CASCADE) - date_taken = models.DateTimeField() - file = models.FileField(upload_to=irm_snapshot_file_path) - created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) - updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) - - class Meta: - ordering = ["sphere__financial_year__slug", "quarter__number"] - verbose_name = "IRM Snapshot" - unique_together = ["sphere", "quarter"] - - def __str__(self): - return ( - f"{self.sphere.name} " - f"{self.sphere.financial_year.slug} Q{self.quarter.number} " - f"taken {self.date_taken.isoformat()[:18]}" - ) - - -class InfraProject(models.Model): - """ - This represents a project, grouping its snapshots by project's ID in IRM. - This assumes the same project will have the same ID in IRM across financial years. - - The internal ID of these instances is used as the ID in the URL, since we don't - want to expose IRM IDs publicly but we want to have a consistent URL for projects. - - We don't delete these even when there's no snapshots associated with them - so that the URL based on this id remains consistent in case projects with this - IRM ID are uploaded after snapshots are deleted. - """ - - IRM_project_id = models.IntegerField() - sphere_slug = models.CharField(max_length=100) - created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) - updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) - - class Meta: - verbose_name = "Infrastructure project" - unique_together = ["sphere_slug", "IRM_project_id"] - - def __str__(self): - if self.project_snapshots.count(): - return self.project_snapshots.latest().name - else: - return f"{self.sphere_slug} IRM project ID {self.IRM_project_id} (no snapshots loaded)" - - def get_slug(self): - return slugify( - "{0} {1}".format( - self.project_snapshots.latest().name, - self.project_snapshots.latest().province, - ) - ) - - def get_absolute_url(self): - args = [self.pk, self.get_slug()] - return reverse("infra-project-detail", args=args) - - @property - def csv_download_url(self): - return reverse( - "infra-project-detail-csv-download", args=(self.id, self.get_slug()), - ) - - -class InfraProjectSnapshot(models.Model): - """This represents a snapshot of a project, as it occurred in an IRM snapshot""" - - irm_snapshot = models.ForeignKey( - IRMSnapshot, on_delete=models.CASCADE, related_name="project_snapshots" - ) - project = models.ForeignKey( - InfraProject, on_delete=models.CASCADE, related_name="project_snapshots" - ) - project_number = models.CharField(max_length=1024, blank=True, null=True) - name = models.CharField(max_length=1024, blank=True, null=True) - department = models.CharField(max_length=1024, blank=True, null=True) - sector = models.CharField(max_length=1024, blank=True, null=True) - province = models.CharField(max_length=1024, blank=True, null=True) - local_municipality = models.CharField(max_length=1024, blank=True, null=True) - district_municipality = models.CharField(max_length=1024, blank=True, null=True) - latitude = models.CharField(max_length=20, blank=True, null=True) - longitude = models.CharField(max_length=20, blank=True, null=True) - status = models.CharField(max_length=1024, blank=True, null=True) - budget_programme = models.CharField(max_length=1024, blank=True, null=True) - primary_funding_source = models.CharField(max_length=1024, blank=True, null=True) - nature_of_investment = models.CharField(max_length=1024, blank=True, null=True) - funding_status = models.CharField(max_length=1024, blank=True, null=True) - program_implementing_agent = models.CharField( - max_length=1024, blank=True, null=True - ) - principle_agent = models.CharField(max_length=1024, blank=True, null=True) - main_contractor = models.CharField(max_length=1024, blank=True, null=True) - other_parties = models.TextField(blank=True, null=True) - - # Dates - start_date = models.DateField(blank=True, null=True) - estimated_construction_start_date = models.DateField(blank=True, null=True) - estimated_completion_date = models.DateField(blank=True, null=True) - contracted_construction_end_date = models.DateField(blank=True, null=True) - estimated_construction_end_date = models.DateField(blank=True, null=True) - - # Budgets and spending - total_professional_fees = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - total_construction_costs = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - variation_orders = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - estimated_total_project_cost = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - expenditure_from_previous_years_professional_fees = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - expenditure_from_previous_years_construction_costs = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - expenditure_from_previous_years_total = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - project_expenditure_total = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - main_appropriation_professional_fees = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - adjusted_appropriation_professional_fees = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - main_appropriation_construction_costs = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - adjusted_appropriation_construction_costs = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - main_appropriation_total = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - adjusted_appropriation_total = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - actual_expenditure_q1 = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - actual_expenditure_q2 = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - actual_expenditure_q3 = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - actual_expenditure_q4 = models.DecimalField( - max_digits=20, decimal_places=2, blank=True, null=True - ) - - # Metadata - created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) - updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) - - class Meta: - ordering = [ - "irm_snapshot__sphere__financial_year__slug", - "irm_snapshot__quarter__number", - ] - get_latest_by = "irm_snapshot" - verbose_name = "Infrastructure project snapshot" - unique_together = ["irm_snapshot", "project"] - - @property - def government(self): - if self.irm_snapshot.sphere.slug == NATIONAL_SLUG: - return "South Africa" - elif self.irm_snapshot.sphere.slug == PROVINCIAL_SLUG: - return self.province - else: - raise Exception(f"Unexpected sphere {self.irm_snapshot.sphere}") - - @property - def government_label(self): - if self.irm_snapshot.sphere.slug == NATIONAL_SLUG: - return "National" - elif self.irm_snapshot.sphere.slug == PROVINCIAL_SLUG: - return self.province - else: - raise Exception(f"Unexpected sphere {self.irm_snapshot.sphere}") - - def __str__(self): - return self.name - - # https://stackoverflow.com/questions/35633037/search-for-document-in-solr-where-a-multivalue-field-is-either-empty # -or-has-a-sp def none_selected_query(vocab_name): @@ -2114,6 +1629,7 @@ def get_vocab_map(): def csv_url(aggregate_url): + print(aggregate_url) querystring = "?api_url=" + quote(aggregate_url) csv_url = reverse("openspending_csv") + querystring if len(csv_url) > URL_LENGTH_LIMIT: @@ -2123,271 +1639,3 @@ def csv_url(aggregate_url): % URL_LENGTH_LIMIT ) return csv_url - - -class NavContextMixin: - def get_context(self, request): - context = super().get_context(request) - nav = MainMenuItem.objects.prefetch_related("children").all() - - context["navbar"] = nav - - for item in nav: - if item.name and item.url and request.path.startswith(item.url): - context["selected_tab"] = item.name - - return context - - -class WagtailHomePage(NavContextMixin, Page): - max_count = 1 - - -class CustomPage(NavContextMixin, Page): - body = StreamField( - [ - ("section", SectionBlock()), - ("html", wagtail_blocks.RawHTMLBlock()), - ("chart_embed", DescriptionEmbedBlock()), - ] - ) - - content_panels = Page.content_panels + [ - StreamFieldPanel("body"), - ] - - -class LearningIndexPage(NavContextMixin, Page): - parent_page_types = ["budgetportal.WagtailHomePage"] - subpage_types = ["budgetportal.GuideIndexPage"] - max_count = 1 - - -class PostIndexPage(NavContextMixin, Page): - parent_page_types = ["budgetportal.WagtailHomePage"] - subpage_types = ["budgetportal.PostPage"] - max_count = 1 - intro = RichTextField(blank=True) - - content_panels = Page.content_panels + [FieldPanel("intro", classname="full")] - - -class GuideIndexPage(NavContextMixin, Page): - max_count = 1 - parent_page_types = ["budgetportal.LearningIndexPage"] - subpage_types = ["budgetportal.GuidePage"] - intro = RichTextField(blank=True) - - content_panels = Page.content_panels + [FieldPanel("intro", classname="full")] - - def get_context(self, request, *args, **kwargs): - context = super().get_context(request, *args, **kwargs) - guides = {p.title: p for p in self.get_children().live()} - for external in CategoryGuide.objects.filter(external_url__isnull=False): - guides[external.external_url_title] = external - - context["guides"] = OrderedDict(sorted(guides.items())).values() - return context - - -class GuidePage(NavContextMixin, Page): - parent_page_types = ["budgetportal.GuideIndexPage"] - body = StreamField( - [ - ("section", SectionBlock()), - ("html", wagtail_blocks.RawHTMLBlock()), - ("chart_embed", DescriptionEmbedBlock()), - ] - ) - created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) - updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) - image = models.ForeignKey( - "wagtailimages.Image", - null=True, - blank=True, - on_delete=models.SET_NULL, - related_name="+", - ) - - content_panels = Page.content_panels + [ - StreamFieldPanel("body"), - ImageChooserPanel("image"), - ] - - -class CategoryGuide(models.Model): - """Link GuidePages or external URLs to dataset category slugs""" - - category_slug = models.SlugField(max_length=200, unique=True) - guide_page = models.ForeignKey( - GuidePage, null=True, blank=True, on_delete=models.CASCADE - ) - external_url = models.URLField(null=True, blank=True) - external_url_title = models.CharField( - max_length=200, - null=True, - blank=True, - help_text=( - "Only shown if External URL is used. This may be truncated so " - "use a short description that will also be seen on the external page." - ), - ) - external_url_description = models.TextField( - null=True, - blank=True, - help_text=( - "Only shown if External URL is used. This may be truncated so " - "use a short description that will also be seen on the external page." - ), - ) - - def __str__(self): - return "{} - {}".format( - self.category_slug, self.guide_page or self.external_url - ) - - def clean(self): - if self.external_url is None and self.guide_page is None: - raise ValidationError("Either Guide Page or External URL must be set.") - if self.external_url is not None and self.guide_page is not None: - raise ValidationError("Only one of Guide Page or External URL may be set.") - if self.external_url is not None and self.external_url_title is None: - raise ValidationError("Title is required when using External URL.") - - return super().clean() - - -class PostPage(NavContextMixin, Page): - parent_page_types = ["budgetportal.PostIndexPage"] - body = StreamField( - [("section", SectionBlock()), ("html", wagtail_blocks.RawHTMLBlock()),] - ) - created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) - updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) - image = models.ForeignKey( - "wagtailimages.Image", - null=True, - blank=True, - on_delete=models.SET_NULL, - related_name="+", - ) - - content_panels = Page.content_panels + [ - StreamFieldPanel("body"), - ] - - -class MainMenuItem(SortableMixin): - name = models.CharField(max_length=100) - label = models.CharField(max_length=200) - url = models.CharField( - max_length=1000, - help_text="Use URLs relative to the site root (e.g. /about) for urls on this site.", - blank=True, - null=True, - ) - align_right = models.BooleanField() - main_menu_item_order = models.PositiveIntegerField( - default=0, editable=False, db_index=True - ) - - class Meta: - ordering = ["main_menu_item_order"] - - def __str__(self): - return f"{self.label} ({self.url}) ({self.children.count()} children)" - - -class SubMenuItem(SortableMixin): - parent = models.ForeignKey( - MainMenuItem, on_delete=models.CASCADE, related_name="children" - ) - name = models.CharField(max_length=100) - label = models.CharField(max_length=200) - url = models.CharField( - max_length=1000, - help_text="Use URLs relative to the site root (e.g. /about) for urls on this site.", - ) - sub_menu_item_order = models.PositiveIntegerField( - default=0, editable=False, db_index=True - ) - - class Meta: - ordering = ["sub_menu_item_order"] - - def __str__(self): - return f"{self.parent.label} > {self.label} ({self.url})" - - -class Notice(SortableMixin): - """ - Any number of notices shown at the top of the site. Intended e.g. to configure - the staging site to make it clear to users that that instance is not necessarily - correct, but for testing only. - """ - - description = models.CharField(max_length=1024) - content = ckeditor_fields.RichTextField() - - notice_order = models.PositiveIntegerField(default=0, editable=False, db_index=True) - - class Meta: - ordering = ["notice_order"] - - def __str__(self): - return self.description - - -class ResourceLink(models.Model): - title = models.CharField(max_length=150) - url = models.URLField(null=True, blank=True) - description = models.CharField(max_length=300) - resource_link_order = models.PositiveIntegerField( - default=0, db_index=True, help_text="Links are shown in this order." - ) - sphere_slug = models.CharField( - max_length=100, - choices=[("all", "All"),] + SPHERE_SLUG_CHOICES, - default="all", - verbose_name="Sphere", - help_text="Only show on pages for this sphere or all spheres.", - ) - - panels = [ - MultiFieldPanel( - [FieldPanel("title"), FieldPanel("url"), FieldPanel("description"),], - heading="Content", - ), - MultiFieldPanel( - [FieldPanel("resource_link_order"), FieldPanel("sphere_slug"),], - heading="Display options", - ), - ] - - def __str__(self): - return self.title - - class Meta: - abstract = True - ordering = ["resource_link_order"] - - -@register_snippet -class ProcurementResourceLink(ResourceLink): - class Meta: - verbose_name = "Procurement resource link" - verbose_name_plural = "Procurement resource links" - - -@register_snippet -class PerformanceResourceLink(ResourceLink): - class Meta: - verbose_name = "Performance resource link" - verbose_name_plural = "Performance resource links" - - -@register_snippet -class InYearMonitoringResourceLink(ResourceLink): - class Meta: - verbose_name = "In-year monitoring resource link" - verbose_name_plural = "In-year monitoring resource links" diff --git a/budgetportal/openspending.py b/budgetportal/openspending.py index 0acc91845..c248c57ea 100644 --- a/budgetportal/openspending.py +++ b/budgetportal/openspending.py @@ -101,8 +101,8 @@ def filter_dept(self, result, dept_name): @staticmethod def aggregate_by_refs(aggregate_refs, cells): - """ Simulates a basic version of aggregation via Open Spending API - Accepts a list of cells and a list of any number of column references. """ + """Simulates a basic version of aggregation via Open Spending API + Accepts a list of cells and a list of any number of column references.""" aggregated_cells = list() unique_reference_combos = list() diff --git a/budgetportal/settings.py b/budgetportal/settings.py index 7b3b5eacc..5af5e455f 100644 --- a/budgetportal/settings.py +++ b/budgetportal/settings.py @@ -47,7 +47,7 @@ SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") DEBUG_TOOLBAR = os.environ.get("DJANGO_DEBUG_TOOLBAR", "false").lower() == "true" -print("Django Debug Toolbar %s." % "enabled" if DEBUG_TOOLBAR else "disabled") +print("Django Debug Toolbar %s." % ("enabled" if DEBUG_TOOLBAR else "disabled")) DEBUG_TOOLBAR_CONFIG = { "SHOW_TOOLBAR_CALLBACK": "budgetportal.debug_toolbar_config.show_toolbar_check" } @@ -62,8 +62,12 @@ INSTALLED_APPS = [ "whitenoise.runserver_nostatic", + "constance", + "constance.backends.database", "budgetportal.apps.BudgetPortalConfig", "budgetportal.webflow", + "performance", + "iym", "wagtail.contrib.forms", "wagtail.contrib.redirects", "wagtail.embeds", @@ -107,6 +111,16 @@ "storages", ] +CONSTANCE_BACKEND = "constance.backends.database.DatabaseBackend" + +CONSTANCE_CONFIG = { + "EQPRS_DATA_ENABLED": ( + False, + "enabling / disabling summary on department page", + bool, + ) +} + if DEBUG_TOOLBAR: INSTALLED_APPS.append("debug_toolbar") @@ -135,7 +149,6 @@ SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies" - db_config = dj_database_url.config() db_config["ATOMIC_REQUESTS"] = True @@ -156,7 +169,6 @@ # https://docs.wagtail.io/en/v2.7.1/advanced_topics/deploying.html AWS_S3_FILE_OVERWRITE = False - SOLR_URL = os.environ["SOLR_URL"] HAYSTACK_CONNECTIONS = { @@ -167,7 +179,6 @@ } } - # Caches if DEBUG: if os.environ.get("DEBUG_CACHE", "false").lower() == "true": @@ -188,7 +199,6 @@ } } - CKAN_URL = os.environ.get("CKAN_URL", "https://data.vulekamali.gov.za") CKAN_API_KEY = os.environ.get("CKAN_API_KEY", None) CKAN = RemoteCKAN(CKAN_URL, apikey=CKAN_API_KEY) @@ -206,6 +216,11 @@ os.environ.get("BUST_OPENSPENDING_CACHE", "false").lower() == "true" ) OPENSPENDING_HOST = os.environ.get("OPENSPENDING_HOST", "https://openspending.org") +OPENSPENDING_USER_ID = os.environ.get("OPENSPENDING_USER_ID", "") +OPENSPENDING_API_KEY = os.environ.get("OPENSPENDING_API_KEY", "") +OPENSPENDING_DATASET_CREATE_SUFFIX = os.environ.get( + "OPENSPENDING_DATASET_CREATE_SUFFIX", "" +) # http://django-allauth.readthedocs.io/en/latest/configuration.html ACCOUNT_ADAPTER = "budgetportal.allauthadapters.AccountAdapter" @@ -380,8 +395,9 @@ Q_CLUSTER = { "name": "Something", "workers": 1, - "timeout": 30 * 60, # Timeout a task after this many seconds - "retry": 5, + "max_attempts": 1, + "timeout": 60 * 60 * 6, # 6 hours - Timeout a task after this many seconds + "retry": 60 * 60 * 6 + 1, # 6 hours - Seconds to wait before retrying a task "queue_limit": 1, "bulk": 1, "orm": "default", # Use Django ORM as storage backend diff --git a/budgetportal/summaries.py b/budgetportal/summaries.py index 336d74eb3..cae35bec7 100644 --- a/budgetportal/summaries.py +++ b/budgetportal/summaries.py @@ -10,8 +10,8 @@ EXPENDITURE_TIME_SERIES_PHASE_MAPPING, Department, FinancialYear, - csv_url, ) +from .models.government import csv_url logger = logging.getLogger(__name__) @@ -21,7 +21,7 @@ def get_focus_area_preview(financial_year): - """ Returns data for the focus area preview pages. """ + """Returns data for the focus area preview pages.""" national_expenditure_results, national_os_api = get_focus_area_data( financial_year, "national" @@ -234,7 +234,7 @@ def get_focus_area_url_path(financial_year_slug, name): def get_consolidated_expenditure_treemap(financial_year): - """ Returns a data object for each function group for a specific year. Used by the consolidated treemap. """ + """Returns a data object for each function group for a specific year. Used by the consolidated treemap.""" expenditure = [] dataset = get_consolidated_expenditure_budget_dataset(financial_year) @@ -336,15 +336,19 @@ def get_preview_page(financial_year_id, phase_slug, government_slug, sphere_slug ] # Used to determine programmes for departments - expenditure_results_filter_government_programme_breakdown = openspending_api.aggregate_by_refs( - [department_ref, geo_ref, programme_ref], - expenditure_results_filter_government_complete_breakdown, + expenditure_results_filter_government_programme_breakdown = ( + openspending_api.aggregate_by_refs( + [department_ref, geo_ref, programme_ref], + expenditure_results_filter_government_complete_breakdown, + ) ) # Used to iterate over unique departments and build department objects - expenditure_results_filter_government_department_breakdown = openspending_api.aggregate_by_refs( - [department_ref, geo_ref], - expenditure_results_filter_government_complete_breakdown, + expenditure_results_filter_government_department_breakdown = ( + openspending_api.aggregate_by_refs( + [department_ref, geo_ref], + expenditure_results_filter_government_complete_breakdown, + ) ) total_budget = 0 diff --git a/budgetportal/tasks.py b/budgetportal/tasks.py index d71f800ed..db07f41ba 100644 --- a/budgetportal/tasks.py +++ b/budgetportal/tasks.py @@ -82,7 +82,11 @@ def import_irm_snapshot(snapshot_id): except RowError as e: raise Exception( ("Error on row %d: %s\n\n" "Technical details: \n\n" "%s") - % (e.row_num, e, "\n".join([format_error(e) for e in e.row_result.errors]),) + % ( + e.row_num, + e, + "\n".join([format_error(e) for e in e.row_result.errors]), + ) ) except Exception as e: raise Exception("Error: %s\n\n%s" % (e, traceback.format_exc())) diff --git a/budgetportal/templates/base.html b/budgetportal/templates/base.html index a52435f4d..9155e1224 100644 --- a/budgetportal/templates/base.html +++ b/budgetportal/templates/base.html @@ -26,25 +26,36 @@ 'self' {{ CKAN_URL }} https://discussions.vulekamali.gov.za - https://api.appzi.io/api/v2/probe/Dzqo1 + https://api.appzi.io/api/ + https://clouderrorreporting.googleapis.com/v1beta1/projects/appzi-prod/ https://openspending.vulekamali.gov.za - https://openspending.org; + https://openspending.org + https://*.google-analytics.com + https://*.analytics.google.com + https://*.g.doubleclick.net + https://*.googletagmanager.com + https://*.google.com + https://*.google.co.za; font-src - https://fonts.gstatic.com/; + https://w.appzi.io + https://fonts.gstatic.com; style-src 'self' 'unsafe-inline' + https://www.googletagmanager.com https://fonts.googleapis.com https://tagmanager.google.com; script-src 'self' - https://www.googletagmanager.com + https://*.googletagmanager.com https://discussions.vulekamali.gov.za https://www.google-analytics.com - https://w.appzi.io/v1-v2-compat.js + https://ssl.google-analytics.com + https://w.appzi.io + https://app.appzi.io + https://api.appzi.io/api/tele/ https://www.youtube.com https://i.ytimg.com - https://app.appzi.io https://d3js.org 'nonce-{{ TAG_MANAGER_SCRIPT_NONCE }}' {% if debug %} @@ -72,12 +83,15 @@ http://s3-eu-west-1.amazonaws.com/ {{ CKAN_URL }} https://img.youtube.com - https://www.google-analytics.com + https://*.google-analytics.com + https://*.analytics.google.com + https://*.googletagmanager.com + https://*.g.doubleclick.net + https://*.google.com http://localhost http://minio:9000/ https://media.sandbox.vulekamali.gov.za https://media.vulekamali.gov.za - https://stats.g.doubleclick.net https://ssl.gstatic.com https://www.gstatic.com; " diff --git a/budgetportal/templates/budget-summary.html b/budgetportal/templates/budget-summary.html new file mode 100644 index 000000000..e58a46b32 --- /dev/null +++ b/budgetportal/templates/budget-summary.html @@ -0,0 +1,10 @@ +{% extends 'page-shell.html' %} +{% load define_action %} +{% block page_content %} + +
    +
    + {% include 'components/budget-summary/index.html' %} +
    +
    +{% endblock %} diff --git a/budgetportal/templates/department.html b/budgetportal/templates/department.html index 087b73e2e..871d9e024 100644 --- a/budgetportal/templates/department.html +++ b/budgetportal/templates/department.html @@ -3,338 +3,352 @@ {% load humanize %} {% block page_content %} -{% if government.slug == 'south-africa' %} - {% assign 'National' as department_location %} -{% else %} - {% assign government.name as department_location %} -{% endif %} - - -{% if government.slug == 'south-africa' %} - {% assign 'Estimates of National Expenditure' as source_type %} - {% assign 'Estimates of National Expenditure' as source_type_revenue %} - {% assign 'ENE' as source_type_revenue_short %} - {% assign "Adjusted Estimates of National Expenditure" as source_type_adjusted %} - {% assign "AENE" as source_type_adjusted_short %} - {% assign "/guides/estimates-of-national-expenditure" as guide %} -{% else %} - {% assign 'Estimates of Provincial Expenditure' as source_type %} - {% assign 'Estimates of Provincial Revenue and Expenditure' as source_type_revenue %} - {% assign 'EPRE' as source_type_revenue_short %} - {% assign "Adjusted Estimates of Provincial Revenue and Expenditure" as source_type_adjusted %} - {% assign "AEPRE" as source_type_adjusted_short %} - {% assign "/guides/estimates-of-provincial-expenditure" as guide %} -{% endif %} - - -{% if department_budget %} - {% assign department_budget.name as chapter_name %} - {% assign department_budget.document.url as pdf_link %} - {% assign department_budget.tables.url as excel_link %} -{% else %} - {% assign "" as chapter_name %} - {% assign "" as pdf_link %} - {% assign "" as excel_link %} -{% endif %} - - -{% if department_adjusted_budget %} - {% assign department_adjusted_budget.document.url as pdf_link_adjusted %} - {% assign department_adjusted_budget.tables.url as excel_link_adjusted %} -{% else %} - {% assign "" as pdf_link_adjusted %} - {% assign "" as excel_link_adjusted %} -{% endif %} - - -{% if treasury_datasets %} - {% for item in treasury_datasets %} - {% assign item.1.formats as modified_formats %} {# | sort: 'format' %} #} - {% assign modified_formats.0.url as pdf %} - {% assign modified_formats.1.url as excel %} - {% endfor %} -{% else %} - {% assign pdf_link as pdf %} - {% assign excel_link as excel %} -{% endif %} - - -{% include 'components/department-budgets/ArrowButtons/index.html' with fixed="true" link_1="#section-plan" link_2="#section-implement" link_3="#section-review" %} - -{% if sphere.slug == "national" and slug == "parliament" %} - {% assign "true" as parliament %} -{% else %} - {% assign "" as parliament %} -{% endif %} - -{% assign department_location|add:" "|add:name|add:" Department Budget "|add:selected_financial_year as subtitle %} - - -
    -
    - -

    - {{ name }} -

    - + {% if government.slug == 'south-africa' %} + {% assign 'National' as department_location %} + {% else %} + {% assign government.name as department_location %} + {% endif %} + + + {% if government.slug == 'south-africa' %} + {% assign 'Estimates of National Expenditure' as source_type %} + {% assign 'Estimates of National Expenditure' as source_type_revenue %} + {% assign 'ENE' as source_type_revenue_short %} + {% assign "Adjusted Estimates of National Expenditure" as source_type_adjusted %} + {% assign "AENE" as source_type_adjusted_short %} + {% assign "/learning-resources/guides/estimates-of-national-expenditure" as guide %} + {% else %} + {% assign 'Estimates of Provincial Expenditure' as source_type %} + {% assign 'Estimates of Provincial Revenue and Expenditure' as source_type_revenue %} + {% assign 'EPRE' as source_type_revenue_short %} + {% assign "Adjusted Estimates of Provincial Revenue and Expenditure" as source_type_adjusted %} + {% assign "AEPRE" as source_type_adjusted_short %} + {% assign "/learning-resources/guides/estimates-of-provincial-expenditure" as guide %} + {% endif %} + + + {% if department_budget %} + {% assign department_budget.name as chapter_name %} + {% assign department_budget.document.url as pdf_link %} + {% assign department_budget.tables.url as excel_link %} + {% else %} + {% assign "" as chapter_name %} + {% assign "" as pdf_link %} + {% assign "" as excel_link %} + {% endif %} + + + {% if department_adjusted_budget %} + {% assign department_adjusted_budget.document.url as pdf_link_adjusted %} + {% assign department_adjusted_budget.tables.url as excel_link_adjusted %} + {% else %} + {% assign "" as pdf_link_adjusted %} + {% assign "" as excel_link_adjusted %} + {% endif %} + + + {% if treasury_datasets %} + {% for item in treasury_datasets %} + {% assign item.1.formats as modified_formats %} {# | sort: 'format' %} #} + {% assign modified_formats.0.url as pdf %} + {% assign modified_formats.1.url as excel %} + {% endfor %} + {% else %} + {% assign pdf_link as pdf %} + {% assign excel_link as excel %} + {% endif %} + + + {% include 'components/department-budgets/ArrowButtons/index.html' with fixed="true" link_1="#section-plan" link_2="#section-implement" link_3="#section-review" %} + + {% if sphere.slug == "national" and slug == "parliament" %} + {% assign "true" as parliament %} + {% else %} + {% assign "" as parliament %} + {% endif %} + + {% assign department_location|add:" "|add:name|add:" Department Budget "|add:selected_financial_year as subtitle %} + + +
    +
    + +

    + {{ name }} +

    + {{ department_location }} Department Budget for {{ selected_financial_year }} - {% if website_url %} - + {% if website_url %} +     {{ website_url }} - {% endif %} - {% include 'components/department-budgets/IntroSection/index.html' with description=intro datasets=treasury_datasets location=department_location %} + {% endif %} + {% include 'components/department-budgets/IntroSection/index.html' with description=intro datasets=treasury_datasets location=department_location %} -
    - The Budget Cycle -
    +
    + The Budget Cycle +
    -
    - {% include 'components/department-budgets/ArrowButtons/index.html' with link_1="#section-plan" link_2="#section-implement" link_3="#section-review" %} -
    +
    + {% include 'components/department-budgets/ArrowButtons/index.html' with link_1="#section-plan" link_2="#section-implement" link_3="#section-review" %} +
    -
    -
    - Plan +
    +
    + Plan +
    -
    -
    -

    {{ selected_financial_year }} Budget

    -

    - The {{ source_type_revenue }} ({{ source_type_revenue_short }}) is a book published along with the - tabling of the budget for the new financial year. -

    +
    +

    {{ selected_financial_year }} Budget

    +

    + The {{ source_type_revenue }} ({{ source_type_revenue_short }}) is a book published along with the + tabling of the budget for the new financial year. +

    - {% assign "View the "|add:source_type_revenue_short|add:" chapter for "|add:chapter_name|add:" (PDF)" as text1 %} - {% assign "View tables in the "|add:source_type_revenue_short|add:" chapter (Excel)" as text2 %} + {% assign "View the "|add:source_type_revenue_short|add:" chapter for "|add:chapter_name|add:" (PDF)" as text1 %} + {% assign "View tables in the "|add:source_type_revenue_short|add:" chapter (Excel)" as text2 %} - {% if pdf_link or excel_link %} -
      - {% include 'components/LinksList/item.html' with text=text1 url=pdf_link type="download" %} + {% if pdf_link or excel_link %} +
        + {% include 'components/LinksList/item.html' with text=text1 url=pdf_link type="download" %} - {% include 'components/LinksList/item.html' with text=text2 url=excel_link type="download" %} -
      - {% endif %} -
    + {% include 'components/LinksList/item.html' with text=text2 url=excel_link type="download" %} + + {% endif %} +
    -
    - {% include 'scenes/department/ProgrammesSection/index.html' with source_type=source_type year=selected_financial_year pdf=pdf excel=excel guide=guide %} -
    +
    + {% include 'scenes/department/ProgrammesSection/index.html' with source_type=source_type year=selected_financial_year pdf=pdf excel=excel guide=guide %} +
    -
    - {% include 'scenes/department/EconClassPackedCircles/econ-class-packed-circles.html' with source_type=source_type year=selected_financial_year pdf=pdf excel=excel guide=guide %} -
    +
    + {% include 'scenes/department/EconClassPackedCircles/econ-class-packed-circles.html' with source_type=source_type year=selected_financial_year pdf=pdf excel=excel guide=guide %} +
    -
    - {% include 'scenes/department/ProgramEconSmallMultiples/programme-econ-small-muls.html' with source_type=source_type year=selected_financial_year pdf=pdf excel=excel guide=guide %} -
    +
    + {% include 'scenes/department/ProgramEconSmallMultiples/programme-econ-small-muls.html' with source_type=source_type year=selected_financial_year pdf=pdf excel=excel guide=guide %} +
    -

    {{ selected_financial_year }} Adjusted Budget

    -

    - The {{ source_type_adjusted }} ({{ source_type_adjusted_short }}) is a book published along with the tabling of the adjusted budget. -

    +

    {{ selected_financial_year }} Adjusted Budget

    +

    + The {{ source_type_adjusted }} ({{ source_type_adjusted_short }}) is a book published along with the + tabling of the adjusted budget. +

    - {% assign "View the "|add:source_type_adjusted_short|add:" chapter for "|add:chapter_name|add:" (PDF)." as text1 %} - {% assign "View tables in the "|add:source_type_adjusted_short|add:" chapter (Excel)." as text2 %} + {% assign "View the "|add:source_type_adjusted_short|add:" chapter for "|add:chapter_name|add:" (PDF)." as text1 %} + {% assign "View tables in the "|add:source_type_adjusted_short|add:" chapter (Excel)." as text2 %} - {% if pdf_link_adjusted or excel_link_adjusted %} -
      - {% include 'components/LinksList/item.html' with text=text1 url=pdf_link_adjusted type="download" %} - {% include 'components/LinksList/item.html' with text=text2 url=excel_link_adjusted type="download" %} -
    - {% else %} + {% if pdf_link_adjusted or excel_link_adjusted %} +
      + {% include 'components/LinksList/item.html' with text=text1 url=pdf_link_adjusted type="download" %} + {% include 'components/LinksList/item.html' with text=text2 url=excel_link_adjusted type="download" %} +
    + {% else %} - {% if excel_link_adjusted %} -
    -
    - {% include 'components/Icon/index.html' with type="info" %} - + {% if excel_link_adjusted %} +
    +
    + {% include 'components/Icon/index.html' with type="info" %} + Please note -
    -
    The {{ source_type_adjusted_short }} chapter for {{ chapter_name }} (PDF) is not available yet -
    -
    - {% elif pdf_link_adjusted %} -
    -
    - {% include 'components/Icon/index.html' with type="info" %} - +
    +
    The {{ source_type_adjusted_short }} chapter + for {{ chapter_name }} (PDF) is not available yet +
    +
    + {% elif pdf_link_adjusted %} +
    +
    + {% include 'components/Icon/index.html' with type="info" %} + Please note -
    -
    The tables in {{ source_type_revenue_short }} - chapter (Excel) is not available yet -
    -
    - {% else %} - {% assign "The "|add:source_type_revenue_short|add:" chapter for "|add:chapter_name|add:" (PDF) is not available yet" as info2 %} - {% assign "The tables in "|add:source_type_revenue_short|add:" chapter (Excel)" as info3 %} - -
    -
    - {% include 'components/Icon/index.html' with type="info" %} - +
    +
    The tables in {{ source_type_revenue_short }} + chapter (Excel) is not available yet +
    +
    + {% else %} + {% assign "The "|add:source_type_revenue_short|add:" chapter for "|add:chapter_name|add:" (PDF) is not available yet" as info2 %} + {% assign "The tables in "|add:source_type_revenue_short|add:" chapter (Excel)" as info3 %} + +
    +
    + {% include 'components/Icon/index.html' with type="info" %} + Please note -
    -
    The adjusted budget documents are not available yet on vulekamali.gov.za -
    -
    - {% endif %} - {% endif %} +
    +
    The adjusted budget documents are not + available yet on vulekamali.gov.za +
    +
    + {% endif %} + {% endif %} - {% if government.slug == 'south-africa' %} -
    - {% include 'scenes/department/AdjustedSection/index.html' with type="adjusted" items=adjusted_budget_summary source_type=source_type source_type_adjusted=source_type_adjusted year=selected_financial_year pdf=pdf excel=excel pdf_adjusted=pdf_link_adjusted excel_adjusted=excel_link_adjusted csv=adjusted_budget_summary.department_data_csv dataset=adjusted_budget_summary.dataset_detail_page parliament=parliament title="Programme budgets" subtitle=subtitle description="Activities of this department" %} -
    - {% endif %} + {% if government.slug == 'south-africa' %} +
    + {% include 'scenes/department/AdjustedSection/index.html' with type="adjusted" items=adjusted_budget_summary source_type=source_type source_type_adjusted=source_type_adjusted year=selected_financial_year pdf=pdf excel=excel pdf_adjusted=pdf_link_adjusted excel_adjusted=excel_link_adjusted csv=adjusted_budget_summary.department_data_csv dataset=adjusted_budget_summary.dataset_detail_page parliament=parliament title="Programme budgets" subtitle=subtitle description="Activities of this department" %} +
    + {% endif %} - {% if procurement_resource_links %} -
    - {% include 'scenes/department/ResourceLinks/resource-links.html' with resource_links=procurement_resource_links section_title="Procurement resources" more_link="/datasets/procurement-portals-and-resources" %} -
    - {% endif %} + {% if procurement_resource_links %} +
    + {% include 'scenes/department/ResourceLinks/resource-links.html' with resource_links=procurement_resource_links section_title="Procurement resources" more_link="/datasets/procurement-portals-and-resources" %} +
    + {% endif %} -
    -
    - Implement +
    +
    + Implement +
    -
    - {% if infra_enabled %} -
    -

    Department infrastructure projects

    - -

    Largest infrastructure projects by this department.

    - -
    - - - - - - - {% for project in projects %} - - - - - - {% empty %} - - - - {% endfor %} -
    Project nameEstimated completion date
    - - {{ project.name }} - - {{ project.estimated_completion_date|default:"Not available" }}
    No projects available for this department.
    -
    - -
    - - View all projects - -
    + {% if infra_enabled %} +
    +

    Department infrastructure projects

    + +

    Largest infrastructure projects by this department.

    + +
    + + + + + + + {% for project in projects %} + + + + + + {% empty %} + + + + {% endfor %} +
    Project nameEstimated completion date
    + + {{ project.name }} + + {{ project.estimated_completion_date|default:"Not available" }}
    No projects available for this department.
    +
    -
    - {% endif %} +
    + + View all projects + +
    - {% if in_year_monitoring_resource_links %} -
    - {% include 'scenes/department/ResourceLinks/resource-links.html' with resource_links=in_year_monitoring_resource_links section_title="In-year monitoring resources" %} -
    - {% endif %} +
    + {% endif %} - {% if performance_resource_links %} -
    - {% include 'scenes/department/ResourceLinks/resource-links.html' with resource_links=performance_resource_links section_title="Performance monitoring resources" %} -
    - {% endif %} + {% if in_year_monitoring_resource_links %} +
    + {% include 'scenes/department/ResourceLinks/resource-links.html' with resource_links=in_year_monitoring_resource_links section_title="In-year monitoring resources" %} +
    + {% endif %} + + {% if performance_resource_links %} +
    + {% include 'scenes/department/ResourceLinks/resource-links.html' with resource_links=performance_resource_links section_title="Performance monitoring resources" %} +
    + {% endif %} - {% if not infra_enabled and not in_year_monitoring_resource_links and not performance_resource_links %} -
    -
    - {% include 'components/Icon/index.html' with type="info" size="l" %} - + {% if EQPRS_DATA_ENABLED %} +
    + {% endif %} + + {% if not infra_enabled and not in_year_monitoring_resource_links and not performance_resource_links %} +
    +
    + {% include 'components/Icon/index.html' with type="info" size="l" %} + Please note -
    -
    Implementation data coming soon.
    -
    - {% endif %} - -
    -
    - Review -
    -
    +
    +
    Implementation data coming soon.
    +
    + {% endif %} - {% if expenditure_over_time %} -
    - {% include 'scenes/department/ExpenditureSection/index.html' with items=expenditure_over_time.expenditure cpi=global_values.cpi_dataset_url source_type=source_type year=selected_financial_year dataset=expenditure_over_time.dataset_detail_page pdf=pdf_link excel=excel csv=expenditure_over_time.department_data_csv guide=guide color="purple" title="Planned compared to historical expenditure" subtitle=subtitle description="Expenditure changes over time" %} +
    +
    + Review +
    - {% endif %} - {% with ""|add:department_location|add:" "|add:name|add:" Department" as text %} - - {% if budget_actual %} + {% if expenditure_over_time %}
    - {% include 'scenes/department/ExpenditurePhaseSection/index.html' with items=budget_actual.expenditure cpi=global_values.cpi_dataset_url source_type="Expenditure Time Series" year=selected_financial_year dataset=budget_actual.dataset_detail_page csv=budget_actual.department_data_csv color="purple" description="Budgeted and Actual Expenditure comparison" subtitle=review_subtitle notices=budget_actual.notices website_url=website_url %} + {% include 'scenes/department/ExpenditureSection/index.html' with items=expenditure_over_time.expenditure cpi=global_values.cpi_dataset_url source_type=source_type year=selected_financial_year dataset=expenditure_over_time.dataset_detail_page pdf=pdf_link excel=excel csv=expenditure_over_time.department_data_csv guide=guide color="purple" title="Planned compared to historical expenditure" subtitle=subtitle description="Expenditure changes over time" %}
    {% endif %} - {% if budget_actual_programmes %} -
    - {% include 'scenes/department/ExpenditureMultiplesSection/index.html' with items=budget_actual_programmes.programmes cpi=global_values.cpi_dataset_url source_type="Expenditure Time Series" year=selected_financial_year dataset=budget_actual_programmes.dataset_detail_page csv=budget_actual_programmes.department_data_csv color="purple" subtitle=review_subtitle description="Budgeted and Actual Expenditure comparison by Programme" notices=budget_actual_programmes.notices %} -
    - {% endif %} - {% endwith %} -
    -
    -
    - {% include 'components/department-budgets/ContributedData/index.html' with datasets=contributed_datasets %} -
    + {% with ""|add:department_location|add:" "|add:name|add:" Department" as text %} -
    - {% include 'components/universal/Participate/index.html' with title="Timelines for this department and ways to participate" description="National Treasury, departments and commitees are busy with different things depending on the time of year:" %} + {% if budget_actual %} +
    + {% include 'scenes/department/ExpenditurePhaseSection/index.html' with items=budget_actual.expenditure cpi=global_values.cpi_dataset_url source_type="Expenditure Time Series" year=selected_financial_year dataset=budget_actual.dataset_detail_page csv=budget_actual.department_data_csv color="purple" description="Budgeted and Actual Expenditure comparison" subtitle=review_subtitle notices=budget_actual.notices website_url=website_url %} +
    + {% endif %} - {% if comments_enabled %} -
    -

    Discuss this budget with others

    -
    -
    -
    + {% if budget_actual_programmes %} +
    + {% include 'scenes/department/ExpenditureMultiplesSection/index.html' with items=budget_actual_programmes.programmes cpi=global_values.cpi_dataset_url source_type="Expenditure Time Series" year=selected_financial_year dataset=budget_actual_programmes.dataset_detail_page csv=budget_actual_programmes.department_data_csv color="purple" subtitle=review_subtitle description="Budgeted and Actual Expenditure comparison by Programme" notices=budget_actual_programmes.notices %} +
    + {% endif %} + {% endwith %} +
    +
    +
    + {% include 'components/department-budgets/ContributedData/index.html' with datasets=contributed_datasets %} +
    + +
    + {% include 'components/universal/Participate/index.html' with title="Timelines for this department and ways to participate" description="National Treasury, departments and commitees are busy with different things depending on the time of year:" %} + + {% if comments_enabled %} +
    +

    Discuss this budget with others

    +
    +
    +
    +
    + {% endif %}
    - {% endif %}
    -
    - + +
    -
    {% endblock %} diff --git a/budgetportal/templates/homepage.html b/budgetportal/templates/homepage.html index 50d45978c..025cbed74 100644 --- a/budgetportal/templates/homepage.html +++ b/budgetportal/templates/homepage.html @@ -1,54 +1,43 @@ {% extends 'page-shell.html' %} +{% load json_script_escape %} {% load define_action %} {% load staticfiles %} {% block page_content %} -
    -
    - {% include 'connections/homepage-hero/index.html' %} -
    -
    -
    +
    +
    + {% include 'connections/homepage-hero/index.html' %} +
    -
    -
    +
    -
    -
    +
    - {% with events_list=events.upcoming %} + {% with events_list=events.upcoming %} - {% include 'scenes/homepage/VideoSection/index.html' %} + {% include 'scenes/homepage/VideoSection/index.html' %} - {% if events_list %} - {% include 'scenes/homepage/EventSection/index.html' %} - {% endif %} + {% if events_list %} + {% include 'scenes/homepage/EventSection/index.html' %} + {% endif %} - {% include 'scenes/homepage/AboutSection/index.html' %} + {% include 'scenes/homepage/AboutSection/index.html' %} - {% endwith %} -
    + {% endwith %} +
    + {% endblock %} diff --git a/budgetportal/tests/helpers.py b/budgetportal/tests/helpers.py index 17f9659fd..06954f9b0 100644 --- a/budgetportal/tests/helpers.py +++ b/budgetportal/tests/helpers.py @@ -4,7 +4,7 @@ import warnings from datetime import datetime -from django.contrib.staticfiles.testing import LiveServerTestCase +from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.core.management import call_command from django.db import connections from django.test import TestCase @@ -13,6 +13,7 @@ from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait +import socket class WagtailHackMixin: @@ -45,29 +46,40 @@ def _fixture_teardown(self): ) -class BaseSeleniumTestCase(WagtailHackMixin, LiveServerTestCase): +class BaseSeleniumTestCase(WagtailHackMixin, StaticLiveServerTestCase): """ Base class for Selenium tests. This saves a screenshot to the current directory on test failure. + + Much learned from https://github.com/marcgibbons/django-selenium-docker """ - def setUp(self): - super(BaseSeleniumTestCase, self).setUp() + host = "0.0.0.0" # Bind to 0.0.0.0 to allow external access + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.host = socket.gethostbyname(socket.gethostname()) chrome_options = webdriver.ChromeOptions() - chrome_options.add_argument("headless") chrome_options.add_argument("--no-sandbox") - d = DesiredCapabilities.CHROME + chrome_options.add_argument("disable-dev-shm-usage") + d = chrome_options.to_capabilities() d["loggingPrefs"] = {"browser": "ALL"} - self.selenium = webdriver.Chrome( - chrome_options=chrome_options, desired_capabilities=d + cls.selenium = webdriver.Remote( + command_executor="http://selenium:4444/wd/hub", desired_capabilities=d ) - self.selenium.implicitly_wait(10) - self.wait = WebDriverWait(self.selenium, 5) + cls.selenium.implicitly_wait(10) + cls.wait = WebDriverWait(cls.selenium, 5) + @classmethod + def tearDownClass(cls): + cls.selenium.quit() + super().tearDownClass() + + def setUp(self): self.addCleanup(self.log_failure_details) - self.addCleanup(self.selenium.quit) def log_failure_details(self): # https://stackoverflow.com/questions/14991244/how-do-i-capture-a-screenshot-if-my-nosetests-fail @@ -87,7 +99,7 @@ def wait_until_text_in(self, selector, text): ) -class WagtailHackLiveServerTestCase(WagtailHackMixin, LiveServerTestCase): +class WagtailHackLiveServerTestCase(WagtailHackMixin, StaticLiveServerTestCase): pass diff --git a/budgetportal/tests/mock_data.py b/budgetportal/tests/mock_data.py index b5987d12a..8ae43baea 100644 --- a/budgetportal/tests/mock_data.py +++ b/budgetportal/tests/mock_data.py @@ -1,160 +1,160 @@ from decimal import Decimal CPI_2019_20 = { - u"1996": { - u"CPI": u"0.081341339073298", - u"Year": u"1996/97", - "financial_year_start": u"1996", + "1996": { + "CPI": "0.081341339073298", + "Year": "1996/97", + "financial_year_start": "1996", "index": Decimal("29.15351953553183659377285178"), }, - u"1997": { - u"CPI": u"0.075536930330016", - u"Year": u"1997/98", - "financial_year_start": u"1997", + "1997": { + "CPI": "0.075536930330016", + "Year": "1997/98", + "financial_year_start": "1997", "index": Decimal("31.35568690956206535036618471"), }, - u"1998": { - u"CPI": u"0.076173777518021", - u"Year": u"1998/99", - "financial_year_start": u"1998", + "1998": { + "CPI": "0.076173777518021", + "Year": "1998/99", + "financial_year_start": "1998", "index": Decimal("33.74416802813576957260265599"), }, - u"1999": { - u"CPI": u"0.037925416364953", - u"Year": u"1999/00", - "financial_year_start": u"1999", + "1999": { + "CPI": "0.037925416364953", + "Year": "1999/00", + "financial_year_start": "1999", "index": Decimal("35.02392965049175369249598743"), }, - u"2000": { - u"CPI": u"0.064969041597628", - u"Year": u"2000/01", - "financial_year_start": u"2000", + "2000": { + "CPI": "0.064969041597628", + "Year": "2000/01", + "financial_year_start": "2000", "index": Decimal("37.29940079286694913746974641"), }, - u"2001": { - u"CPI": u"0.052898788077301", - u"Year": u"2001/02", - "financial_year_start": u"2001", + "2001": { + "CPI": "0.052898788077301", + "Year": "2001/02", + "financial_year_start": "2001", "index": Decimal("39.27249389081913077278894942"), }, - u"2002": { - u"CPI": u"0.103981956758438", - u"Year": u"2002/03", - "financial_year_start": u"2002", + "2002": { + "CPI": "0.103981956758438", + "Year": "2002/03", + "financial_year_start": "2002", "index": Decimal("43.35612465237030615432841586"), }, - u"2003": { - u"CPI": u"0.033392039450511", - u"Year": u"2003/04", - "financial_year_start": u"2003", + "2003": { + "CPI": "0.033392039450511", + "Year": "2003/04", + "financial_year_start": "2003", "index": Decimal("44.80387407718352793313968934"), }, - u"2004": { - u"CPI": u"0.019905924057536", - u"Year": u"2004/05", - "financial_year_start": u"2004", + "2004": { + "CPI": "0.019905924057536", + "Year": "2004/05", + "financial_year_start": "2004", "index": Decimal("45.69573659204734907313347654"), }, - u"2005": { - u"CPI": u"0.036227524898069", - u"Year": u"2005/06", - "financial_year_start": u"2005", + "2005": { + "CPI": "0.036227524898069", + "Year": "2005/06", + "financial_year_start": "2005", "index": Decimal("47.35118002717134708630016805"), }, - u"2006": { - u"CPI": u"0.051860930142553", - u"Year": u"2006/07", - "financial_year_start": u"2006", + "2006": { + "CPI": "0.051860930142553", + "Year": "2006/07", + "financial_year_start": "2006", "index": Decimal("49.80685626672793118196184207"), }, - u"2007": { - u"CPI": u"0.081345434475992", - u"Year": u"2007/08", - "financial_year_start": u"2007", + "2007": { + "CPI": "0.081345434475992", + "Year": "2007/08", + "financial_year_start": "2007", "index": Decimal("53.85841662962819963199302250"), }, - u"2008": { - u"CPI": u"0.098760881277115", - u"Year": u"2008/09", - "financial_year_start": u"2008", + "2008": { + "CPI": "0.098760881277115", + "Year": "2008/09", + "financial_year_start": "2008", "index": Decimal("59.17752132016030645441194773"), }, - u"2009": { - u"CPI": u"0.064516129032258", - u"Year": u"2009/10", - "financial_year_start": u"2009", + "2009": { + "CPI": "0.064516129032258", + "Year": "2009/10", + "financial_year_start": "2009", "index": Decimal("62.99542592146096756905005273"), }, - u"2010": { - u"CPI": u"0.038181818181818", - u"Year": u"2010/11", - "financial_year_start": u"2010", + "2010": { + "CPI": "0.038181818181818", + "Year": "2010/11", + "financial_year_start": "2010", "index": Decimal("65.40070582028037487706361448"), }, - u"2011": { - u"CPI": u"0.055458260361938", - u"Year": u"2011/12", - "financial_year_start": u"2011", + "2011": { + "CPI": "0.055458260361938", + "Year": "2011/12", + "financial_year_start": "2011", "index": Decimal("69.02771519151599784307391477"), }, - u"2012": { - u"CPI": u"0.055420353982301", - u"Year": u"2012/13", - "financial_year_start": u"2012", + "2012": { + "CPI": "0.055420353982301", + "Year": "2012/13", + "financial_year_start": "2012", "index": Decimal("72.85325560201927070902566593"), }, - u"2013": { - u"CPI": u"0.058170003144324", - u"Year": u"2013/14", - "financial_year_start": u"2013", + "2013": { + "CPI": "0.058170003144324", + "Year": "2013/14", + "financial_year_start": "2013", "index": Decimal("77.09112970946297175373333027"), }, - u"2014": { - u"CPI": u"0.056259904912837", - u"Year": u"2014/15", - "financial_year_start": u"2014", + "2014": { + "CPI": "0.056259904912837", + "Year": "2014/15", + "financial_year_start": "2014", "index": Decimal("81.42826933654054200675012982"), }, - u"2015": { - u"CPI": u"0.051669167291823", - u"Year": u"2015/16", - "financial_year_start": u"2015", + "2015": { + "CPI": "0.051669167291823", + "Year": "2015/16", + "financial_year_start": "2015", "index": Decimal("85.63560020717387631656468800"), }, - u"2016": { - u"CPI": u"0.062951404369147", - u"Year": u"2016/17", - "financial_year_start": u"2016", + "2016": { + "CPI": "0.062951404369147", + "Year": "2016/17", + "financial_year_start": "2016", "index": Decimal("91.02648150421028761249239849"), }, - u"2017": { - u"CPI": u"0.047143695998659", - u"Year": u"2017/18", - "financial_year_start": u"2017", + "2017": { + "CPI": "0.047143695998659", + "Year": "2017/18", + "financial_year_start": "2017", "index": Decimal("95.31780627607233362007115993"), }, - u"2018": { - u"CPI": u"0.049121920728709", - u"Year": u"2018/19", - "financial_year_start": u"2018", + "2018": { + "CPI": "0.049121920728709", + "Year": "2018/19", + "financial_year_start": "2018", "index": 100, }, - u"2019": { - u"CPI": u"0.051581145336389", - u"Year": u"2019/20", - "financial_year_start": u"2019", + "2019": { + "CPI": "0.051581145336389", + "Year": "2019/20", + "financial_year_start": "2019", "index": Decimal("105.158114533638900"), }, - u"2020": { - u"CPI": u"0.054786768395168", - u"Year": u"2020/21", - "financial_year_start": u"2020", + "2020": { + "CPI": "0.054786768395168", + "Year": "2020/21", + "financial_year_start": "2020", "index": Decimal("110.9193877994659244141042168"), }, - u"2021": { - u"CPI": u"0.054285094208856", - u"Year": u"2021/22", - "financial_year_start": u"2021", + "2021": { + "CPI": "0.054285094208856", + "Year": "2021/22", + "financial_year_start": "2021", "index": Decimal("116.9406572157485649289660142"), }, } diff --git a/budgetportal/tests/test_bulk_upload.py b/budgetportal/tests/test_bulk_upload.py index 9f06e3053..5083718d9 100644 --- a/budgetportal/tests/test_bulk_upload.py +++ b/budgetportal/tests/test_bulk_upload.py @@ -44,7 +44,7 @@ def setUp(self): self.CKANMockClass.action.group_show.side_effect = NotFound() self.addCleanup(self.ckan_patch.stop) - self.ckan_patch2 = patch("budgetportal.models.ckan") + self.ckan_patch2 = patch("budgetportal.models.government.ckan") self.CKANMockClass2 = self.ckan_patch2.start() self.CKANMockClass2.action.package_search.return_value = {"results": []} self.CKANMockClass2.action.package_show.side_effect = NotFound() diff --git a/budgetportal/tests/test_datasets.py b/budgetportal/tests/test_datasets.py index f97303714..0534e0ccb 100644 --- a/budgetportal/tests/test_datasets.py +++ b/budgetportal/tests/test_datasets.py @@ -29,16 +29,16 @@ def setUp(self): def test_get_latest_cpi_resource(self): results = [ { - "financial_year": [u"2020-21"], + "financial_year": ["2020-21"], "resources": [ - {"format": u"CSV", "id": u"0c173948-9674-4ca9-aec6-f144bde5cc1e"} + {"format": "CSV", "id": "0c173948-9674-4ca9-aec6-f144bde5cc1e"} ], }, { - "financial_year": [u"2018-19"], + "financial_year": ["2018-19"], "resources": [ - {"format": u"XLSX", "id": u"d1f96183-83e5-4ff1-87f5-c58e279b6f63"}, - {"format": u"CSV", "id": u"5b315ff0-55e9-4ba8-b88c-2d70093bfe9d"}, + {"format": "XLSX", "id": "d1f96183-83e5-4ff1-87f5-c58e279b6f63"}, + {"format": "CSV", "id": "5b315ff0-55e9-4ba8-b88c-2d70093bfe9d"}, ], }, ] @@ -52,9 +52,9 @@ def test_get_latest_cpi_resource(self): def test_get_latest_cpi_resource_multiple_financial_year_values(self): results = [ { - "financial_year": [u"2019-20", "2020-21"], + "financial_year": ["2019-20", "2020-21"], "resources": [ - {"format": u"CSV", "id": u"0c173948-9674-4ca9-aec6-f144bde5cc1e"} + {"format": "CSV", "id": "0c173948-9674-4ca9-aec6-f144bde5cc1e"} ], } ] diff --git a/budgetportal/tests/test_department.py b/budgetportal/tests/test_department.py index c2ef22f9c..4ee01b862 100644 --- a/budgetportal/tests/test_department.py +++ b/budgetportal/tests/test_department.py @@ -107,7 +107,7 @@ def setUp(self): self.department._get_budget_virements = Mock(return_value=Mock()) self.department._get_budget_special_appropriations = Mock(return_value=Mock()) self.department._get_budget_direct_charges = Mock(return_value=Mock()) - models.csv_url = Mock(return_value=Mock()) + models.government.csv_url = Mock(return_value=Mock()) def test_no_adjustment(self): self.department._get_total_budget_adjustment = Mock(return_value=(123, 0)) @@ -168,7 +168,7 @@ def setUp(self): return_value=self.mock_openspending_api ) - @mock.patch("budgetportal.models.get_expenditure_time_series_dataset") + @mock.patch("budgetportal.models.government.get_expenditure_time_series_dataset") def test_no_cells_null_response(self, mock_get_dataset): self.mock_openspending_api.filter_dept = Mock(return_value={"cells": []}) mock_get_dataset.return_value = self.mock_dataset @@ -176,16 +176,20 @@ def test_no_cells_null_response(self, mock_get_dataset): result = self.department.get_expenditure_time_series_by_programme() self.assertEqual(result, None) - @mock.patch("budgetportal.models.get_expenditure_time_series_dataset") - @mock.patch("budgetportal.models.get_cpi", return_value=mock_data.CPI_2019_20) + @mock.patch("budgetportal.models.government.get_expenditure_time_series_dataset") + @mock.patch( + "budgetportal.models.government.get_cpi", return_value=mock_data.CPI_2019_20 + ) def test_complete_data_no_notices(self, mock_get_cpi, mock_get_dataset): mock_get_dataset.return_value = self.mock_dataset result = self.department.get_expenditure_time_series_by_programme() self.assertEqual(result["notices"], []) - @mock.patch("budgetportal.models.get_expenditure_time_series_dataset") - @mock.patch("budgetportal.models.get_cpi", return_value=mock_data.CPI_2019_20) + @mock.patch("budgetportal.models.government.get_expenditure_time_series_dataset") + @mock.patch( + "budgetportal.models.government.get_cpi", return_value=mock_data.CPI_2019_20 + ) def test_missing_data_prog_did_not_exist(self, mock_get_cpi, mock_get_dataset): """ Here we feed an incomplete set of cells and expect it to tell us that @@ -258,7 +262,7 @@ def setUp(self): dataset_patch.start() self.addCleanup(dataset_patch.stop) - @mock.patch("budgetportal.models.get_expenditure_time_series_dataset") + @mock.patch("budgetportal.models.government.get_expenditure_time_series_dataset") def test_no_cells_null_response(self, mock_get_dataset): self.mock_openspending_api.aggregate_by_refs = Mock(return_value=[]) mock_get_dataset.return_value = self.mock_dataset @@ -266,8 +270,10 @@ def test_no_cells_null_response(self, mock_get_dataset): result = self.department.get_expenditure_time_series_summary() self.assertEqual(result, None) - @mock.patch("budgetportal.models.get_expenditure_time_series_dataset") - @mock.patch("budgetportal.models.get_cpi", return_value=mock_data.CPI_2019_20) + @mock.patch("budgetportal.models.government.get_expenditure_time_series_dataset") + @mock.patch( + "budgetportal.models.government.get_cpi", return_value=mock_data.CPI_2019_20 + ) def test_complete_data_no_notices(self, mock_get_cpi, mock_get_dataset): mock_get_dataset.return_value = self.mock_dataset self.mock_openspending_api.aggregate_by_refs = Mock( @@ -277,8 +283,10 @@ def test_complete_data_no_notices(self, mock_get_cpi, mock_get_dataset): result = self.department.get_expenditure_time_series_summary() self.assertEqual(result["notices"], []) - @mock.patch("budgetportal.models.get_expenditure_time_series_dataset") - @mock.patch("budgetportal.models.get_cpi", return_value=mock_data.CPI_2019_20) + @mock.patch("budgetportal.models.government.get_expenditure_time_series_dataset") + @mock.patch( + "budgetportal.models.government.get_cpi", return_value=mock_data.CPI_2019_20 + ) def test_missing_data_not_published(self, mock_get_cpi, mock_get_dataset): """ Here we feed an incomplete set of cells and expect it to tell us that @@ -301,8 +309,10 @@ def test_missing_data_not_published(self, mock_get_cpi, mock_get_dataset): ], ) - @mock.patch("budgetportal.models.get_expenditure_time_series_dataset") - @mock.patch("budgetportal.models.get_cpi", return_value=mock_data.CPI_2019_20) + @mock.patch("budgetportal.models.government.get_expenditure_time_series_dataset") + @mock.patch( + "budgetportal.models.government.get_cpi", return_value=mock_data.CPI_2019_20 + ) def test_missing_data_dept_did_not_exist(self, mock_get_cpi, mock_get_dataset): """ Here we feed an incomplete set of cells and expect it to tell us @@ -325,7 +335,7 @@ def test_missing_data_dept_did_not_exist(self, mock_get_cpi, mock_get_dataset): class DepartmentWebsiteUrlTestCase(TestCase): - """ Integration test to verify that website urls are retrieved and output correctly """ + """Integration test to verify that website urls are retrieved and output correctly""" def setUp(self): year_old = FinancialYear.objects.create(slug="2017-18") @@ -356,8 +366,8 @@ def setUp(self): ) def test_website_url_always_returns_latest_department_year(self): - """ Make sure that any given department for any given year always returns the website url of the - latest department instance in that sphere, where it is not null """ + """Make sure that any given department for any given year always returns the website url of the + latest department instance in that sphere, where it is not null""" self.assertEqual( self.department.get_latest_website_url(), "https://governmentwebsite.co.za" ) @@ -368,7 +378,7 @@ def test_website_url_always_returns_latest_department_year(self): class NationalTreemapExpenditureByDepartmentTestCase(TestCase): - """ Unit tests for the treemap expenditure by department function. """ + """Unit tests for the treemap expenditure by department function.""" def setUp(self): self.mock_data = TREEMAP_MOCK_DATA @@ -422,7 +432,7 @@ def setUp(self): "budgetportal.models.Department.get_all_budget_totals_by_year_and_phase", return_value=mock.MagicMock(), ) - @mock.patch("budgetportal.models.get_expenditure_time_series_dataset") + @mock.patch("budgetportal.models.government.get_expenditure_time_series_dataset") def test_no_cells_null_response(self, mock_get_dataset, total_budgets_mock): self.mock_openspending_api.aggregate_by_refs = Mock(return_value=[]) mock_get_dataset.return_value = self.mock_dataset @@ -436,7 +446,7 @@ def test_no_cells_null_response(self, mock_get_dataset, total_budgets_mock): "budgetportal.models.Department.get_all_budget_totals_by_year_and_phase", return_value=mock.MagicMock(), ) - @mock.patch("budgetportal.models.get_expenditure_time_series_dataset") + @mock.patch("budgetportal.models.government.get_expenditure_time_series_dataset") def test_complete_data(self, mock_get_dataset, total_budgets_mock): self.mock_openspending_api.aggregate_by_refs = Mock( return_value=self.mock_data["complete"] diff --git a/budgetportal/tests/test_department_page.py b/budgetportal/tests/test_department_page.py index 2cc373605..1a27a1d1c 100644 --- a/budgetportal/tests/test_department_page.py +++ b/budgetportal/tests/test_department_page.py @@ -14,6 +14,8 @@ class DepartmentPageTestCase(TestCase): + dataset_year_note = "Budget (Main appropriation) 2018-19" + def setUp(self): self.mock_openspending_api = MagicMock() self.mock_openspending_api.get_adjustment_kind_ref.return_value = ( @@ -49,7 +51,7 @@ def setUp(self): government=south_africa, name="The Presidency", vote_number=1, intro="" ) - models_ckan_patch = patch("budgetportal.models.ckan") + models_ckan_patch = patch("budgetportal.models.government.ckan") ModelsCKANMockClass = models_ckan_patch.start() ModelsCKANMockClass.action.package_search.return_value = {"results": []} self.addCleanup(models_ckan_patch.stop) @@ -72,12 +74,8 @@ def test_no_resource_links(self): PerformanceResourceLink.objects.all().delete() InYearMonitoringResourceLink.objects.all().delete() - with patch( - "budgetportal.views.DepartmentSubprogrammes.get_openspending_api", - MagicMock(return_value=self.mock_openspending_api), - ): - c = Client() - response = c.get("/2018-19/national/departments/the-presidency/") + c = Client() + response = c.get("/2018-19/national/departments/the-presidency/") self.assertContains( response, "The Presidency budget data for the 2018-19 financial year" @@ -98,12 +96,8 @@ def test_basic_links(self): title="an in-year link", url="a.com", description="abc" ) - with patch( - "budgetportal.views.DepartmentSubprogrammes.get_openspending_api", - MagicMock(return_value=self.mock_openspending_api), - ): - c = Client() - response = c.get("/2018-19/national/departments/the-presidency/") + c = Client() + response = c.get("/2018-19/national/departments/the-presidency/") self.assertContains( response, "The Presidency budget data for the 2018-19 financial year" @@ -136,12 +130,8 @@ def test_sphere_specific_links(self): sphere_slug="all", ) - with patch( - "budgetportal.views.DepartmentSubprogrammes.get_openspending_api", - MagicMock(return_value=self.mock_openspending_api), - ): - c = Client() - response = c.get("/2018-19/national/departments/the-presidency/") + c = Client() + response = c.get("/2018-19/national/departments/the-presidency/") self.assertContains( response, "The Presidency budget data for the 2018-19 financial year" @@ -149,3 +139,27 @@ def test_sphere_specific_links(self): self.assertContains(response, "a national link") self.assertNotContains(response, "a provincial link") self.assertContains(response, "an all-sphere link") + + def test_missing_budget_dataset(self): + c = Client() + response = c.get("/2018-19/national/departments/the-presidency/") + + self.assertContains(response, "Data not available") + self.assertNotContains(response, self.dataset_year_note) + + def test_budget_dataset_available(self): + # mock get dataset to return mock dataset which includes opn_spending _api mocks + mock_dataset = MagicMock() + mock_dataset.get_openspending_api.return_value = self.mock_openspending_api + with patch( + "budgetportal.views.DepartmentSubprogrammes.get_dataset", + MagicMock(return_value=mock_dataset), + ): + c = Client() + response = c.get("/2018-19/national/departments/the-presidency/") + + self.assertContains(response, self.dataset_year_note) + self.assertNotContains(response, "Data not available") + self.assertContains( + response, "/2018-19/national/departments/the-presidency/viz/subprog-treemap" + ) diff --git a/budgetportal/tests/test_featured_infra_projects.py b/budgetportal/tests/test_featured_infra_projects.py index 3f4fbc72b..405191638 100644 --- a/budgetportal/tests/test_featured_infra_projects.py +++ b/budgetportal/tests/test_featured_infra_projects.py @@ -5,7 +5,7 @@ class ProjectedExpenditureTestCase(TestCase): - """ Unit tests for get_projected_expenditure function """ + """Unit tests for get_projected_expenditure function""" fixtures = ["test-infrastructure-pages-detail"] @@ -18,7 +18,7 @@ def test_success(self): class CoordinatesTestCase(TestCase): - """ Unit tests for parsing coordinates """ + """Unit tests for parsing coordinates""" def test_success_simple_format(self): raw_coord_string = "-26.378582,27.654933" @@ -54,7 +54,7 @@ def test_empty_response_for_invalid_value(self): class ExpenditureTestCase(TestCase): - """ Unit tests for expenditure functions """ + """Unit tests for expenditure functions""" fixtures = ["test-infrastructure-pages-detail"] @@ -152,7 +152,7 @@ def setUp(self): @mock.patch("requests.get", return_value=empty_ckan_response) def test_success_empty_projects(self, mock_get): - """ Test that it exists and that the correct years are linked. """ + """Test that it exists and that the correct years are linked.""" InfrastructureProjectPart.objects.all().delete() c = Client() response = c.get("/json/infrastructure-projects.json") @@ -170,7 +170,7 @@ def test_success_empty_projects(self, mock_get): @mock.patch("requests.get", side_effect=mocked_requests_get) def test_success_with_projects(self, mock_get): - """ Test that it exists and that the correct years are linked. """ + """Test that it exists and that the correct years are linked.""" c = Client() response = c.get("/json/infrastructure-projects.json") content = response.json() @@ -242,7 +242,7 @@ def setUp(self): @mock.patch("requests.get", side_effect=mocked_requests_get) def test_success_with_projects(self, mock_get): - """ Test that it exists and that the correct years are linked. """ + """Test that it exists and that the correct years are linked.""" c = Client() response = c.get( "/json/infrastructure-projects/{}.json".format(self.project.project_slug) diff --git a/budgetportal/tests/test_featured_projects.py b/budgetportal/tests/test_featured_projects.py index 4b50071ba..564c8441c 100644 --- a/budgetportal/tests/test_featured_projects.py +++ b/budgetportal/tests/test_featured_projects.py @@ -48,7 +48,7 @@ def test_ppp_project_detail_page_fields(self): project_title = selenium.find_element_by_css_selector("#project-title").text budget = selenium.find_element_by_css_selector("#total-budget").text line = selenium.find_element_by_css_selector(".recharts-line") - self.assertEqual(partnership_type, u"Fake Partnership") - self.assertEqual(project_title, u"School Infrastructure Backlogs Grant") - self.assertEqual(budget, u"R4 billion") + self.assertEqual(partnership_type, "Fake Partnership") + self.assertEqual(project_title, "School Infrastructure Backlogs Grant") + self.assertEqual(budget, "R4 billion") self.assertEqual(line.is_displayed(), True) diff --git a/budgetportal/tests/test_infra_project_chart.py b/budgetportal/tests/test_infra_project_chart.py index 47012902b..51b98382e 100644 --- a/budgetportal/tests/test_infra_project_chart.py +++ b/budgetportal/tests/test_infra_project_chart.py @@ -30,7 +30,7 @@ def setUp(self): def test_dates_are_end_of_quarters(self): """Test that all dates are end day of a quarter""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 4) # Q1->06-30, Q2->09-30, Q3->12-31, Q4->03-31 @@ -42,7 +42,7 @@ def test_dates_are_end_of_quarters(self): def test_dates_match_with_quarters(self): """Test that dates and quarter_labels match""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 4) # Q1->06-30, Q2->09-30, Q3->12-31, Q4->03-31 @@ -72,7 +72,7 @@ def setUp(self): def test_estimated_total_project_cost_is_null(self): """Test that total project cost for Q1 (which created by Q2 snapshot) is Null""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 2) # Check Q1 values @@ -81,7 +81,7 @@ def test_estimated_total_project_cost_is_null(self): def test_estimated_total_project_cost_assigned_correctly(self): """Test that total project cost for Q2 is 100""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 2) # Check Q2 values @@ -102,7 +102,7 @@ def setUp(self): def test_status_is_null(self): """Test that status for Q1 (which created by Q2 snapshot) is Null""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 2) # Check Q1 values @@ -111,7 +111,7 @@ def test_status_is_null(self): def test_status_assigned_correctly(self): """Test that status for Q2 is Tender""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 2) # Check Q2 values @@ -135,7 +135,7 @@ def setUp(self): def test_q1_updated_after_q2_snapshot_inserted(self): """Test that Q1 values are updated correctly when Q2 snapshot is added""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 1) # Check Q1 values @@ -156,7 +156,7 @@ def test_q1_updated_after_q2_snapshot_inserted(self): snapshots_data = time_series_data( [self.project_snapshot, self.project_snapshot_2] ) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 2) # Check Q1 values @@ -192,7 +192,7 @@ def test_q1_q2_updated_after_q3_snapshot_inserted(self): snapshots_data = time_series_data( [self.project_snapshot, self.project_snapshot_2] ) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 2) # Check Q1 values @@ -218,7 +218,7 @@ def test_q1_q2_updated_after_q3_snapshot_inserted(self): snapshots_data = time_series_data( [self.project_snapshot, self.project_snapshot_2, self.project_snapshot_3] ) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 3) # Check Q1 values @@ -249,7 +249,7 @@ def setUp(self): def test_total_spends_are_correct(self): """Test that total spends are none because of actual_expenditure_q1""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 3) # Check total_spent_to_date values for Q1, Q2 and Q3 @@ -277,7 +277,7 @@ def test_correct_value_used_for_previous_total(self): Q2 snapshot's expenditure_from_previous_years_total updates total_spent of Q1 chart item. """ snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 1) self.assertEqual(snapshots_data[0]["total_spent_to_date"], 110) @@ -295,7 +295,7 @@ def test_correct_value_used_for_previous_total(self): snapshots_data = time_series_data( [self.project_snapshot, self.project_snapshot_2] ) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 2) # Check total_spent_to_date values for Q1 and Q2 @@ -322,7 +322,7 @@ def test_total_spends_are_none(self): """Test that Q1 and Q2 total_spent values when expenditure_ from_previous_years_total is empty.""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 2) # Check total_spent_to_date values for Q1 and Q2 @@ -344,7 +344,7 @@ def setUp(self): def test_two_snapshots_emitted(self): """Test that if the first snapshot is Q2, items are created for Q1 and Q2 but nothing later than Q2.""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 2) # Check Q1 values @@ -380,7 +380,7 @@ def test_six_snapshots_emitted(self): snapshots_data = time_series_data( [self.project_snapshot, self.project_snapshot_2] ) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 6) # Check 2018's Q1 and Q2 in a row @@ -435,7 +435,7 @@ def test_total_spent_to_dates_are_correct(self): snapshots_data = time_series_data( [self.project_snapshot, self.project_snapshot_2] ) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 5) # Check that 2019 Q1 Snapshot's total_spent_to_date is correct @@ -457,7 +457,7 @@ def setUp(self): def test_label_is_assigned_to_q1(self): """Test that financial year label is correctly assigned for Q1""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 2) # Check Q1 values @@ -467,7 +467,7 @@ def test_label_is_assigned_to_q1(self): def test_label_is_empty_for_q2(self): """Test that financial year label is empty for quarters except Q1""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 2) # Check Q2 values @@ -489,7 +489,7 @@ def setUp(self): def test_label_is_correct(self): """Test that quarter labels start with 'END Q' and ends with (1,2,3,4)""" snapshots_data = time_series_data([self.project_snapshot]) - snapshots_data = snapshots_data[u"snapshots"] + snapshots_data = snapshots_data["snapshots"] self.assertEqual(len(snapshots_data), 4) # Check quarter label texts for all quarters @@ -519,7 +519,7 @@ def setUp(self): def test_events_assigned_correctly(self): """Test that all dates are assigned correctly""" events_data = time_series_data([self.project_snapshot]) - events_data = events_data[u"events"] + events_data = events_data["events"] self.assertEqual(len(events_data), 5) # Project Start Date @@ -541,7 +541,7 @@ def test_events_when_latest_snapshot_has_empty_dates(self): irm_snapshot=irm_snapshot_2, project=self.project, start_date="2029-09-30" ) events_data = time_series_data([self.project_snapshot, self.project_snapshot_2]) - events_data = events_data[u"events"] + events_data = events_data["events"] self.assertEqual(len(events_data), 1) # Project Start Date diff --git a/budgetportal/tests/test_infra_projects.py b/budgetportal/tests/test_infra_projects.py index 3e1bc6623..9e71e9021 100644 --- a/budgetportal/tests/test_infra_projects.py +++ b/budgetportal/tests/test_infra_projects.py @@ -237,11 +237,11 @@ def test_project_detail_page_fields(self): selenium = self.selenium self.wait.until( EC.text_to_be_present_in_element( - (By.CSS_SELECTOR, ".page-heading"), u"BLUE JUNIOR SECONDARY SCHOOL" + (By.CSS_SELECTOR, ".page-heading"), "BLUE JUNIOR SECONDARY SCHOOL" ) ) title = selenium.find_element_by_css_selector(".page-heading").text - self.assertEqual(title, u"BLUE JUNIOR SECONDARY SCHOOL") + self.assertEqual(title, "BLUE JUNIOR SECONDARY SCHOOL") source = selenium.find_element_by_css_selector( ".primary-funding-source-field" @@ -256,9 +256,9 @@ def test_project_detail_page_fields(self): ".header__download" ).get_attribute("href") - self.assertEqual(source, u"Education Infrastructure Grant") - self.assertEqual(investment, u"Upgrading and Additions") - self.assertEqual(funding_status, u"Tabled") + self.assertEqual(source, "Education Infrastructure Grant") + self.assertEqual(investment, "Upgrading and Additions") + self.assertEqual(funding_status, "Tabled") self.assertIn(self.project.csv_download_url, csv_download_url) department = selenium.find_element_by_css_selector(".department-field").text @@ -270,12 +270,12 @@ def test_project_detail_page_fields(self): ".project-number-field" ).text - self.assertEqual(department, u"Education") + self.assertEqual(department, "Education") self.assertEqual( - budget_programme, u"Programme 2 - Public Ordinary School Education" + budget_programme, "Programme 2 - Public Ordinary School Education" ) - self.assertEqual(project_status, u"Construction") - self.assertEqual(project_number, u"W/50042423/WS") + self.assertEqual(project_status, "Construction") + self.assertEqual(project_number, "W/50042423/WS") province = selenium.find_element_by_css_selector(".province-field").text local_muni = selenium.find_element_by_css_selector( @@ -286,10 +286,10 @@ def test_project_detail_page_fields(self): ).text gps_location = selenium.find_element_by_css_selector(".coordinates-field").text - self.assertEqual(province, u"KwaZulu-Natal") - self.assertEqual(local_muni, u"Dr Nkosazana Dlamini Zuma") - self.assertEqual(district_muni, u"Harry Gwala") - self.assertEqual(gps_location, u"Not available") + self.assertEqual(province, "KwaZulu-Natal") + self.assertEqual(local_muni, "Dr Nkosazana Dlamini Zuma") + self.assertEqual(district_muni, "Harry Gwala") + self.assertEqual(gps_location, "Not available") implementing_agent = selenium.find_element_by_css_selector( ".program-implementing-agent-field" @@ -304,17 +304,17 @@ def test_project_detail_page_fields(self): ".other-service-providers-field" ).text - self.assertEqual(implementing_agent, u"DOPW") - self.assertEqual(principle_agent, u"PRINCIPLE AGENT") - self.assertEqual(main_contractor, u"MAIN CONTRACTOR") - self.assertEqual(others, u"OTHERS") + self.assertEqual(implementing_agent, "DOPW") + self.assertEqual(principle_agent, "PRINCIPLE AGENT") + self.assertEqual(main_contractor, "MAIN CONTRACTOR") + self.assertEqual(others, "OTHERS") professional_fees = selenium.find_element_by_css_selector( "#total-professional-fees-field" ).text - self.wait_until_text_in("#total-construction-costs-field", u"R 562,000") - self.assertEqual(professional_fees, u"R 118,000") + self.wait_until_text_in("#total-construction-costs-field", "R 562,000") + self.assertEqual(professional_fees, "R 118,000") expenditure_from_prev = selenium.find_element_by_css_selector( ".expenditure-from-previous-years-total-field" @@ -329,10 +329,10 @@ def test_project_detail_page_fields(self): ".variation-orders-field" ).text - self.assertEqual(expenditure_from_prev, u"R 556,479") - self.assertEqual(const_cost_from_prev, u"R 0") - self.assertEqual(prof_cost_from_prev, u"R 118,000") - self.assertEqual(variation_order, u"R 0") + self.assertEqual(expenditure_from_prev, "R 556,479") + self.assertEqual(const_cost_from_prev, "R 0") + self.assertEqual(prof_cost_from_prev, "R 118,000") + self.assertEqual(variation_order, "R 0") total_main_approp = selenium.find_element_by_css_selector( ".main-appropriation-total-field" @@ -344,17 +344,17 @@ def test_project_detail_page_fields(self): ".main-appropriation-professional-fees-field" ).text - self.assertEqual(total_main_approp, u"R 337,000") - self.assertEqual(const_cost_main_approp, u"R 276,000") - self.assertEqual(prof_fees_main_approp, u"R 61,000") + self.assertEqual(total_main_approp, "R 337,000") + self.assertEqual(const_cost_main_approp, "R 276,000") + self.assertEqual(prof_fees_main_approp, "R 61,000") start_date = selenium.find_element_by_css_selector(".start-date-field").text estimated_completion = selenium.find_element_by_css_selector( ".estimated-completion-date-field" ).text - self.assertEqual(start_date, u"2016-06-13") - self.assertEqual(estimated_completion, u"2021-06-30") + self.assertEqual(start_date, "2016-06-13") + self.assertEqual(estimated_completion, "2021-06-30") est_const_start_date = selenium.find_element_by_css_selector( ".estimated-construction-start-date-field" @@ -366,9 +366,9 @@ def test_project_detail_page_fields(self): ".estimated-construction-end-date-field" ).text - self.assertEqual(est_const_start_date, u"2017-02-01") - self.assertEqual(contracted_const_end_date, u"2021-01-01") - self.assertEqual(est__const_end_date, u"2020-12-31") + self.assertEqual(est_const_start_date, "2017-02-01") + self.assertEqual(contracted_const_end_date, "2021-01-01") + self.assertEqual(est__const_end_date, "2020-12-31") class InfraProjectSearchPageTestCase(BaseSeleniumTestCase): @@ -425,7 +425,7 @@ def test_search_homepage_correct_numbers(self): selenium.get("%s%s" % (self.live_server_url, self.url)) self.wait.until( EC.text_to_be_present_in_element( - (By.CSS_SELECTOR, "#num-matching-projects-field"), u"11" + (By.CSS_SELECTOR, "#num-matching-projects-field"), "11" ) ) num_of_projects = selenium.find_element_by_css_selector( @@ -447,7 +447,7 @@ def test_number_updated_after_search(self): selenium.get("%s%s" % (self.live_server_url, self.url)) self.wait.until( EC.text_to_be_present_in_element( - (By.CSS_SELECTOR, "#num-matching-projects-field"), u"11" + (By.CSS_SELECTOR, "#num-matching-projects-field"), "11" ) ) num_of_projects = selenium.find_element_by_css_selector( @@ -464,7 +464,7 @@ def test_number_updated_after_search(self): search_button.click() self.wait.until( EC.text_to_be_present_in_element( - (By.CSS_SELECTOR, "#num-matching-projects-field"), u"5" + (By.CSS_SELECTOR, "#num-matching-projects-field"), "5" ) ) filtered_num_of_projects = selenium.find_element_by_css_selector( @@ -478,7 +478,7 @@ def test_csv_download_button_populating(self): selenium.get("%s%s" % (self.live_server_url, self.url)) self.wait.until( EC.text_to_be_present_in_element( - (By.CSS_SELECTOR, "#num-matching-projects-field"), u"11" + (By.CSS_SELECTOR, "#num-matching-projects-field"), "11" ) ) csv_download_url = selenium.find_element_by_css_selector( @@ -1488,7 +1488,8 @@ def test_csv_download_empty_file(self): csv_download_url = response.data["csv_download_url"] response = self.client.get(csv_download_url) self._test_response_correctness( - response, "infrastructure-projects-q-data-that-won-t-be-found.csv", + response, + "infrastructure-projects-q-data-that-won-t-be-found.csv", ) content = b"".join(response.streaming_content) diff --git a/budgetportal/tests/test_pages.py b/budgetportal/tests/test_pages.py index 85263bfa9..f752e49f3 100644 --- a/budgetportal/tests/test_pages.py +++ b/budgetportal/tests/test_pages.py @@ -52,7 +52,7 @@ def setUp(self): Department.objects.create( government=fake_cape, name="Fake Health", vote_number=1, intro="" ) - models_ckan_patch = patch("budgetportal.models.ckan") + models_ckan_patch = patch("budgetportal.models.government.ckan") ModelsCKANMockClass = models_ckan_patch.start() ModelsCKANMockClass.action.package_search.return_value = {"results": []} self.addCleanup(models_ckan_patch.stop) @@ -141,7 +141,6 @@ def test_department_detail_page(self): self.assertContains( response, "The Presidency budget data for the 2019-20 financial year" ) - self.assertContains(response, "Budget (Main appropriation) 2019-20") def test_department_preview_page(self): """Test that it loads and that some text is present""" diff --git a/budgetportal/tests/test_summaries.py b/budgetportal/tests/test_summaries.py index 9cfb810d4..5933562b8 100644 --- a/budgetportal/tests/test_summaries.py +++ b/budgetportal/tests/test_summaries.py @@ -41,7 +41,7 @@ class ConsolidatedTreemapTestCase(TestCase): - """ Unit tests for the consolidated treemap function(s) """ + """Unit tests for the consolidated treemap function(s)""" def setUp(self): self.year = FinancialYear.objects.create(slug="2019-20") @@ -86,7 +86,7 @@ def test_complete_data(self, mock_get_dataset): class FocusAreaPagesTestCase(TestCase): - """ Integration test focus area page data generation """ + """Integration test focus area page data generation""" def setUp(self): self.year = FinancialYear.objects.create(slug="2019-20") @@ -184,7 +184,7 @@ def test_get_focus_area_preview( class NationalDepartmentPreviewTestCase(TestCase): - """ Unit tests for the national department preview department function. """ + """Unit tests for the national department preview department function.""" def setUp(self): self.mock_data = NATIONAL_DEPARTMENT_PREVIEW_MOCK_DATA diff --git a/budgetportal/urls.py b/budgetportal/urls.py index 193f5b2e9..9b3a36f8e 100644 --- a/budgetportal/urls.py +++ b/budgetportal/urls.py @@ -20,7 +20,6 @@ admin.autodiscover() CACHE_MINUTES_SECS = 60 * 5 # minutes -CACHE_DAYS_SECS = 60 * 60 * 24 * 5 # days def permission_denied(request): @@ -37,17 +36,17 @@ def trigger_error(request): ), url( r"^viz/subprog-treemap$", - cache_page(CACHE_DAYS_SECS)(views.department_viz_subprog_treemap), + cache_page(CACHE_MINUTES_SECS)(views.department_viz_subprog_treemap), name="department-viz-subprog-treemap", ), url( r"^viz/subprog-econ4-circles$", - cache_page(CACHE_DAYS_SECS)(views.department_viz_subprog_econ4_circles), + cache_page(CACHE_MINUTES_SECS)(views.department_viz_subprog_econ4_circles), name="department-viz-subprog-econ4-circles", ), url( r"^viz/subprog-econ4-bars$", - cache_page(CACHE_DAYS_SECS)(views.department_viz_subprog_econ4_bars), + cache_page(CACHE_MINUTES_SECS)(views.department_viz_subprog_econ4_bars), name="department-viz-subprog-econ4-bars", ), ] @@ -61,7 +60,7 @@ def trigger_error(request): ), url( r"^json/(?P\d{4}-\d{2})" "/focus.json", - cache_page(CACHE_DAYS_SECS)(views.focus_preview_json), + cache_page(CACHE_MINUTES_SECS)(views.focus_preview_json), name="focus-json", ), # National and provincial treemap data @@ -69,7 +68,7 @@ def trigger_error(request): r"^json/(?P\d{4}-\d{2})" "/(?P[\w-]+)" "/(?P[\w-]+).json", - cache_page(CACHE_DAYS_SECS)(views.treemaps_json), + cache_page(CACHE_MINUTES_SECS)(views.treemaps_json), ), # Preview pages url( @@ -87,13 +86,13 @@ def trigger_error(request): "/(?P[\w-]+)" "/(?P[\w-]+)" "/(?P[\w-]+).json", - cache_page(CACHE_DAYS_SECS)(views.department_preview_json), + cache_page(CACHE_MINUTES_SECS)(views.department_preview_json), name="department-preview-json", ), # Consolidated url( r"^json/(?P\d{4}-\d{2})" "/consolidated.json", - cache_page(CACHE_DAYS_SECS)(views.consolidated_treemap_json), + cache_page(CACHE_MINUTES_SECS)(views.consolidated_treemap_json), name="consolidated-json", ), # Homepage @@ -101,7 +100,7 @@ def trigger_error(request): # Search results url( r"^json/static-search.json", - cache_page(CACHE_DAYS_SECS)(views.static_search_data), + cache_page(CACHE_MINUTES_SECS)(views.static_search_data), ), # Department list as CSV url( @@ -130,14 +129,14 @@ def trigger_error(request): # CSV url( r"^csv/$", - cache_page(CACHE_DAYS_SECS)(views.openspending_csv), + cache_page(CACHE_MINUTES_SECS)(views.openspending_csv), name="openspending_csv", ), # Admin url(r"^admin/", admin.site.urls), url(r"^admin/bulk_upload/template", bulk_upload.template_view), # Budget Portal - url(r"^about/?$", cache_page(CACHE_DAYS_SECS)(views.about), name="about"), + url(r"^about/?$", cache_page(CACHE_MINUTES_SECS)(views.about), name="about"), url(r"^events/?$", cache_page(CACHE_MINUTES_SECS)(views.events), name="events"), url( r"^learning-resources/?$", @@ -151,12 +150,12 @@ def trigger_error(request): ), url( r"^terms-and-conditions/?$", - cache_page(CACHE_DAYS_SECS)(views.terms_and_conditions), + cache_page(CACHE_MINUTES_SECS)(views.terms_and_conditions), name="terms-and-conditions", ), url( r"^learning-resources/resources/?$", - cache_page(CACHE_DAYS_SECS)(views.resources), + cache_page(CACHE_MINUTES_SECS)(views.resources), name="resources", ), url( @@ -245,16 +244,29 @@ def trigger_error(request): "/(?P[\w-]+)/", include((department_urlpatterns, "provincial"), namespace="provincial"), ), - url(r"^robots\.txt$", views.robots,), + url( + r"^robots\.txt$", + views.robots, + ), + # Performance app + path("performance/", include("performance.urls")), + # IYM app + path("iym/", include("iym.urls")), + # Budget summary + url( + r"^budget-summary/?$", + cache_page(CACHE_MINUTES_SECS)(views.budget_summary_view), + name="budget-summary", + ), # Sitemap url( r"^sitemap\.xml$", - cache_page(CACHE_DAYS_SECS)(sitemap_views.index), + cache_page(CACHE_MINUTES_SECS)(sitemap_views.index), {"sitemaps": sitemaps}, ), url( r"^sitemap-(?P
    .+)\.xml$", - cache_page(CACHE_DAYS_SECS)(sitemap_views.sitemap), + cache_page(CACHE_MINUTES_SECS)(sitemap_views.sitemap), {"sitemaps": sitemaps}, name="django.contrib.sitemaps.views.sitemap", ), @@ -264,6 +276,7 @@ def trigger_error(request): re_path(r"^", include(wagtail_urls)), ] + static.static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + if settings.DEBUG_TOOLBAR: import debug_toolbar diff --git a/budgetportal/views.py b/budgetportal/views.py index db1e94f95..e95fc31fc 100644 --- a/budgetportal/views.py +++ b/budgetportal/views.py @@ -11,6 +11,7 @@ from budgetportal.csv_gen import generate_csv_response from budgetportal.openspending import PAGE_SIZE from django.conf import settings +from django.core.serializers import serialize from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Count from django.forms.models import model_to_dict @@ -18,6 +19,7 @@ from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse from haystack.query import SearchQuerySet +from constance import config from .datasets import Category, Dataset from .models import ( @@ -35,6 +37,7 @@ ProcurementResourceLink, Sphere, Video, + ShowcaseItem, ) from .summaries import ( DepartmentProgrammesEcon4, @@ -44,6 +47,7 @@ get_focus_area_preview, get_preview_page, ) +from .json_encoder import JSONEncoder logger = logging.getLogger(__name__) @@ -51,6 +55,25 @@ COMMON_DESCRIPTION_ENDING = "from National Treasury in partnership with IMALI YETHU." +def serialize_showcase(showcase_items): + showcase_items_dicts = [ + { + "name": i.name, + "description": i.description, + "cta_text_1": i.cta_text_1, + "cta_link_1": i.cta_link_1, + "cta_text_2": i.cta_text_2, + "cta_link_2": i.cta_link_2, + "second_cta_type": i.second_cta_type, + "thumbnail_url": i.file.url, + } + for i in showcase_items + ] + return json.dumps( + showcase_items_dicts, cls=DjangoJSONEncoder, sort_keys=True, indent=4 + ) + + def homepage(request): year = FinancialYear.get_latest_year() titles = { @@ -68,6 +91,8 @@ def homepage(request): .first() ) + showcase_items = ShowcaseItem.objects.all() + context = { "selected_financial_year": None, "financial_years": [], @@ -91,6 +116,7 @@ def homepage(request): "call_to_action_heading": page_data.call_to_action_heading, "call_to_action_link_label": page_data.call_to_action_link_label, "call_to_action_link_url": page_data.call_to_action_link_url, + "showcase_items_json": serialize_showcase(showcase_items), } return render(request, "homepage.html", context) @@ -375,17 +401,31 @@ def department_page( "selected_tab": "departments", "title": "%s budget %s - vulekamali" % (department.name, selected_year.slug), "description": "%s department: %s budget data for the %s financial year %s" - % (govt_label, department.name, selected_year.slug, COMMON_DESCRIPTION_ENDING,), + % ( + govt_label, + department.name, + selected_year.slug, + COMMON_DESCRIPTION_ENDING, + ), "department_budget": department_budget, "department_adjusted_budget": department_adjusted_budget, "procurement_resource_links": ProcurementResourceLink.objects.filter( - sphere_slug__in=("all", department.government.sphere.slug,) + sphere_slug__in=( + "all", + department.government.sphere.slug, + ) ), "performance_resource_links": PerformanceResourceLink.objects.filter( - sphere_slug__in=("all", department.government.sphere.slug,) + sphere_slug__in=( + "all", + department.government.sphere.slug, + ) ), "in_year_monitoring_resource_links": InYearMonitoringResourceLink.objects.filter( - sphere_slug__in=("all", department.government.sphere.slug,) + sphere_slug__in=( + "all", + department.government.sphere.slug, + ) ), "vote_number": department.vote_number, "vote_primary": { @@ -403,6 +443,8 @@ def department_page( context["admin_url"] = reverse( "admin:budgetportal_department_change", args=(department.pk,) ) + context["EQPRS_DATA_ENABLED"] = config.EQPRS_DATA_ENABLED + return render(request, "department.html", context) @@ -475,7 +517,7 @@ def department_viz_subprog_econ4_bars( def infrastructure_projects_overview(request): - """ Overview page to showcase all featured infrastructure projects """ + """Overview page to showcase all featured infrastructure projects""" infrastructure_projects = InfrastructureProjectPart.objects.filter( featured=True ).distinct("project_slug") @@ -966,7 +1008,7 @@ def department_list_json(request, financial_year_id): def treemaps_data(financial_year_id, phase_slug, sphere_slug): - """ The data for the vulekamali home page treemaps """ + """The data for the vulekamali home page treemaps""" dept = Department.objects.filter(government__sphere__slug=sphere_slug)[0] if sphere_slug == "national": page_data = dept.get_national_expenditure_treemap(financial_year_id, phase_slug) @@ -990,7 +1032,7 @@ def treemaps_json(request, financial_year_id, phase_slug, sphere_slug): def consolidated_treemap(financial_year_id): - """ The data for the vulekamali home page treemaps """ + """The data for the vulekamali home page treemaps""" financial_year = FinancialYear.objects.get(slug=financial_year_id) page_data = get_consolidated_expenditure_treemap(financial_year) if page_data is None: @@ -1009,7 +1051,7 @@ def consolidated_treemap_json(request, financial_year_id): def focus_preview_data(financial_year_id): - """ The data for the focus area preview pages for a specific year """ + """The data for the focus area preview pages for a specific year""" financial_year = FinancialYear.objects.get(slug=financial_year_id) page_data = get_focus_area_preview(financial_year) return page_data @@ -1079,3 +1121,19 @@ def robots(request): def read_object_from_yaml(path_file): with open(path_file, "r") as f: return yaml.load(f, Loader=yaml.FullLoader) + + +def budget_summary_view(request): + latest_provincial_year = ( + FinancialYear.objects.filter(spheres__slug="provincial") + .annotate(num_depts=Count("spheres__governments__departments")) + .filter(num_depts__gt=0) + .first() + ) + context = { + "navbar": MainMenuItem.objects.prefetch_related("children").all(), + "latest_year": FinancialYear.get_latest_year().slug, + "latest_provincial_year": latest_provincial_year + and latest_provincial_year.slug, + } + return render(request, "budget-summary.html", context) diff --git a/budgetportal/webflow/views.py b/budgetportal/webflow/views.py index 523b2e686..fd9c69fee 100644 --- a/budgetportal/webflow/views.py +++ b/budgetportal/webflow/views.py @@ -55,9 +55,11 @@ def infrastructure_project_detail(request, id, slug): ) page_data["department_url"] = department.get_url_path() if department else None - page_data["province_depts_url"] = ( - "/%s/departments?province=%s&sphere=provincial" - % (models.FinancialYear.get_latest_year().slug, slugify(snapshot.province),) + page_data[ + "province_depts_url" + ] = "/%s/departments?province=%s&sphere=provincial" % ( + models.FinancialYear.get_latest_year().slug, + slugify(snapshot.province), ) page_data[ "latest_snapshot_financial_year" diff --git a/docker-compose.yml b/docker-compose.yml index 5b09906fd..9925c74dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,7 @@ services: GROUP_ID: ${GROUP_ID:-1001} command: python manage.py runserver_plus --nopin 0.0.0.0:8000 environment: + - TAG_MANAGER_ID - DATABASE_URL=postgresql://budgetportal:devpassword@db/budgetportal - DJANGO_DEBUG_TOOLBAR - AWS_ACCESS_KEY_ID=minio-access-key @@ -30,6 +31,13 @@ services: - DEBUG_CACHE - DJANGO_WHITENOISE_AUTOREFRESH=TRUE - PORT=8000 + - CKAN_URL + - DJANGO_Q_SYNC=${DJANGO_Q_SYNC} + - CKAN_API_KEY + - OPENSPENDING_USER_ID + - OPENSPENDING_API_KEY + - OPENSPENDING_HOST + - OPENSPENDING_DATASET_CREATE_SUFFIX=-dev volumes: - .:/app ports: @@ -39,6 +47,8 @@ services: - solr - minio-client - minio + links: + - selenium # Should be same as app except for command and ports worker: @@ -56,30 +66,12 @@ services: - AWS_STORAGE_BUCKET_NAME=budgetportal-storage - AWS_S3_ENDPOINT_URL=http://minio:9000 - SOLR_URL=http://solr:8983/solr/budgetportal - volumes: - - .:/app - depends_on: - - db - - solr - - minio - restart: on-failure - - test: - build: - context: . - args: - USER_ID: ${USER_ID:-1001} - GROUP_ID: ${GROUP_ID:-1001} - command: bin/wait-for-db.sh coverage run --source='budgetportal' manage.py test - environment: - - DATABASE_URL=postgresql://budgetportal:devpassword@db/budgetportal - - AWS_ACCESS_KEY_ID=minio-access-key - - AWS_SECRET_ACCESS_KEY=minio-secret-key - - AWS_STORAGE_BUCKET_NAME=budgetportal-storage - - AWS_S3_ENDPOINT_URL=http://minio:9000 - - AWS_S3_SECURE_URLS=True - - SOLR_URL=http://solr:8983/solr/budgetportal-test - - DJANGO_Q_SYNC=TRUE + - CKAN_URL + - CKAN_API_KEY + - OPENSPENDING_USER_ID + - OPENSPENDING_API_KEY + - OPENSPENDING_HOST + - OPENSPENDING_DATASET_CREATE_SUFFIX=-dev volumes: - .:/app depends_on: @@ -120,8 +112,18 @@ services: - "8983:8983" volumes: - solr-data:/opt/solr/server/solr/budgetportal/data + ulimits: + nofile: + soft: 65536 + hard: 65536 - + selenium: + image: selenium/standalone-chrome:3.141 + ports: + - 4444:4444 + - 5900:5900 + - 7900:7900 + shm_size: '2gb' volumes: db-data: diff --git a/iym/__init__.py b/iym/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/iym/admin.py b/iym/admin.py new file mode 100644 index 000000000..f981d3f6b --- /dev/null +++ b/iym/admin.py @@ -0,0 +1,76 @@ +from django.contrib import admin +from iym import models +from django_q.tasks import async_task, fetch + +import iym +from iym.tasks import process_uploaded_file + + +class IYMFileUploadAdmin(admin.ModelAdmin): + readonly_fields = ( + "import_report", + "user", + "processing_completed", + "status", + "task_id", + ) + fieldsets = ( + ( + "", + { + "fields": ( + "user", + "financial_year", + "latest_quarter", + "file", + "task_id", + "import_report", + "status", + "processing_completed", + ) + }, + ), + ) + list_display = ( + "created_at", + "user", + "financial_year", + "latest_quarter", + "status", + "processing_completed", + "updated_at", + ) + + def save_model(self, request, obj, form, change): + if not obj.pk: + obj.user = request.user + super().save_model(request, obj, form, change) + + obj.task_id = async_task(func=process_uploaded_file, obj_id=obj.id) + obj.save() + + def get_readonly_fields(self, request, obj=None): + if obj: # editing an existing object + return ( + "financial_year", + "latest_quarter", + "file", + ) + self.readonly_fields + return self.readonly_fields + + def has_change_permission(self, request, obj=None): + super(IYMFileUploadAdmin, self).has_change_permission(request, obj) + + def has_delete_permission(self, request, obj=None): + super(IYMFileUploadAdmin, self).has_delete_permission(request, obj) + + def processing_completed(self, obj): + task = fetch(obj.task_id) + if task: + return task.success + + processing_completed.boolean = True + processing_completed.short_description = "Processing completed" + + +admin.site.register(models.IYMFileUpload, IYMFileUploadAdmin) diff --git a/iym/apps.py b/iym/apps.py new file mode 100644 index 000000000..b9e0df6d6 --- /dev/null +++ b/iym/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class IymConfig(AppConfig): + name = "In-year monitoring" diff --git a/iym/data_package/data_package_template.json b/iym/data_package/data_package_template.json new file mode 100644 index 000000000..3e55d4219 --- /dev/null +++ b/iym/data_package/data_package_template.json @@ -0,0 +1,998 @@ +{ + "promise": {}, + "resources": [ + { + "format": "csv", + "mediatype": "text/csv", + "dialect": { + "csvddfVersion": 1, + "delimiter": ",", + "lineTerminator": "\n" + }, + "encoding": "utf-8", + "schema": { + "fields": [ + { + "title": "VoteNumber", + "name": "Vote No#", + "slug": "VoteNumber", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level1:code", + "conceptType": "administrative-classification" + }, + { + "title": "Department", + "name": "Department", + "slug": "Department", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level1:label", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel1", + "name": "Responsibility_Level_1", + "slug": "ResponsibilityLevel1", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level2:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel2", + "name": "Responsibility_Level_2", + "slug": "ResponsibilityLevel2", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level3:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel3", + "name": "Responsibility_Level_3", + "slug": "ResponsibilityLevel3", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level4:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel4", + "name": "Responsibility_Level_4", + "slug": "ResponsibilityLevel4", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level5:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel5", + "name": "Responsibility_Level_5", + "slug": "ResponsibilityLevel5", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level6:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel6", + "name": "Responsibility_Level_6", + "slug": "ResponsibilityLevel6", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level7:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel7", + "name": "Responsibility_Level_7", + "slug": "ResponsibilityLevel7", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level8:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel8", + "name": "Responsibility_Level_8", + "slug": "ResponsibilityLevel8", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level9:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel9", + "name": "Responsibility_Level_9", + "slug": "ResponsibilityLevel9", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level10:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel10", + "name": "Responsibility_Level_10", + "slug": "ResponsibilityLevel10", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level11:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel11", + "name": "Responsibility_Level_11", + "slug": "ResponsibilityLevel11", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level12:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel12", + "name": "Responsibility_Level_12", + "slug": "ResponsibilityLevel12", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level13:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel13", + "name": "Responsibility_Level_13", + "slug": "ResponsibilityLevel13", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level14:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel14", + "name": "Responsibility_Level_14", + "slug": "ResponsibilityLevel14", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level15:code", + "conceptType": "administrative-classification" + }, + { + "title": "ResponsibilityLevel15", + "name": "Responsibility_Lowest_Level", + "slug": "ResponsibilityLevel15", + "type": "string", + "format": "default", + "columnType": "administrative-classification:generic:level16:code", + "conceptType": "administrative-classification" + }, + { + "title": "ProgNumber", + "name": "Programme No#", + "slug": "ProgNumber", + "type": "string", + "format": "default", + "columnType": "activity:generic:program:code", + "conceptType": "activity" + }, + { + "title": "Programme", + "name": "Programme", + "slug": "Programme", + "type": "string", + "format": "default", + "columnType": "activity:generic:program:label", + "conceptType": "activity" + }, + { + "title": "SubprogNumber", + "name": "Subprogramme No#", + "slug": "SubprogNumber", + "type": "string", + "format": "default", + "columnType": "activity:generic:subprogram:code", + "conceptType": "activity" + }, + { + "title": "Subprogramme", + "name": "Subprogramme", + "slug": "Subprogramme", + "type": "string", + "format": "default", + "columnType": "activity:generic:subprogram:label", + "conceptType": "activity" + }, + + { + "title": "EconomicClassification1", + "name": "econClass_L1", + "slug": "EconomicClassification1", + "type": "string", + "format": "default", + "columnType": "economic-classification:generic:level1:code", + "conceptType": "economic-classification" + }, + { + "title": "EconomicClassification2", + "name": "econClass_L2", + "slug": "EconomicClassification2", + "type": "string", + "format": "default", + "columnType": "economic-classification:generic:level2:code", + "conceptType": "economic-classification" + }, + { + "title": "EconomicClassification3", + "name": "econClass_L3", + "slug": "EconomicClassification3", + "type": "string", + "format": "default", + "columnType": "economic-classification:generic:level3:code", + "conceptType": "economic-classification" + }, + { + "title": "EconomicClassification4", + "name": "econClass_L4", + "slug": "EconomicClassification4", + "type": "string", + "format": "default", + "columnType": "economic-classification:generic:level4:code", + "conceptType": "economic-classification" + }, + { + "title": "EconomicClassification5", + "name": "econClass_L5", + "slug": "EconomicClassification5", + "type": "string", + "format": "default", + "columnType": "economic-classification:generic:level5:code", + "conceptType": "economic-classification" + }, + { + "title": "EconomicClassificationLowestLevel", + "name": "IYM_econLowestLevel", + "slug": "EconomicClassificationLowestLEvel", + "type": "string", + "format": "default", + "columnType": "economic-classification:generic:level6:code", + "conceptType": "economic-classification" + }, + + + { + "title": "AssetClassification1", + "name": "Assets_Level_1", + "slug": "AssetClassification1", + "type": "string", + "format": "default", + "columnType": "asset:generic:level1:code", + "conceptType": "asset" + }, + { + "title": "AssetClassification2", + "name": "Assets_Level_2", + "slug": "AssetClassification2", + "type": "string", + "format": "default", + "columnType": "asset:generic:level2:code", + "conceptType": "asset" + }, + { + "title": "AssetClassification3", + "name": "Assets_Level_3", + "slug": "AssetClassification3", + "type": "string", + "format": "default", + "columnType": "asset:generic:level3:code", + "conceptType": "asset" + }, + { + "title": "AssetClassification4", + "name": "Assets_Level_4", + "slug": "AssetClassification4", + "type": "string", + "format": "default", + "columnType": "asset:generic:level4:code", + "conceptType": "asset" + }, + { + "title": "AssetClassification5", + "name": "Assets_Level_5", + "slug": "AssetClassification5", + "type": "string", + "format": "default", + "columnType": "asset:generic:level5:code", + "conceptType": "asset" + }, + { + "title": "AssetClassification6", + "name": "Assets_Level_6", + "slug": "AssetClassification6", + "type": "string", + "format": "default", + "columnType": "asset:generic:level6:code", + "conceptType": "asset" + }, + { + "title": "AssetClassificationLowestLevel", + "name": "Assets_Lowest_Level", + "slug": "AssetClassificationLowestLevel", + "type": "string", + "format": "default", + "columnType": "asset:generic:level7:code", + "conceptType": "asset" + }, + + { + "title": "ProjectLevel1", + "name": "Project_Level_1", + "slug": "ProjectLevel1", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level1:code" + }, + { + "title": "ProjectLevel2", + "name": "Project_Level_2", + "slug": "ProjectLevel2", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level2:code" + }, + { + "title": "ProjectLevel3", + "name": "Project_Level_3", + "slug": "ProjectLevel3", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level3:code" + }, + { + "title": "ProjectLevel4", + "name": "Project_Level_4", + "slug": "ProjectLevel4", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level4:code" + }, + { + "title": "ProjectLevel5", + "name": "Project_Level_5", + "slug": "ProjectLevel5", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level5:code" + }, + { + "title": "ProjectLevel6", + "name": "Project_Level_6", + "slug": "ProjectLevel6", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level6:code" + }, + { + "title": "ProjectLevel7", + "name": "Project_Level_7", + "slug": "ProjectLevel7", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level7:code" + }, + { + "title": "ProjectLevel8", + "name": "Project_Level_8", + "slug": "ProjectLevel8", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level8:code" + }, + { + "title": "ProjectLevel9", + "name": "Project_Level_9", + "slug": "ProjectLevel9", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level9:code" + }, + { + "title": "ProjectLevel10", + "name": "Project_Level_10", + "slug": "ProjectLevel10", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level10:code" + }, + { + "title": "ProjectLevel11", + "name": "Project_Level_11", + "slug": "ProjectLevel11", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level11:code" + }, + { + "title": "ProjectLowestLevel", + "name": "Project_Lowest_Level", + "slug": "ProjectLowest", + "type": "string", + "format": "default", + "columnType": "activity:generic:project:level12:code" + }, + + + { + "title": "FundLevel1", + "name": "Fund_Level_1", + "slug": "FundLevel1", + "type": "string", + "format": "default", + "columnType": "fin-source:generic:level1:code" + }, + { + "title": "FundLevel2", + "name": "Fund_Level_2", + "slug": "FundLevel2", + "type": "string", + "format": "default", + "columnType": "fin-source:generic:level2:code" + }, + { + "title": "FundLevel3", + "name": "Fund_Level_3", + "slug": "FundLevel3", + "type": "string", + "format": "default", + "columnType": "fin-source:generic:level3:code" + }, + { + "title": "FundLevel4", + "name": "Fund_Level_4", + "slug": "FundLevel4", + "type": "string", + "format": "default", + "columnType": "fin-source:generic:level4:code" + }, + { + "title": "FundLevel5", + "name": "Fund_Level_5", + "slug": "FundLevel5", + "type": "string", + "format": "default", + "columnType": "fin-source:generic:level5:code" + }, + { + "title": "FundLevel6", + "name": "Fund_Level_6", + "slug": "FundLevel6", + "type": "string", + "format": "default", + "columnType": "fin-source:generic:level6:code" + }, + { + "title": "FundLevel7", + "name": "Fund_Level_7", + "slug": "FundLevel7", + "type": "string", + "format": "default", + "columnType": "fin-source:generic:level7:code" + }, + { + "title": "FundLevel8", + "name": "Fund_Level_8", + "slug": "FundLevel8", + "type": "string", + "format": "default", + "columnType": "fin-source:generic:level8:code" + }, + { + "title": "FundLowestLevel", + "name": "Fund_Lowest_Level", + "slug": "FundLowestLevel", + "type": "string", + "format": "default", + "columnType": "fin-source:generic:level9:code" + }, + + { + "title": "InfrastructureLevel1", + "name": "Infrastructure_Level_1", + "slug": "InfrastructureLevel1", + "type": "string", + "format": "default", + "columnType": "infrastructure:generic:level1:code" + }, + { + "title": "InfrastructureLevel2", + "name": "Infrastructure_Level_2", + "slug": "InfrastructureLevel2", + "type": "string", + "format": "default", + "columnType": "infrastructure:generic:level2:code" + }, + { + "title": "InfrastructureLevel3", + "name": "Infrastructure_Level_3", + "slug": "InfrastructureLevel3", + "type": "string", + "format": "default", + "columnType": "infrastructure:generic:level3:code" + }, + { + "title": "InfrastructureLevel4", + "name": "Infrastructure_Level_4", + "slug": "InfrastructureLevel4", + "type": "string", + "format": "default", + "columnType": "infrastructure:generic:level4:code" + }, + { + "title": "InfrastructureLevel5", + "name": "Infrastructure_Level_5", + "slug": "InfrastructureLevel5", + "type": "string", + "format": "default", + "columnType": "infrastructure:generic:level5:code" + }, + { + "title": "InfrastructureLevel6", + "name": "Infrastructure_Level_6", + "slug": "InfrastructureLevel6", + "type": "string", + "format": "default", + "columnType": "infrastructure:generic:level6:code" + }, + { + "title": "InfrastructureLowestLevel", + "name": "Infrastructure_Lowest_Level", + "slug": "InfrastructureLowestLevel", + "type": "string", + "format": "default", + "columnType": "infrastructure:generic:level7:code" + }, + + { + "title": "ItemLevel1", + "name": "Item_Level_1", + "slug": "ItemLevel1", + "type": "string", + "format": "default", + "columnType": "item:generic:level1:code" + }, + { + "title": "ItemLevel2", + "name": "Item_Level_2", + "slug": "ItemLevel2", + "type": "string", + "format": "default", + "columnType": "item:generic:level2:code" + }, + { + "title": "ItemLevel3", + "name": "Item_Level_3", + "slug": "ItemLevel3", + "type": "string", + "format": "default", + "columnType": "item:generic:level3:code" + }, + { + "title": "ItemLevel4", + "name": "Item_Level_4", + "slug": "ItemLevel4", + "type": "string", + "format": "default", + "columnType": "item:generic:level4:code" + }, + { + "title": "ItemLevel5", + "name": "Item_Level_5", + "slug": "ItemLevel5", + "type": "string", + "format": "default", + "columnType": "item:generic:level5:code" + }, + { + "title": "ItemLevel6", + "name": "Item_Level_6", + "slug": "ItemLevel6", + "type": "string", + "format": "default", + "columnType": "item:generic:level6:code" + }, + { + "title": "ItemLevel7", + "name": "Item_Level_7", + "slug": "ItemLevel7", + "type": "string", + "format": "default", + "columnType": "item:generic:level7:code" + }, + { + "title": "ItemLevel8", + "name": "Item_Level_8", + "slug": "ItemLevel8", + "type": "string", + "format": "default", + "columnType": "item:generic:level8:code" + }, + + { + "title": "ItemLowestLevel", + "name": "Item_Lowest_Level", + "slug": "ItemLowestLevel", + "type": "string", + "format": "default", + "columnType": "item:generic:level9:code" + }, + + { + "title": "RegionalIDLevel1", + "name": "Regional_ID_Level_1", + "slug": "RegionalIDLevel1", + "type": "string", + "format": "default", + "columnType": "geo-source:target:level1:code" + }, + { + "title": "RegionalIDLevel2", + "name": "Regional_ID_Level_2", + "slug": "RegionalIDLevel2", + "type": "string", + "format": "default", + "columnType": "geo-source:target:level2:code" + }, + { + "title": "RegionalIDLevel3", + "name": "Regional_ID_Level_3", + "slug": "RegionalIDLevel3", + "type": "string", + "format": "default", + "columnType": "geo-source:target:level3:code" + }, + { + "title": "RegionalIDLevel4", + "name": "Regional_ID_Level_4", + "slug": "RegionalIDLevel4", + "type": "string", + "format": "default", + "columnType": "geo-source:target:level4:code" + }, + { + "title": "RegionalIDLevel5", + "name": "Regional_ID_Level_5", + "slug": "RegionalIDLevel5", + "type": "string", + "format": "default", + "columnType": "geo-source:target:level5:code" + }, + { + "title": "RegionalIDLevel6", + "name": "Regional_ID_Level_6", + "slug": "RegionalIDLevel6", + "type": "string", + "format": "default", + "columnType": "geo-source:target:level6:code" + }, + { + "title": "RegionalIDLevel7", + "name": "Regional_ID_Level_7", + "slug": "RegionalIDLevel7", + "type": "string", + "format": "default", + "columnType": "geo-source:target:level7:code" + }, + { + "title": "RegionalIDLevel8", + "name": "Regional_ID_Level_8", + "slug": "RegionalIDLevel8", + "type": "string", + "format": "default", + "columnType": "geo-source:target:level8:code" + }, + { + "title": "RegionalIDLowestLevel", + "name": "Regional_ID_Lowest_Level", + "slug": "RegionalIDLowestLevel", + "type": "string", + "format": "default", + "columnType": "geo-source:target:level9:code" + }, + + { + "title": "year", + "name": "Financial_Year", + "slug": "year", + "type": "integer", + "format": "default", + "columnType": "date:fiscal-year", + "conceptType": "date" + }, + { + "title": "Budget", + "name": "Budget", + "slug": "Budget", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "AdjustmentBudget", + "name": "AdjustmentBudget", + "slug": "AdjustmentBudget", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "April", + "name": "April", + "slug": "April", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "May", + "name": "May", + "slug": "May", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "June", + "name": "June", + "slug": "June", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "July", + "name": "July", + "slug": "July", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "August", + "name": "August", + "slug": "August", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "September", + "name": "September", + "slug": "September", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "October", + "name": "October", + "slug": "October", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "November", + "name": "November", + "slug": "November", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "December", + "name": "December", + "slug": "December", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "January", + "name": "January", + "slug": "January", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "February", + "name": "February", + "slug": "February", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "March", + "name": "March", + "slug": "March", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "Q1", + "name": "Q1", + "slug": "Q1", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "Q2", + "name": "Q2", + "slug": "Q2", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "Q3", + "name": "Q3", + "slug": "Q3", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + }, + { + "title": "Q4", + "name": "Q4", + "slug": "Q4", + "type": "number", + "format": "default", + "columnType": "value", + "conceptType": "value", + "decimalChar": ".", + "groupChar": "," + } + ], + "primaryKey": [ + "VoteNumber", + "Department", + "ResponsibilityLevel1", + "ResponsibilityLevel2", + "ResponsibilityLevel3", + "ResponsibilityLevel4", + "ResponsibilityLevel5", + "ResponsibilityLevel6", + "ResponsibilityLevel7", + "ResponsibilityLevel8", + "ResponsibilityLevel9", + "ResponsibilityLevel10", + "ResponsibilityLevel11", + "ResponsibilityLevel12", + "ResponsibilityLevel13", + "ResponsibilityLevel14", + "ResponsibilityLevel15", + "ProgNumber", + "Programme", + "SubprogNumber", + "Subprogramme", + "EconomicClassification1", + "EconomicClassification2", + "EconomicClassification3", + "EconomicClassification4", + "EconomicClassification5", + "EconomicClassificationLowestLEvel", + "AssetClassification1", + "AssetClassification2", + "AssetClassification3", + "AssetClassification4", + "AssetClassification5", + "AssetClassification6", + "AssetClassificationLowestLEvel", + "ProjectLevel1", + "ProjectLevel2", + "ProjectLevel3", + "ProjectLevel4", + "ProjectLevel5", + "ProjectLevel6", + "ProjectLevel7", + "ProjectLevel8", + "ProjectLevel9", + "ProjectLevel10", + "ProjectLevel11", + "ProjectLowest", + "FundLevel1", + "FundLevel2", + "FundLevel3", + "FundLevel4", + "FundLevel5", + "FundLevel6", + "FundLevel7", + "FundLevel8", + "FundLowestLevel", + "InfrastructureLevel1", + "InfrastructureLevel2", + "InfrastructureLevel3", + "InfrastructureLevel4", + "InfrastructureLevel5", + "InfrastructureLevel6", + "InfrastructureLowestLevel", + "ItemLevel1", + "ItemLevel2", + "ItemLevel3", + "ItemLevel4", + "ItemLevel5", + "ItemLevel6", + "ItemLevel7", + "ItemLevel8", + "ItemLowestLevel", + "RegionalIDLevel1", + "RegionalIDLevel2", + "RegionalIDLevel3", + "RegionalIDLevel4", + "RegionalIDLevel5", + "RegionalIDLevel6", + "RegionalIDLevel7", + "RegionalIDLevel8", + "RegionalIDLowestLevel", + "year" + ] + } + } + ], + "@context": "http://schemas.frictionlessdata.io/fiscal-data-package.jsonld", + "owner": "b9d2af843f3a7ca223eea07fb608e62a", + "author": "vulekamali South African Budget Portal", + "count_of_rows": 8 +} \ No newline at end of file diff --git a/iym/migrations/0001_initial.py b/iym/migrations/0001_initial.py new file mode 100644 index 000000000..0373eff8f --- /dev/null +++ b/iym/migrations/0001_initial.py @@ -0,0 +1,65 @@ +# Generated by Django 2.2.28 on 2023-06-07 08:23 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import iym.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("budgetportal", "0071_auto_20230605_1521"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="IYMFileUpload", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "latest_quarter", + models.CharField( + choices=[ + ("Q1", "Q1"), + ("Q2", "Q2"), + ("Q3", "Q3"), + ("Q4", "Q4"), + ], + max_length=2, + ), + ), + ("process_completed", models.BooleanField(default=False)), + ("import_report", models.TextField()), + ("file", models.FileField(upload_to=iym.models.iym_file_path)), + ("created_at", models.DateTimeField(auto_now_add=True, null=True)), + ("updated_at", models.DateTimeField(auto_now=True, null=True)), + ( + "financial_year", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="budgetportal.FinancialYear", + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + ] diff --git a/iym/migrations/0002_iymfileupload_status.py b/iym/migrations/0002_iymfileupload_status.py new file mode 100644 index 000000000..20373793d --- /dev/null +++ b/iym/migrations/0002_iymfileupload_status.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.28 on 2023-06-14 17:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("iym", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="iymfileupload", + name="status", + field=models.TextField(default=""), + preserve_default=False, + ), + ] diff --git a/iym/migrations/0003_iymfileupload_task_id.py b/iym/migrations/0003_iymfileupload_task_id.py new file mode 100644 index 000000000..4ef91fede --- /dev/null +++ b/iym/migrations/0003_iymfileupload_task_id.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.28 on 2023-06-26 04:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("iym", "0002_iymfileupload_status"), + ] + + operations = [ + migrations.AddField( + model_name="iymfileupload", + name="task_id", + field=models.TextField(default=""), + preserve_default=False, + ), + ] diff --git a/iym/migrations/0004_auto_20230703_1449.py b/iym/migrations/0004_auto_20230703_1449.py new file mode 100644 index 000000000..d5a8253c0 --- /dev/null +++ b/iym/migrations/0004_auto_20230703_1449.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2023-07-03 14:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("iym", "0003_iymfileupload_task_id"), + ] + + operations = [ + migrations.AlterField( + model_name="iymfileupload", + name="process_completed", + field=models.BooleanField(), + ), + ] diff --git a/iym/migrations/0005_auto_20230703_1456.py b/iym/migrations/0005_auto_20230703_1456.py new file mode 100644 index 000000000..724410190 --- /dev/null +++ b/iym/migrations/0005_auto_20230703_1456.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2023-07-03 14:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("iym", "0004_auto_20230703_1449"), + ] + + operations = [ + migrations.AlterField( + model_name="iymfileupload", + name="process_completed", + field=models.BooleanField(default=False), + ), + ] diff --git a/iym/migrations/__init__.py b/iym/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/iym/models.py b/iym/models.py new file mode 100644 index 000000000..759a09839 --- /dev/null +++ b/iym/models.py @@ -0,0 +1,28 @@ +from django.contrib.auth.models import User +from django.db import models +from budgetportal.models import FinancialYear +import uuid + +QUARTERS = ( + ("Q1", "Q1"), + ("Q2", "Q2"), + ("Q3", "Q3"), + ("Q4", "Q4"), +) + + +def iym_file_path(instance, filename): + return f"iym_uploads/{uuid.uuid4()}/{filename}" + + +class IYMFileUpload(models.Model): + user = models.ForeignKey(User, models.DO_NOTHING, blank=True) + financial_year = models.ForeignKey(FinancialYear, on_delete=models.CASCADE) + latest_quarter = models.CharField(max_length=2, choices=QUARTERS) + process_completed = models.BooleanField(default=False) + import_report = models.TextField() + status = models.TextField() + file = models.FileField(upload_to=iym_file_path) + task_id = models.TextField() + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) + updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) diff --git a/iym/tasks.py b/iym/tasks.py new file mode 100644 index 000000000..dab354355 --- /dev/null +++ b/iym/tasks.py @@ -0,0 +1,454 @@ +from django.contrib import admin +from iym import models +from io import StringIO +import petl as etl +from decimal import Decimal +from urllib.parse import urlencode +from slugify import slugify +from zipfile import ZipFile +from django.conf import settings +from django_q.tasks import async_task +import logging + +import os +import csv +import tempfile +import requests +import hashlib +import base64 +import json +import time +import re +import iym +import datetime + +logger = logging.getLogger(__name__) +ckan = settings.CKAN + +RE_END_YEAR = re.compile(r"/\d+") + +MEASURES = [ + "Budget", + "AdjustmentBudget", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + "January", + "February", + "March", + "Q1", + "Q2", + "Q3", + "Q4", +] + + +def authorise_upload(path, filename, userid, data_package_name, datastore_token): + md5 = hashlib.md5() + with open(path, "rb") as fh: + while True: + bytes = fh.read(1000) + if not bytes: + break + else: + md5.update(bytes) + + md5_b64 = base64.b64encode(md5.digest()) + + authorize_upload_url = f"{settings.OPENSPENDING_HOST}/datastore/" + authorize_upload_payload = { + "metadata": { + "owner": userid, + "name": data_package_name, + "author": "Vulekamali", + }, + "filedata": { + filename: { + "md5": md5_b64, + "name": filename, + "length": os.stat(path).st_size, + "type": "application/octet-stream", + } + }, + } + authorize_upload_headers = {"auth-token": datastore_token} + r = requests.post( + authorize_upload_url, + json=authorize_upload_payload, + headers=authorize_upload_headers, + ) + + r.raise_for_status() + return r.json() + + +def upload(path, authorisation): + # Unpack values out of lists + upload_query = {k: v[0] for k, v in authorisation["upload_query"].items()} + upload_url = f"{authorisation['upload_url']}?{urlencode(upload_query)}" + upload_headers = { + "content-type": "application/octet-stream", + "Content-MD5": authorisation["md5"], + } + with open(path, "rb") as file: + r = requests.put(upload_url, data=file, headers=upload_headers) + r.raise_for_status() + + +def unzip_uploaded_file(obj_to_update): + relative_path = "iym/temp_files/" + zip_file = obj_to_update.file + + with ZipFile(zip_file, "r") as zip: + file_name = zip.namelist()[0] + zip.extractall(path=relative_path) + + original_csv_path = os.path.join(settings.BASE_DIR, relative_path, file_name) + + return original_csv_path + + +def create_composite_key_using_csv_headers(original_csv_path): + # Get all the headers to come up with the composite key + with open(original_csv_path) as original_csv_file: + reader = csv.DictReader(original_csv_file) + first_row = next(reader) + fields = first_row.keys() + composite_key = list(set(fields) - set(MEASURES)) + + return composite_key + + +def tidy_csv_table(original_csv_path, composite_key): + table1 = etl.fromcsv(original_csv_path) + table2 = etl.convert(table1, MEASURES, lambda v: v.replace(",", ".")) + table3 = etl.convert(table2, "Financial_Year", lambda v: RE_END_YEAR.sub("", v)) + table4 = etl.convert(table3, MEASURES, Decimal) + + # Roll up rows with the same composite key into one, summing values together + # Prefixing each new measure header with "sum" because petl seems to need + # different headers for aggregation output + aggregation = {f"sum{measure}": (measure, sum) for measure in MEASURES} + table5 = etl.aggregate(table4, composite_key, aggregation) + + # Strip sum prefix from aggregation results + measure_rename = {key: key[3:] for key in aggregation} + table6 = etl.rename(table5, measure_rename) + + return table6 + + +def authenticate_openspending(): + headers = {"x-api-key": settings.OPENSPENDING_API_KEY} + url = f"{settings.OPENSPENDING_HOST}/user/authenticate_api_key" + r = requests.post(url, headers=headers) + r.raise_for_status() + return r.json()["token"] + + +def create_data_package( + csv_filename, + csv_table, + userid, + data_package_name, + data_package_title, + obj_to_update, +): + data_package_template_path = "iym/data_package/data_package_template.json" + base_token = authenticate_openspending() + + with tempfile.NamedTemporaryFile(mode="w", delete=True) as csv_file: + csv_path = csv_file.name + etl.tocsv(csv_table, csv_path) + update_import_report(obj_to_update, "Getting authorisation for datastore") + + authorize_query = { + "jwt": base_token, + "service": "os.datastore", + "userid": userid, + } + authorize_url = ( + f"{settings.OPENSPENDING_HOST}/user/authorize?{urlencode(authorize_query)}" + ) + r = requests.get(authorize_url) + + r.raise_for_status() + + authorize_result = r.json() + if "token" not in authorize_result: + raise Exception("Authorization with OpenSpending failed.") + + datastore_token = authorize_result["token"] + + update_import_report(obj_to_update, f"Uploading CSV {csv_path}") + + authorise_csv_upload_result = authorise_upload( + csv_path, csv_filename, userid, data_package_name, datastore_token + ) + + upload(csv_path, authorise_csv_upload_result["filedata"][csv_filename]) + + ##=============================================== + update_import_report(obj_to_update, "Creating and uploading datapackage.json") + with open(data_package_template_path) as data_package_file: + data_package = json.load(data_package_file) + + data_package["title"] = data_package_title + data_package["name"] = data_package_name + data_package["resources"][0]["name"] = slugify( + os.path.splitext(csv_filename)[0] + ) + data_package["resources"][0]["path"] = csv_filename + data_package["resources"][0]["bytes"] = os.path.getsize(csv_path) + + return {"data_package": data_package, "datastore_token": datastore_token} + + +def upload_data_package( + data_package, userid, data_package_name, datastore_token, obj_to_update +): + with tempfile.NamedTemporaryFile(mode="w", delete=True) as data_package_file: + json.dump(data_package, data_package_file) + data_package_file.flush() + data_package_path = data_package_file.name + authorise_data_package_upload_result = authorise_upload( + data_package_path, + "data_package.json", + userid, + data_package_name, + datastore_token, + ) + + data_package_upload_authorisation = authorise_data_package_upload_result[ + "filedata" + ]["data_package.json"] + upload(data_package_path, data_package_upload_authorisation) + update_import_report( + obj_to_update, + f'Datapackage url: {data_package_upload_authorisation["upload_url"]}', + ) + + return data_package_upload_authorisation + + +def import_uploaded_package(data_package_url, datastore_token, obj_to_update): + import_query = {"datapackage": data_package_url, "jwt": datastore_token} + import_url = ( + f"{settings.OPENSPENDING_HOST}/package/upload?{urlencode(import_query)}" + ) + r = requests.post(import_url) + update_import_report(obj_to_update, f"Initial status: {r.text}") + + r.raise_for_status() + status = r.json()["status"] + + return status + + +def update_import_report(obj_to_update, message): + now = datetime.datetime.now().strftime("%H:%M:%S") + obj_to_update.import_report += f"{now} - {message}" + os.linesep + obj_to_update.save() + + +def check_and_update_status(status, data_package_url, obj_to_update): + last_logged_progress = -1 + status_query = { + "datapackage": data_package_url, + } + status_url = ( + f"{settings.OPENSPENDING_HOST}/package/status?{urlencode(status_query)}" + ) + update_import_report( + obj_to_update, f"Monitoring status until completion ({status_url}):" + ) + while status not in ["done", "fail"]: + time.sleep(5) + r = requests.get(status_url) + r.raise_for_status() + status_result = r.json() + new_progress = int(float(status_result["progress"]) * 100) + if new_progress != last_logged_progress: + update_status( + obj_to_update, + f"loading data ({new_progress}%)", + ) + update_import_report( + obj_to_update, + f"loading data ({new_progress}%)", + ) + last_logged_progress = new_progress + status = status_result["status"] + + if status == "fail": + print(status_result["error"]) + + update_status(obj_to_update, status) + + +def update_status(obj_to_update, status): + obj_to_update.status = status + obj_to_update.save() + + +def process_uploaded_file(obj_id): + # read file + obj_to_update = models.IYMFileUpload.objects.get(id=obj_id) + if obj_to_update.process_completed: + return + try: + update_status(obj_to_update, "process started") + update_import_report(obj_to_update, "Cleaning CSV") + + financial_year = obj_to_update.financial_year.slug + userid = settings.OPENSPENDING_USER_ID + data_package_name = f"national-in-year-spending-{financial_year}{settings.OPENSPENDING_DATASET_CREATE_SUFFIX}" + data_package_title = f"National in-year spending {financial_year}{settings.OPENSPENDING_DATASET_CREATE_SUFFIX}" + + original_csv_path = unzip_uploaded_file(obj_to_update) + + update_status(obj_to_update, "cleaning data") + + csv_filename = os.path.basename(original_csv_path) + + composite_key = create_composite_key_using_csv_headers(original_csv_path) + + csv_table = tidy_csv_table(original_csv_path, composite_key) + + update_status(obj_to_update, "uploading data") + + func_result = create_data_package( + csv_filename, + csv_table, + userid, + data_package_name, + data_package_title, + obj_to_update, + ) + + data_package = func_result["data_package"] + datastore_token = func_result["datastore_token"] + + data_package_upload_authorisation = upload_data_package( + data_package, userid, data_package_name, datastore_token, obj_to_update + ) + + ##=============================================== + # Starting import of uploaded data_package + update_status(obj_to_update, "import queued") + update_import_report(obj_to_update, "Starting import of uploaded datapackage.") + data_package_url = data_package_upload_authorisation["upload_url"] + status = import_uploaded_package( + data_package_url, datastore_token, obj_to_update + ) + + ##=============================================== + + check_and_update_status(status, data_package_url, obj_to_update) + + os.remove(original_csv_path) + + create_or_update_dataset( + obj_to_update, + financial_year, + userid, + data_package_name, + obj_to_update.latest_quarter, + ) + + obj_to_update.process_completed = True + obj_to_update.save() + except Exception as e: + logger.exception("Error processing file") + update_import_report(obj_to_update, str(e)) + update_status(obj_to_update, "fail") + + +def create_or_update_dataset( + obj_to_update, financial_year, userid, data_package_name, latest_quarter +): + update_import_report(obj_to_update, "CKAN process started") + vocab_map = get_vocab_map() + tags = [ + {"vocabulary_id": vocab_map["financial_years"], "name": financial_year}, + {"vocabulary_id": vocab_map["spheres"], "name": "national"}, + ] + + dataset_fields = { + "title": f"National in-year spending {financial_year}", + "name": f"national_in_year_spending_{financial_year}", + "owner_org": "national-treasury", + "groups": [{"name": "in-year-spending"}], + "tags": tags, + "extras": [{"key": "latest_quarter", "value": latest_quarter}], + } + + query = {"fq": (f"+name:{dataset_fields['name']}")} + search_response = ckan.action.package_search(**query) + + if search_response["count"] == 0: + # create dataset and add resource + update_import_report(obj_to_update, "Creating a new dataset in CKAN") + response = create_dataset(dataset_fields) + add_resource(response, dataset_fields, userid, data_package_name) + else: + # update dataset + dataset_fields["id"] = search_response["results"][0]["id"] + update_import_report(obj_to_update, "Updating the dataset in CKAN") + response = update_dataset(dataset_fields) + + +def add_resource(response, dataset_fields, userid, data_package_name): + query = {"id": response["id"]} + + dataset_data = ckan.action.package_show(**query) + + if len(dataset_data["resources"]) == 0: + # add resource + should_add_resource = True + else: + should_add_resource = True + for resource in dataset_data["resources"]: + if resource["format"] == "OpenSpending API": + # resource is added - update it + should_add_resource = False + + if should_add_resource: + add_resource_to_dataset(dataset_fields, userid, data_package_name) + + +def create_dataset(dataset_fields): + response = ckan.action.package_create(**dataset_fields) + return response + + +def update_dataset(dataset_fields): + response = ckan.action.package_patch(**dataset_fields) + return response + + +def add_resource_to_dataset(dataset_fields, userid, data_package_name): + url = ( + f"{settings.OPENSPENDING_HOST}/api/3/cubes/{userid}:{data_package_name}/model/" + ) + resource_fields = { + "package_id": dataset_fields["name"], + "name": data_package_name, + "url": url, + "format": "OpenSpending API", + } + result = ckan.action.resource_create(**resource_fields) + + +def get_vocab_map(): + vocab_map = {} + for vocab in ckan.action.vocabulary_list(): + vocab_map[vocab["name"]] = vocab["id"] + + return vocab_map diff --git a/iym/temp_files/test_data.csv b/iym/temp_files/test_data.csv new file mode 100644 index 000000000..3599d627e --- /dev/null +++ b/iym/temp_files/test_data.csv @@ -0,0 +1,4 @@ +Vote No#,Department,Programme No#,Programme,Subprogramme No#,Subprogramme,econClass_L1,econClass_L2,econClass_L3,econClass_L4,econClass_L5,IYM_econLowestLevel,Item_Lowest_Level,Assets_Level_1,Assets_Level_2,Assets_Level_3,Assets_Level_4,Assets_Level_5,Assets_Level_6,Assets_Lowest_Level,Project_Level_1,Project_Level_2,Project_Level_3,Project_Level_4,Project_Level_5,Project_Level_6,Project_Level_7,Project_Level_8,Project_Level_9,Project_Level_10,Project_Level_11,Project_Lowest_Level,Responsibility_Level_2,Responsibility_Level_3,Responsibility_Level_4,Responsibility_Level_5,Responsibility_Level_6,Responsibility_Level_7,Responsibility_Level_8,Responsibility_Level_9,Responsibility_Level_10,Responsibility_Level_11,Responsibility_Level_12,Responsibility_Level_13,Responsibility_Level_14,Responsibility_Level_15,Responsibility_Lowest_Level,Fund_Level_1,Fund_Level_2,Fund_Level_3,Fund_Level_4,Fund_Level_5,Fund_Level_6,Fund_Level_7,Fund_Level_8,Fund_Lowest_Level,Infrastructure_Level_1,Infrastructure_Level_2,Infrastructure_Level_3,Infrastructure_Level_4,Infrastructure_Level_5,Infrastructure_Level_6,Infrastructure_Lowest_Level,Item_Level_1,Item_Level_2,Item_Level_3,Item_Level_4,Item_Level_5,Item_Level_6,Item_Level_7,Item_Level_8,Regional_ID_Level_1,Regional_ID_Level_2,Regional_ID_Level_3,Regional_ID_Level_4,Regional_ID_Level_5,Regional_ID_Level_6,Regional_ID_Level_7,Regional_ID_Level_8,Regional_ID_Lowest_Level,Budget,AdjustmentBudget,April,May,June,July,August,September,October,November,December,January,February,March,Q1,Q2,Q3,Q4,Financial_Year +5,Home Affairs,2,Citizen Affairs,5,Service Delivery to Provinces,Current,Current payments,Goods and services,Travel and subsistence,Travel and subsistence,Travel and subsistence,T&S DOM:FOOD&BEVER,NON-ASSETS RELATED,NON-ASSETS RELATED,,,,,NON-ASSETS RELATED,NO PROJECTS,NO PROJECTS,,,,,,,,,,NO PROJECTS,DEPARTMENT HOME AFFAIRS 17,DIRECTOR-GENERAL 17,CITIZEN AFFAIRS 17,EASTERN CAPE 17,EASTERN CAPE 17,DISTR MUN: JOE GQABI 17,MO MT FLETCHER 17,,,,,,,,MO MT FLETCHER 17,EXPENDITURE:VOTED,DEPARTMENTAL APPROPRIATION,VOTED FUNDS DISCRETIONARY,DEPARTMENTAL VOTE,VOTED FUNDS,,,,VOTED FUNDS,EXPENDITURE,NON INFRASTRUCTURE,NON INFRASTRUCTURE: CURRENT,NON INFRASTRUCTURE: CURRENT,NON INFRA/ST ALONE:CURRENT,,NON INFRA/ST ALONE:CURRENT,PAYMENTS,PAYMENTS,GOODS AND SERVICES,TRAVEL AND SUBSISTENCE,T&S DOMESTIC,T&S DOM:FOOD&BEVER,,,REGIONAL IDENTIFIER,NAT FUNCTION:WHOLE COUNTRY DOM,EASTERN CAPE 17,,,,,,EASTERN CAPE 17,0,0,0.8,0,1.28,1.08,0,2.76,1.46,0.92,0,3.16,0,3.08,2.08,3.84,2.38,6.24,2021 +5,Home Affairs,2,Citizen Affairs,5,Service Delivery to Provinces,Current,Current payments,Goods and services,Travel and subsistence,Travel and subsistence,Travel and subsistence,T&S DOM:ACCOMMODATION,NON-ASSETS RELATED,NON-ASSETS RELATED,,,,,NON-ASSETS RELATED,NO PROJECTS,NO PROJECTS,,,,,,,,,,NO PROJECTS,DEPARTMENT HOME AFFAIRS 17,DIRECTOR-GENERAL 17,CITIZEN AFFAIRS 17,GAUTENG 17,GAUTENG 17,DISTR MUN: SEBIDENG 17,LO VEREENIGING 17,,,,,,,,LO VEREENIGING 17,EXPENDITURE:VOTED,DEPARTMENTAL APPROPRIATION,VOTED FUNDS DISCRETIONARY,DEPARTMENTAL VOTE,VOTED FUNDS,,,,VOTED FUNDS,EXPENDITURE,NON INFRASTRUCTURE,NON INFRASTRUCTURE: CURRENT,NON INFRASTRUCTURE: CURRENT,NON INFRA/ST ALONE:CURRENT,,NON INFRA/ST ALONE:CURRENT,PAYMENTS,PAYMENTS,GOODS AND SERVICES,TRAVEL AND SUBSISTENCE,T&S DOMESTIC,T&S DOM:ACCOMMODATION,,,REGIONAL IDENTIFIER,NAT FUNCTION:WHOLE COUNTRY DOM,GAUTENG 17,,,,,,GAUTENG 17,0,0,0,0,2.46,0,0,0,3.2825,0,0,0,0,0,2.46,0,3.2825,0,2021 +5,Home Affairs,2,Citizen Affairs,5,Service Delivery to Provinces,Current,Current payments,Goods and services,Travel and subsistence,Travel and subsistence,Travel and subsistence,T&S DOM:FOOD&BEVER,NON-ASSETS RELATED,NON-ASSETS RELATED,,,,,NON-ASSETS RELATED,NO PROJECTS,NO PROJECTS,,,,,,,,,,NO PROJECTS,DEPARTMENT HOME AFFAIRS 17,DIRECTOR-GENERAL 17,CITIZEN AFFAIRS 17,FREE STATE 17,FREE STATE 17,DISTR MUN: LEJWELEPUTSWA 17,MO BULTFONTEIN 17,,,,,,,,MO BULTFONTEIN 17,EXPENDITURE:VOTED,DEPARTMENTAL APPROPRIATION,VOTED FUNDS DISCRETIONARY,DEPARTMENTAL VOTE,VOTED FUNDS,,,,VOTED FUNDS,EXPENDITURE,NON INFRASTRUCTURE,NON INFRASTRUCTURE: CURRENT,NON INFRASTRUCTURE: CURRENT,NON INFRA/ST ALONE:CURRENT,,NON INFRA/ST ALONE:CURRENT,PAYMENTS,PAYMENTS,GOODS AND SERVICES,TRAVEL AND SUBSISTENCE,T&S DOMESTIC,T&S DOM:FOOD&BEVER,,,REGIONAL IDENTIFIER,NAT FUNCTION:WHOLE COUNTRY DOM,FREE STATE 17,,,,,,FREE STATE 17,0,0,1.84,4.54,5.2,1.07,0,0,0.32,2.44,1.12,0.36,0,0.36,11.58,1.07,3.88,0.72,2021 diff --git a/iym/tests/static/test_data.zip b/iym/tests/static/test_data.zip new file mode 100644 index 000000000..cff197a1a Binary files /dev/null and b/iym/tests/static/test_data.zip differ diff --git a/iym/tests/test_iym_uploads.py b/iym/tests/test_iym_uploads.py new file mode 100644 index 000000000..fda54d8a7 --- /dev/null +++ b/iym/tests/test_iym_uploads.py @@ -0,0 +1,181 @@ +from django.test import TestCase +from django.contrib.auth.models import User +from django.core.files import File +from iym.models import IYMFileUpload +from budgetportal.models.government import FinancialYear +from django.conf import settings + +import os +import iym.tasks +import mock + +USERNAME = "testuser" +EMAIL = "testuser@domain.com" +PASSWORD = "12345" + + +class MockResponse: + def __init__(self, json_data, status_code, text): + self.json_data = json_data + self.status_code = status_code + self.text = text + + def json(self): + return self.json_data + + def raise_for_status(self): + return None + + +def mocked_requests_get(*args, **kwargs): + if f"{settings.OPENSPENDING_HOST}/user/authorize" in args[0]: + return MockResponse({"token": "test token"}, 200, "") + elif f"{settings.OPENSPENDING_HOST}/package/status?" in args[0]: + return MockResponse({"progress": 1, "status": "done"}, 200, "") + elif args[0] == "https://test-upload-url.com/data_package.json?": + return MockResponse({"name": "mock datapackage"}, 200, "") + + raise Exception(f"Unmocked GET request {args}") + + +def mocked_requests_put(*args, **kwargs): + if ( + f"{settings.OPENSPENDING_HOST}/package/upload?datapackage=https%3A%2F%2Ftest-upload-url.com%2Fdata_package.json&jwt=test+token" + in args[0] + ): + return MockResponse({"status": "0.0"}, 200, "initial status") + elif "https://test-upload-url.com/test_data.csv" in args[0]: + return MockResponse({"test": "file"}, 200, "") + elif args[0] == "https://test-upload-url.com/data_package.json?": + return MockResponse({"name": "mock datapackage"}, 200, "") + + raise Exception(f"Unmocked PUT request {args}") + + +def mocked_requests_post(*args, **kwargs): + if f"{settings.OPENSPENDING_HOST}/user/authenticate_api_key" in args[0]: + assert "x-api-key" in kwargs["headers"] + return MockResponse({"token": "fake.jwt.blah"}, 200, "") + elif f"{settings.OPENSPENDING_HOST}/datastore/" in args[0]: + return MockResponse( + { + "filedata": { + "test_data.csv": { + "md5": "LTP9Xtp/f8n2DrFWG2/h1g==", + "name": "test_data.csv", + "length": 2354098, + "upload_url": "https://test-upload-url.com/test_data.csv", + "upload_query": {}, + "type": "application/octet-stream", + }, + "data_package.json": { + "md5": "T57ewjs7A5m0f0fJfCW2Iw==", + "name": "data_package.json", + "length": 21510, + "upload_url": "https://test-upload-url.com/data_package.json", + "upload_query": {}, + "type": "application/octet-stream", + }, + } + }, + 200, + "", + ) + elif ( + f"{settings.OPENSPENDING_HOST}/package/upload?datapackage=https%3A%2F%2Ftest-upload-url.com%2Fdata_package.json&jwt=test+token" + in args[0] + ): + return MockResponse({"status": "0.0"}, 200, "initial status") + + raise Exception(f"Unmocked POST request {args}") + + +def mocked_wrong_requests_get(*args, **kwargs): + if f"{settings.OPENSPENDING_HOST}/user/authorize" in args[0]: + return MockResponse({"permissions": {}}, 200, "") + + raise Exception(f"Unmocked GET request {args}") + + +class IYMFileUploadTestCase(TestCase): + def setUp(self): + self.superuser = User.objects.create_user( + username=USERNAME, + password=PASSWORD, + is_staff=True, + is_superuser=True, + is_active=True, + ) + test_file_path = os.path.abspath(("iym/tests/static/test_data.zip")) + self.zip_file = File(open(test_file_path, "rb")) + + self.ckan_patch = mock.patch("iym.tasks.ckan") + self.CKANMockClass = self.ckan_patch.start() + self.CKANMockClass.action.package_search.return_value = {"count": 0} + self.CKANMockClass.action.package_create.return_value = {"id": "whatever"} + self.CKANMockClass.action.resource_create.return_value = {} + self.CKANMockClass.action.vocabulary_list.return_value = [ + {"name": "financial_years", "id": "a"}, + {"name": "spheres", "id": "b"}, + ] + self.addCleanup(self.ckan_patch.stop) + + def tearDown(self): + self.zip_file.close() + + @mock.patch("requests.get", side_effect=mocked_requests_get) + @mock.patch("requests.post", side_effect=mocked_requests_post) + @mock.patch("requests.put", side_effect=mocked_requests_put) + def test_uploading(self, mock_get, mock_post, mock_put): + financial_year = FinancialYear.objects.create(slug="2021-22") + test_element = IYMFileUpload.objects.create( + user=self.superuser, + file=self.zip_file, + financial_year=financial_year, + latest_quarter="Q1", + ) + + iym.tasks.process_uploaded_file(test_element.id) + test_element.refresh_from_db() + + import_report_lines = test_element.import_report.split("\n") + assert " - Cleaning CSV" in import_report_lines[0] + assert " - Getting authorisation for datastore" in import_report_lines[1] + assert " - Uploading CSV /tmp/" in import_report_lines[2] + assert " - Creating and uploading datapackage.json" in import_report_lines[3] + assert ( + " - Datapackage url: https://test-upload-url.com/data_package.json" + in import_report_lines[4] + ) + assert " - Starting import of uploaded datapackage." in import_report_lines[5] + assert " - Initial status: initial status" in import_report_lines[6] + assert ( + f" - Monitoring status until completion ({settings.OPENSPENDING_HOST}/package/status?datapackage=https%3A%2F%2Ftest-upload-url.com%2Fdata_package.json):" + in import_report_lines[7] + ) + assert test_element.status == "done" + + @mock.patch("requests.get", side_effect=mocked_wrong_requests_get) + @mock.patch("requests.post", side_effect=mocked_requests_post) + @mock.patch("requests.put", side_effect=mocked_requests_put) + def test_uploading_with_wrong_token(self, mock_get, mock_post, mock_put): + financial_year = FinancialYear.objects.create(slug="2021-22") + test_element = IYMFileUpload.objects.create( + user=self.superuser, + file=self.zip_file, + financial_year=financial_year, + latest_quarter="Q1", + ) + + iym.tasks.process_uploaded_file(test_element.id) + test_element.refresh_from_db() + + import_report_lines = test_element.import_report.split("\n") + assert " - Cleaning CSV" in import_report_lines[0] + assert ( + " - Getting authorisation for datastore" in import_report_lines[1] + ), import_report_lines + assert ( + " - Authorization with OpenSpending failed." in import_report_lines[2] + ), import_report_lines + assert test_element.status == "fail" diff --git a/iym/urls.py b/iym/urls.py new file mode 100644 index 000000000..5342a8afd --- /dev/null +++ b/iym/urls.py @@ -0,0 +1,10 @@ +from django.contrib import admin +from django.conf.urls import url +from rest_framework.routers import DefaultRouter +from django.urls import path, include +from performance import views + +urlpatterns = [ + # IYM + path("", views.performance_tabular_view, name="iym"), +] diff --git a/iym/views.py b/iym/views.py new file mode 100644 index 000000000..e69de29bb diff --git a/package.json b/package.json index 92b002f50..d64d87053 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "build:webapp": "cd ./packages/webapp && node ./scripts/build", "build:root": "webpack --display errors-only --progress -p", "build:dev": "webpack --progress --watch --mode=development", - "build": "yarn build:webapp && yarn build:root" + "build": "yarn build:webapp && yarn build:root", + "start-test": "react-app-rewired --progress --watch --mode=development" }, "devDependencies": { "app-root-dir": "^1.0.2", @@ -66,7 +67,6 @@ "opn": "^5.4.0", "postcss-loader": "^2.1.6", "postcss-normalize": "^4.0.0", - "preact-render-to-string": "^3.8.2", "prettier": "^1.17.1", "prop-types": "^15.6.1", "sass-loader": "^6.0.6", @@ -97,17 +97,11 @@ "presets": [ "env", "stage-0" - ], - "plugins": [ - [ - "transform-react-jsx", - { - "pragma": "h" - } - ] ] }, "dependencies": { + "@material-ui/core": "^4.12.4", + "@material-ui/icons": "^4.11.3", "array.from": "^1.0.3", "array.prototype.every": "^1.0.0", "array.prototype.findindex": "^2.0.2", @@ -116,20 +110,23 @@ "canvg-browser": "^1.0.0", "chart.js": "^2.7.3", "color-string": "^1.5.3", + "d3-scale": "^2.2.2", + "d3-selection": "^1.3.0", "jquery": "^3.3.1", "js-yaml": "^3.11.0", "lodash": "^4.17.11", + "lodash.debounce": "^4.0.8", "lunr": "^2.3.5", "mini-css-extract-plugin": "^0.9.0", "object.assign": "^4.1.0", - "preact": "^8.3.1", - "preact-css-transition-group": "^1.3.0", - "preact-transition-group": "^1.1.1", "promise-polyfill": "^6.0.2", "pym.js": "^1.3.2", "query-string": "^5.1.1", + "react": "^16.8.1", + "react-dom": "^16.8.1", "react-ga": "^2.5.3", "react-html-connector": "^0.2.6", + "react-lines-ellipsis": "^0.15.3", "redux": "^4.0.1", "save-svg-as-png": "^1.4.6", "url-search-params-polyfill": "^8.1.0", diff --git a/performance/__init__.py b/performance/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/performance/admin.py b/performance/admin.py new file mode 100644 index 000000000..d7f0fc01a --- /dev/null +++ b/performance/admin.py @@ -0,0 +1,531 @@ +from django.contrib import admin +from io import StringIO +from performance import models +from django_q.tasks import async_task, fetch + +from frictionless import validate + +import os +import csv +import budgetportal + +VALID_REPORT_TYPES = [ + "Provincial Institutions Oversight Performance Report", + "National Institutions Oversight Performance Report", +] + + +def generate_import_report( + report_type_validated, frictionless_report, not_matching_departments +): + report = "" + if not report_type_validated: + report += "Report type must be for one of " + os.linesep + for report_type in VALID_REPORT_TYPES: + report += f"* {report_type} {os.linesep}" + + if frictionless_report: + if not frictionless_report.valid: + for error in frictionless_report.tasks[0].errors: + report += f"* {error.message} {os.linesep}" + + if len(not_matching_departments) > 0: + report += "Department names that could not be matched on import : " + os.linesep + for department in not_matching_departments: + report += f"* {department} {os.linesep}" + + return report + + +def save_imported_indicators(obj_id): + # read file + obj_to_update = models.EQPRSFileUpload.objects.get(id=obj_id) + full_text = obj_to_update.file.read().decode("utf-8") + + # validate report type + report_type_validated = validate_report_type(full_text, obj_id) + if not report_type_validated: + return + + # clean the csv & extract data + financial_year = get_financial_year(full_text) + sphere = get_sphere(full_text) + clean_text = full_text.split("\n", 3)[3] + f = StringIO(clean_text) + reader = csv.DictReader(f) + parsed_data = list(reader) + + # find the objects + department_government_pairs = set( + [(x["Institution"], x["Programme"]) for x in parsed_data] + ) # Programme column in CSV is mislabeled + num_imported = 0 + total_record_count = len(parsed_data) + not_matching_departments = set() + + for department, government_name in department_government_pairs: + if government_name == "National": + government_name = "South Africa" + + if department.startswith(f"{government_name}: "): + department = department.replace(f"{government_name}: ", "") + + # clear by department + models.Indicator.objects.filter( + department__name=department, + department__government__name=government_name, + department__government__sphere__name=sphere, + department__government__sphere__financial_year__slug=financial_year, + ).delete() + + # clear by alias + alias_obj = models.EQPRSDepartmentAlias.objects.filter( + alias=department, + department__government__name=government_name, + department__government__sphere__name=sphere, + department__government__sphere__financial_year__slug=financial_year, + ).first() + if alias_obj: + models.Indicator.objects.filter(department=alias_obj.department).delete() + + # create new indicators + for indicator_data in parsed_data: + frequency = indicator_data["Frequency"] + government_name = indicator_data["Programme"] + if government_name == "National": + government_name = "South Africa" + department_name = indicator_data["Institution"] + if department_name.startswith(f"{government_name}: "): + department_name = department_name.replace(f"{government_name}: ", "") + + department_matches = models.Department.objects.filter( + name=department_name, + government__name=government_name, + government__sphere__name=sphere, + government__sphere__financial_year__slug=financial_year, + ) + + assert department_matches.count() <= 1 + department_obj = department_matches.first() + + if not department_obj: + alias_matches = models.EQPRSDepartmentAlias.objects.filter( + alias=department_name, + department__government__name=government_name, + department__government__sphere__name=sphere, + department__government__sphere__financial_year__slug=financial_year, + ) + assert alias_matches.count() <= 1 + if len(alias_matches) > 0: + department_obj = alias_matches.first().department + + if department_obj: + models.Indicator.objects.create( + indicator_name=indicator_data["Indicator"], + department=department_obj, + source_id=obj_id, + q1_target=indicator_data["Target_Q1"], + q1_actual_output=indicator_data["ActualOutput_Q1"], + q1_deviation_reason=indicator_data["ReasonforDeviation_Q1"], + q1_corrective_action=indicator_data["CorrectiveAction_Q1"], + q1_national_comments=indicator_data.get("National_Q1", ""), + q1_otp_comments=indicator_data.get("OTP_Q1", ""), + q1_dpme_coordinator_comments=indicator_data.get("OTP_Q1", ""), + q1_treasury_comments=indicator_data.get("National_Q1", ""), + q2_target=indicator_data["Target_Q2"], + q2_actual_output=indicator_data["ActualOutput_Q2"], + q2_deviation_reason=indicator_data["ReasonforDeviation_Q2"], + q2_corrective_action=indicator_data["CorrectiveAction_Q2"], + q2_national_comments=indicator_data.get("National_Q2", ""), + q2_otp_comments=indicator_data.get("OTP_Q2", ""), + q2_dpme_coordinator_comments=indicator_data.get("OTP_Q2", ""), + q2_treasury_comments=indicator_data.get("National_Q2", ""), + q3_target=indicator_data["Target_Q3"], + q3_actual_output=indicator_data["ActualOutput_Q3"], + q3_deviation_reason=indicator_data["ReasonforDeviation_Q3"], + q3_corrective_action=indicator_data["CorrectiveAction_Q3"], + q3_national_comments=indicator_data.get("National_Q3", ""), + q3_otp_comments=indicator_data.get("OTP_Q3", ""), + q3_dpme_coordinator_comments=indicator_data.get("OTP_Q3", ""), + q3_treasury_comments=indicator_data.get("National_Q3", ""), + q4_target=indicator_data["Target_Q4"], + q4_actual_output=indicator_data["ActualOutput_Q4"], + q4_deviation_reason=indicator_data["ReasonforDeviation_Q4"], + q4_corrective_action=indicator_data["CorrectiveAction_Q4"], + q4_national_comments=indicator_data.get("National_Q4", ""), + q4_otp_comments=indicator_data.get("OTP_Q4", ""), + q4_dpme_coordinator_comments=indicator_data.get("OTP_Q4", ""), + q4_treasury_comments=indicator_data.get("National_Q4", ""), + annual_target=indicator_data["AnnualTarget_Summary2"], + annual_aggregate_output="", + annual_pre_audit_output=indicator_data["PrelimaryAudited_Summary2"], + annual_deviation_reason=indicator_data["ReasonforDeviation_Summary"], + annual_corrective_action=indicator_data["CorrectiveAction_Summary"], + annual_otp_comments=indicator_data.get("OTP_Summary", ""), + annual_national_comments=indicator_data.get("National_Summary", ""), + annual_dpme_coordinator_comments=indicator_data.get("OTP_Summary", ""), + annual_treasury_comments=indicator_data.get("National_Summary", ""), + annual_audited_output=indicator_data["ValidatedAudited_Summary2"], + sector=indicator_data["Sector"], + programme_name=indicator_data[ + "SubProgramme" + ], # SubProgramme column in CSV is mislabeled + subprogramme_name=indicator_data[ + "Location" + ], # Location column in CSV is mislabeled + frequency=[i[0] for i in models.FREQUENCIES if i[1] == frequency][0], + type=indicator_data["Type"], + subtype=indicator_data["SubType"], + mtsf_outcome=indicator_data["Outcome"], + cluster=indicator_data["Cluster"], + uid=indicator_data["UID"], + ) + num_imported = num_imported + 1 + else: + not_matching_departments.add(department_name) + + # update object + obj_to_update.num_imported = num_imported + obj_to_update.num_not_imported = total_record_count - num_imported + obj_to_update.import_report = generate_import_report( + True, None, not_matching_departments + ) + obj_to_update.save() + + +def validate_report_type(full_text, obj_id): + validated = False + for report_type in VALID_REPORT_TYPES: + validated = validated or (report_type in full_text) + + if not validated: + obj_to_update = models.EQPRSFileUpload.objects.get(id=obj_id) + obj_to_update.num_imported = None + obj_to_update.num_not_imported = None + obj_to_update.import_report = generate_import_report(False, None, []) + obj_to_update.save() + + return validated + + +def validate_frictionless(data, obj_id): + report = validate(data) + validated = report.valid + if not validated: + obj_to_update = models.EQPRSFileUpload.objects.get(id=obj_id) + obj_to_update.num_imported = None + obj_to_update.num_not_imported = None + obj_to_update.import_report = generate_import_report(True, report, []) + obj_to_update.save() + + return validated + + +def get_financial_year(full_text): + financial_year = full_text.split("\n", 1)[1] + financial_year = financial_year.replace("QPR for FY ", "") + financial_year = financial_year[: financial_year.index(" ")] + financial_year = financial_year.strip() + + return financial_year + + +def get_sphere(full_text): + line = full_text.split("\n", 2)[1] + if "Provincial" in line: + sphere = "Provincial" + else: + sphere = "National" + + return sphere + + +class EQPRSFileUploadAdmin(admin.ModelAdmin): + exclude = ("num_imported", "import_report", "num_not_imported") + readonly_fields = ( + "num_imported", + "import_report", + "num_not_imported", + "user", + ) + list_display = ( + "created_at", + "user", + "num_imported", + "num_not_imported", + "processing_completed", + "updated_at", + ) + fieldsets = ( + ( + "", + { + "fields": ( + "user", + "file", + "import_report", + "num_imported", + "num_not_imported", + ) + }, + ), + ) + + def get_readonly_fields(self, request, obj=None): + if obj: # editing an existing object + return ( + "user", + "file", + ) + self.readonly_fields + return self.readonly_fields + + def render_change_form( + self, request, context, add=False, change=False, form_url="", obj=None + ): + response = super(EQPRSFileUploadAdmin, self).render_change_form( + request, context, add, change, form_url, obj + ) + response.context_data["title"] = ( + "EQPRS file upload" + if response.context_data["object_id"] + else "Upload EQPRS file" + ) + return response + + def has_change_permission(self, request, obj=None): + super(EQPRSFileUploadAdmin, self).has_change_permission(request, obj) + + def has_delete_permission(self, request, obj=None): + super(EQPRSFileUploadAdmin, self).has_delete_permission(request, obj) + + def save_model(self, request, obj, form, change): + if not obj.pk: + obj.user = request.user + super().save_model(request, obj, form, change) + # It looks like the task isn't saved synchronously, so we can't set the + # task as a related object synchronously. We have to fetch it by its ID + # when we want to see if it's available yet. + + obj.task_id = async_task(func=save_imported_indicators, obj_id=obj.id) + obj.save() + + def processing_completed(self, obj): + task = fetch(obj.task_id) + if task: + return task.success + + processing_completed.boolean = True + processing_completed.short_description = "Processing completed" + + +class IndicatorAdmin(admin.ModelAdmin): + list_display = ( + "indicator_name", + "financial_year", + "sphere", + "government", + "get_department", + "created_at", + ) + list_filter = ( + "department__government__sphere__financial_year__slug", + "department__government__sphere__name", + "department__government__name", + "department__name", + ) + search_fields = ( + "department__government__sphere__financial_year__slug", + "department__government__sphere__name", + "department__government__name", + "department__name", + "indicator_name", + ) + + readonly_fields = ( + "source", + "indicator_name", + "department", + "q1_target", + "q1_actual_output", + "q1_deviation_reason", + "q1_corrective_action", + "q1_national_comments", + "q1_otp_comments", + "q1_dpme_coordinator_comments", + "q1_treasury_comments", + "q2_target", + "q2_actual_output", + "q2_deviation_reason", + "q2_corrective_action", + "q2_national_comments", + "q2_otp_comments", + "q2_dpme_coordinator_comments", + "q2_treasury_comments", + "q3_target", + "q3_actual_output", + "q3_deviation_reason", + "q3_corrective_action", + "q3_national_comments", + "q3_otp_comments", + "q3_dpme_coordinator_comments", + "q3_treasury_comments", + "q4_target", + "q4_actual_output", + "q4_deviation_reason", + "q4_corrective_action", + "q4_national_comments", + "q4_otp_comments", + "q4_dpme_coordinator_comments", + "q4_treasury_comments", + "annual_target", + "annual_aggregate_output", + "annual_pre_audit_output", + "annual_deviation_reason", + "annual_corrective_action", + "annual_otp_comments", + "annual_national_comments", + "annual_dpme_coordinator_comments", + "annual_treasury_comments", + "annual_audited_output", + "sector", + "programme_name", + "subprogramme_name", + "frequency", + "type", + "subtype", + "mtsf_outcome", + "cluster", + "uid", + ) + + fieldsets = ( + ( + None, + { + "fields": ( + "source", + "indicator_name", + "department", + "sector", + "programme_name", + "subprogramme_name", + "frequency", + "type", + "subtype", + "mtsf_outcome", + "cluster", + ), + }, + ), + ( + "Quarter 1", + { + "fields": ( + "q1_target", + "q1_actual_output", + "q1_deviation_reason", + "q1_corrective_action", + "q1_national_comments", + "q1_otp_comments", + "q1_dpme_coordinator_comments", + "q1_treasury_comments", + ), + }, + ), + ( + "Quarter 2", + { + "fields": ( + "q2_target", + "q2_actual_output", + "q2_deviation_reason", + "q2_corrective_action", + "q2_national_comments", + "q2_otp_comments", + "q2_dpme_coordinator_comments", + "q2_treasury_comments", + ), + }, + ), + ( + "Quarter 3", + { + "fields": ( + "q3_target", + "q3_actual_output", + "q3_deviation_reason", + "q3_corrective_action", + "q3_national_comments", + "q3_otp_comments", + "q3_dpme_coordinator_comments", + "q3_treasury_comments", + ), + }, + ), + ( + "Quarter 4", + { + "fields": ( + "q4_target", + "q4_actual_output", + "q4_deviation_reason", + "q4_corrective_action", + "q4_national_comments", + "q4_otp_comments", + "q4_dpme_coordinator_comments", + "q4_treasury_comments", + ), + }, + ), + ( + "Full year", + { + "fields": ( + "annual_target", + "annual_aggregate_output", + "annual_pre_audit_output", + "annual_deviation_reason", + "annual_corrective_action", + "annual_otp_comments", + "annual_national_comments", + "annual_dpme_coordinator_comments", + "annual_treasury_comments", + "annual_audited_output", + ), + }, + ), + ) + + def get_department(self, obj): + return obj.department.name + + get_department.short_description = "Department" + + def government(self, obj): + return obj.department.government.name + + def sphere(self, obj): + return obj.department.government.sphere.name + + def financial_year(self, obj): + return obj.department.government.sphere.financial_year.slug + + +class EQPRSDepartmentAliasAdmin(admin.ModelAdmin): + list_display = ("alias", "department") + list_filter = ( + "department__government__sphere__financial_year__slug", + "department__government__sphere__name", + "department__government__name", + ) + search_fields = ( + "department__name", + "alias", + ) + + autocomplete_fields = ("department",) + + +admin.site.register(models.EQPRSFileUpload, EQPRSFileUploadAdmin) +admin.site.register(models.Indicator, IndicatorAdmin) +admin.site.register(models.EQPRSDepartmentAlias, EQPRSDepartmentAliasAdmin) diff --git a/performance/apps.py b/performance/apps.py new file mode 100644 index 000000000..35b57e429 --- /dev/null +++ b/performance/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class PerformanceConfig(AppConfig): + name = "performance" diff --git a/performance/fixtures/test_api.json b/performance/fixtures/test_api.json new file mode 100644 index 000000000..3deefa8d5 --- /dev/null +++ b/performance/fixtures/test_api.json @@ -0,0 +1,180 @@ +[{ + "model": "auth.user", + "pk": 1, + "fields": { + "password": "pbkdf2_sha256$36000$0lM3TbZBY94V$xirXvIgKYBggcH0tMDjE6GTjoS6KYuim4sl7UbpYsrA=", + "last_login": "2019-04-16T05:45:02Z", + "is_superuser": true, + "username": "admin@localhost", + "first_name": "", + "last_name": "", + "email": "admin@localhost", + "is_staff": true, + "is_active": true, + "date_joined": "2019-04-15T20:06:18Z", + "groups": [], + "user_permissions": [] + } +}, +{ + "model": "budgetportal.financialyear", + "pk": 1, + "fields": { + "slug": "2015-16", + "published": false + } +}, +{ + "model": "budgetportal.sphere", + "pk": 1, + "fields": { + "name": "National", + "slug": "national", + "financial_year": 1 + } +}, +{ + "model": "budgetportal.government", + "pk": 1, + "fields": { + "sphere": 1, + "name": "South Africa", + "slug": "south-africa" + } +}, +{ + "model": "budgetportal.department", + "pk": 1, + "fields": { + "government": 1, + "name": "Agriculture Forestry and Fisheries", + "vote_number": 1, + "intro": " ", + "slug": "agriculture-forestry-and-fisheries" + } +}, + { + "pk": 1, + "model": "performance.eqprsfileupload", + "fields": { + "user": 1, + "task_id": "43553e179fab466aaeb967dda549bf7c", + "import_report": "Everything was good.\n\nJust look at the data!", + "num_imported": 123, + "num_not_imported": 0 + } + }, + + { + "pk": 1, + "model": "django_q.task", + "fields": { + "id": "43553e179fab466aaeb967dda549bf7c", + "func": "somefunc", + "started": "2022-11-29T14:16:55Z", + "stopped": "2022-11-29T14:16:56Z" + } + }, + + { + "pk": 1, + "model": "performance.indicator", + "fields": { + "sector": "", + "department": 1, + "programme_name": "Programme 1: Administration", + "subprogramme_name": "Financial Management Services", + "frequency": "Annually", + "indicator_name": "1.1.1 Unqualified audit opinion", + "type": "Non-Standardized", + "subtype": "Not Applicable", + "mtsf_outcome": "Priority 1: A Capable, Ethical and Developmental State", + "cluster": "Governance and Administration cluster", + "q1_target": "", + "q1_actual_output": "", + "q1_deviation_reason": "", + "q1_corrective_action": "", + "q1_otp_comments": "", + "q1_national_comments": "", + "q2_target": "", + "q2_actual_output": "", + "q2_deviation_reason": "", + "q2_corrective_action": "", + "q2_otp_comments": "", + "q2_national_comments": "", + "q3_target": "", + "q3_actual_output": "", + "q3_deviation_reason": "", + "q3_corrective_action": "", + "q3_otp_comments": "", + "q3_national_comments": "", + "q4_target": "", + "q4_actual_output": "", + "q4_deviation_reason": "", + "q4_corrective_action": "", + "q4_otp_comments": "", + "q4_national_comments": "", + "annual_target": "Unqualified audit opinion on 2020/21 annual financial statements", + "annual_aggregate_output": "-", + "annual_pre_audit_output": "Unqualified audit opinion on 2020/21 annual financial statements", + "annual_deviation_reason": "", + "annual_corrective_action": "", + "annual_otp_comments": "", + "annual_national_comments": "", + "annual_audited_output": "Unqualified audit opinion on 2020/21 annual financial statements", + "uid": 50, + "source_id": 1 + } + }, + { + "pk": 2, + "model": "performance.indicator", + "fields": { + "sector": "", + "department": 1, + "programme_name": "Programme 1: Administration", + "subprogramme_name": "Financial Management Services", + "frequency": "Quarterly", + "indicator_name": "1.2.1 Percentage of valid invoices paid within 30 days upon receipt by the department", + "type": "Non-Standardized", + "subtype": "Not Applicable", + "mtsf_outcome": "Priority 1: A Capable, Ethical and Developmental State", + "cluster": "Governance and Administration cluster", + "q1_target": "100 %", + "q1_actual_output": "97 %", + "q1_deviation_reason": "The variance was caused by the re-allocation of resources to the finalisation of the year-end closure process and audit\n", + "q1_corrective_action": "The closure of the year (2020/21) has now been finalised; thus it should not be a factor in hampering the processing of invoices going forth\n", + "q1_otp_comments": "", + "q1_national_comments": "", + "q2_target": "100 %", + "q2_actual_output": "97% %", + "q2_deviation_reason": "Constant closure of offices due to exposure to COVID- 19 cases. Outstanding Official, Flight, Transport and Accommodation Request (OFTARs). Entities not registered on Basic Accounting System (BAS). Service Providers changing banking details after services are rendered. Officials using photocopier machines after the contract has expired resulting in exposit-facto\n", + "q2_corrective_action": "Constant awareness to both officials and service providers the importance of adhering to Treasury Regulations and ensuring documents are complaint at all times\n", + "q2_otp_comments": "", + "q2_national_comments": "", + "q3_target": "100 %", + "q3_actual_output": "96% %", + "q3_deviation_reason": "Service providers not registered on Basic Accounting System (BAS). Payments rejected on safety web. Awaiting verification/approval\n", + "q3_corrective_action": "Continuous consultation with relevant directorates\n", + "q3_otp_comments": "", + "q3_national_comments": "", + "q4_target": "100 %", + "q4_actual_output": "95% %", + "q4_deviation_reason": "Payment submitted to finance after 30 days. Supplier banking detail on invoice not the same as on the system", + "q4_corrective_action": "Weekly monitoring of payments at hand\n", + "q4_otp_comments": "", + "q4_national_comments": "", + "annual_target": "100 %", + "annual_aggregate_output": "0 %", + "annual_pre_audit_output": "96", + "annual_deviation_reason": "Supplier banking detail on invoice not the same as on the system. This caused delays in payment as the process need to be followed in verifying the correct details. ", + "annual_corrective_action": "Weekly monitoring of payments at hand\n", + "annual_otp_comments": "", + "annual_national_comments": "", + "annual_audited_output": "", + "uid": 50, + "source_id": 1 + } + } + + ] \ No newline at end of file diff --git a/performance/fixtures/test_repetitive_data.json b/performance/fixtures/test_repetitive_data.json new file mode 100644 index 000000000..3e6187730 --- /dev/null +++ b/performance/fixtures/test_repetitive_data.json @@ -0,0 +1,180 @@ +[{ + "model": "auth.user", + "pk": 1, + "fields": { + "password": "pbkdf2_sha256$36000$0lM3TbZBY94V$xirXvIgKYBggcH0tMDjE6GTjoS6KYuim4sl7UbpYsrA=", + "last_login": "2019-04-16T05:45:02Z", + "is_superuser": true, + "username": "admin@localhost", + "first_name": "", + "last_name": "", + "email": "admin@localhost", + "is_staff": true, + "is_active": true, + "date_joined": "2019-04-15T20:06:18Z", + "groups": [], + "user_permissions": [] + } +}, +{ + "model": "budgetportal.financialyear", + "pk": 1, + "fields": { + "slug": "2015-16", + "published": false + } +}, +{ + "model": "budgetportal.sphere", + "pk": 1, + "fields": { + "name": "National", + "slug": "national", + "financial_year": 1 + } +}, +{ + "model": "budgetportal.government", + "pk": 1, + "fields": { + "sphere": 1, + "name": "South Africa", + "slug": "south-africa" + } +}, +{ + "model": "budgetportal.department", + "pk": 1, + "fields": { + "government": 1, + "name": "Agriculture Forestry and Fisheries", + "vote_number": 1, + "intro": " ", + "slug": "agriculture-forestry-and-fisheries" + } +}, + { + "pk": 1, + "model": "performance.eqprsfileupload", + "fields": { + "user": 1, + "task_id": "43553e179fab466aaeb967dda549bf7c", + "import_report": "Everything was good.\n\nJust look at the data!", + "num_imported": 123, + "num_not_imported": 0 + } + }, + + { + "pk": 1, + "model": "django_q.task", + "fields": { + "id": "43553e179fab466aaeb967dda549bf7c", + "func": "somefunc", + "started": "2022-11-29T14:16:55Z", + "stopped": "2022-11-29T14:16:56Z" + } + }, + + { + "pk": 1, + "model": "performance.indicator", + "fields": { + "sector": "", + "department": 1, + "programme_name": "Programme 1: Administration", + "subprogramme_name": "Financial Management Services", + "frequency": "Annually", + "indicator_name": "1.1.1 Unqualified audit opinion", + "type": "Non-Standardized", + "subtype": "Not Applicable", + "mtsf_outcome": "Priority 1: A Capable, Ethical and Developmental State", + "cluster": "Governance and Administration cluster", + "q1_target": "", + "q1_actual_output": "", + "q1_deviation_reason": "", + "q1_corrective_action": "", + "q1_otp_comments": "", + "q1_national_comments": "", + "q2_target": "", + "q2_actual_output": "", + "q2_deviation_reason": "", + "q2_corrective_action": "", + "q2_otp_comments": "", + "q2_national_comments": "", + "q3_target": "", + "q3_actual_output": "", + "q3_deviation_reason": "", + "q3_corrective_action": "", + "q3_otp_comments": "", + "q3_national_comments": "", + "q4_target": "", + "q4_actual_output": "", + "q4_deviation_reason": "", + "q4_corrective_action": "", + "q4_otp_comments": "", + "q4_national_comments": "", + "annual_target": "Unqualified audit opinion on 2020/21 annual financial statements", + "annual_aggregate_output": "-", + "annual_pre_audit_output": "Unqualified audit opinion on 2020/21 annual financial statements", + "annual_deviation_reason": "", + "annual_corrective_action": "", + "annual_otp_comments": "", + "annual_national_comments": "", + "annual_audited_output": "Unqualified audit opinion on 2020/21 annual financial statements", + "uid": 50, + "source_id": 1 + } + }, + { + "pk": 2, + "model": "performance.indicator", + "fields": { + "sector": "", + "department": 1, + "programme_name": "Programme 1: Administration", + "subprogramme_name": "Financial Management Services", + "frequency": "Annually", + "indicator_name": "1.1.1 Unqualified audit opinion", + "type": "Non-Standardized", + "subtype": "Not Applicable", + "mtsf_outcome": "Priority 1: A Capable, Ethical and Developmental State", + "cluster": "Governance and Administration cluster", + "q1_target": "", + "q1_actual_output": "", + "q1_deviation_reason": "", + "q1_corrective_action": "", + "q1_otp_comments": "", + "q1_national_comments": "", + "q2_target": "", + "q2_actual_output": "", + "q2_deviation_reason": "", + "q2_corrective_action": "", + "q2_otp_comments": "", + "q2_national_comments": "", + "q3_target": "", + "q3_actual_output": "", + "q3_deviation_reason": "", + "q3_corrective_action": "", + "q3_otp_comments": "", + "q3_national_comments": "", + "q4_target": "", + "q4_actual_output": "", + "q4_deviation_reason": "", + "q4_corrective_action": "", + "q4_otp_comments": "", + "q4_national_comments": "", + "annual_target": "Unqualified audit opinion on 2020/21 annual financial statements", + "annual_aggregate_output": "-", + "annual_pre_audit_output": "Unqualified audit opinion on 2020/21 annual financial statements", + "annual_deviation_reason": "", + "annual_corrective_action": "", + "annual_otp_comments": "", + "annual_national_comments": "", + "annual_audited_output": "Unqualified audit opinion on 2020/21 annual financial statements", + "uid": 51, + "source_id": 1 + } + } + + ] \ No newline at end of file diff --git a/performance/migrations/0001_initial.py b/performance/migrations/0001_initial.py new file mode 100644 index 000000000..15eaf5182 --- /dev/null +++ b/performance/migrations/0001_initial.py @@ -0,0 +1,139 @@ +# Generated by Django 2.2.20 on 2022-11-29 07:50 + +from django.conf import settings +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion +import performance.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("budgetportal", "0064_auto_20200728_1054"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("django_q", "0009_auto_20171009_0915"), + ] + + operations = [ + migrations.CreateModel( + name="EQPRSFileUpload", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "file", + models.FileField(upload_to=performance.models.eqprs_file_path), + ), + ( + "file_validation_report", + django.contrib.postgres.fields.jsonb.JSONField(), + ), + ("import_report", models.TextField()), + ("num_imported", models.IntegerField(null=True)), + ("num_not_imported", models.IntegerField(null=True)), + ( + "task", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="django_q.Task" + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.DO_NOTHING, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + migrations.CreateModel( + name="Indicator", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("indicator_name", models.TextField()), + ("q1_target", models.TextField(blank=True)), + ("q1_actual_outcome", models.TextField(blank=True)), + ("q1_deviation_reason", models.TextField(blank=True)), + ("q1_corrective_action", models.TextField(blank=True)), + ("q1_national_comments", models.TextField(blank=True)), + ("q1_otp_comments", models.TextField(blank=True)), + ("q1_dpme_coordinator_comments", models.TextField(blank=True)), + ("q1_treasury_comments", models.TextField(blank=True)), + ("q2_target", models.TextField(blank=True)), + ("q2_actual_outcome", models.TextField(blank=True)), + ("q2_deviation_reason", models.TextField(blank=True)), + ("q2_corrective_action", models.TextField(blank=True)), + ("q2_national_comments", models.TextField(blank=True)), + ("q2_otp_comments", models.TextField(blank=True)), + ("q2_dpme_coordinator_comments", models.TextField(blank=True)), + ("q2_treasury_comments", models.TextField(blank=True)), + ("q3_target", models.TextField(blank=True)), + ("q3_actual_outcome", models.TextField(blank=True)), + ("q3_deviation_reason", models.TextField(blank=True)), + ("q3_corrective_action", models.TextField(blank=True)), + ("q3_national_comments", models.TextField(blank=True)), + ("q3_otp_comments", models.TextField(blank=True)), + ("q3_dpme_coordinator_comments", models.TextField(blank=True)), + ("q3_treasury_comments", models.TextField(blank=True)), + ("q4_target", models.TextField(blank=True)), + ("q4_actual_outcome", models.TextField(blank=True)), + ("q4_deviation_reason", models.TextField(blank=True)), + ("q4_corrective_action", models.TextField(blank=True)), + ("q4_national_comments", models.TextField(blank=True)), + ("q4_otp_comments", models.TextField(blank=True)), + ("q4_dpme_coordinator_comments", models.TextField(blank=True)), + ("q4_treasury_comments", models.TextField(blank=True)), + ("sector", models.TextField(blank=True)), + ("programme_name", models.TextField(blank=True)), + ("subprogramme_name", models.TextField(blank=True)), + ( + "frequency", + models.CharField( + choices=[("annually", "Annually"), ("quarterly", "Quarterly")], + max_length=9, + ), + ), + ("type", models.TextField(blank=True)), + ("subtype", models.TextField(blank=True)), + ("mtsf_outcome", models.TextField(blank=True)), + ("uid", models.TextField(blank=True)), + ( + "department", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="indicator_values", + to="budgetportal.Department", + ), + ), + ( + "source", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="indicator_values", + to="performance.EQPRSFileUpload", + ), + ), + ], + options={ + "unique_together": {("department", "indicator_name")}, + }, + ), + ] diff --git a/performance/migrations/0002_auto_20221129_0819.py b/performance/migrations/0002_auto_20221129_0819.py new file mode 100644 index 000000000..2d9877531 --- /dev/null +++ b/performance/migrations/0002_auto_20221129_0819.py @@ -0,0 +1,88 @@ +# Generated by Django 2.2.20 on 2022-11-29 08:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0001_initial"), + ] + + operations = [ + migrations.RenameField( + model_name="indicator", + old_name="q1_actual_outcome", + new_name="q1_actual_output", + ), + migrations.RenameField( + model_name="indicator", + old_name="q2_actual_outcome", + new_name="q2_actual_output", + ), + migrations.RenameField( + model_name="indicator", + old_name="q3_actual_outcome", + new_name="q3_actual_output", + ), + migrations.RenameField( + model_name="indicator", + old_name="q4_actual_outcome", + new_name="q4_actual_output", + ), + migrations.AddField( + model_name="indicator", + name="annual_aggregate_output", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="indicator", + name="annual_audited_output", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="indicator", + name="annual_corrective_action", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="indicator", + name="annual_deviation_reason", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="indicator", + name="annual_dpme_coordincator_comments", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="indicator", + name="annual_national_comments", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="indicator", + name="annual_otp_comments", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="indicator", + name="annual_pre_audit_output", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="indicator", + name="annual_target", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="indicator", + name="annual_treasury_comments", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="indicator", + name="cluster", + field=models.TextField(blank=True), + ), + ] diff --git a/performance/migrations/0003_remove_eqprsfileupload_file_validation_report.py b/performance/migrations/0003_remove_eqprsfileupload_file_validation_report.py new file mode 100644 index 000000000..33f6f1681 --- /dev/null +++ b/performance/migrations/0003_remove_eqprsfileupload_file_validation_report.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.20 on 2022-11-29 12:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0002_auto_20221129_0819"), + ] + + operations = [ + migrations.RemoveField( + model_name="eqprsfileupload", + name="file_validation_report", + ), + ] diff --git a/performance/migrations/0004_auto_20221130_2138.py b/performance/migrations/0004_auto_20221130_2138.py new file mode 100644 index 000000000..2b092eb9a --- /dev/null +++ b/performance/migrations/0004_auto_20221130_2138.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.20 on 2022-11-30 21:38 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0003_remove_eqprsfileupload_file_validation_report"), + ] + + operations = [ + migrations.AlterField( + model_name="eqprsfileupload", + name="task", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="django_q.Task", + ), + ), + ] diff --git a/performance/migrations/0004_auto_20221205_1147.py b/performance/migrations/0004_auto_20221205_1147.py new file mode 100644 index 000000000..e01cb690d --- /dev/null +++ b/performance/migrations/0004_auto_20221205_1147.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.20 on 2022-12-05 11:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0003_remove_eqprsfileupload_file_validation_report"), + ] + + operations = [ + migrations.AddField( + model_name="eqprsfileupload", + name="created_at", + field=models.DateTimeField(auto_now_add=True, null=True), + ), + migrations.AddField( + model_name="eqprsfileupload", + name="updated_at", + field=models.DateTimeField(auto_now=True, null=True), + ), + ] diff --git a/performance/migrations/0005_indicator_created_at.py b/performance/migrations/0005_indicator_created_at.py new file mode 100644 index 000000000..78df9b46c --- /dev/null +++ b/performance/migrations/0005_indicator_created_at.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.20 on 2022-12-05 11:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0004_auto_20221205_1147"), + ] + + operations = [ + migrations.AddField( + model_name="indicator", + name="created_at", + field=models.DateTimeField(auto_now_add=True, null=True), + ), + ] diff --git a/performance/migrations/0006_merge_20221206_1915.py b/performance/migrations/0006_merge_20221206_1915.py new file mode 100644 index 000000000..0468bd532 --- /dev/null +++ b/performance/migrations/0006_merge_20221206_1915.py @@ -0,0 +1,13 @@ +# Generated by Django 2.2.20 on 2022-12-06 19:15 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0005_indicator_created_at"), + ("performance", "0004_auto_20221130_2138"), + ] + + operations = [] diff --git a/performance/migrations/0007_auto_20221213_2013.py b/performance/migrations/0007_auto_20221213_2013.py new file mode 100644 index 000000000..d7417342b --- /dev/null +++ b/performance/migrations/0007_auto_20221213_2013.py @@ -0,0 +1,42 @@ +# Generated by Django 2.2.20 on 2022-12-13 20:13 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0006_merge_20221206_1915"), + ] + + operations = [ + migrations.AlterField( + model_name="eqprsfileupload", + name="num_imported", + field=models.IntegerField( + default=0, null=True, verbose_name="Number of rows we could import" + ), + ), + migrations.AlterField( + model_name="eqprsfileupload", + name="num_not_imported", + field=models.IntegerField( + default=0, null=True, verbose_name="Number of rows we could not import" + ), + ), + migrations.AlterField( + model_name="eqprsfileupload", + name="user", + field=models.ForeignKey( + default=0, + on_delete=django.db.models.deletion.DO_NOTHING, + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AlterUniqueTogether( + name="indicator", + unique_together=set(), + ), + ] diff --git a/performance/migrations/0008_auto_20230104_2150.py b/performance/migrations/0008_auto_20230104_2150.py new file mode 100644 index 000000000..8c81b3b5e --- /dev/null +++ b/performance/migrations/0008_auto_20230104_2150.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.20 on 2023-01-04 21:50 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0007_auto_20221213_2013"), + ] + + operations = [ + migrations.AlterField( + model_name="eqprsfileupload", + name="user", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to=settings.AUTH_USER_MODEL, + ), + ), + ] diff --git a/performance/migrations/0009_auto_20230113_1028.py b/performance/migrations/0009_auto_20230113_1028.py new file mode 100644 index 000000000..14154155f --- /dev/null +++ b/performance/migrations/0009_auto_20230113_1028.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.28 on 2023-01-13 10:28 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0008_auto_20230104_2150"), + ] + + operations = [ + migrations.RemoveField( + model_name="eqprsfileupload", + name="task", + ), + migrations.AlterField( + model_name="eqprsfileupload", + name="user", + field=models.ForeignKey( + blank=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to=settings.AUTH_USER_MODEL, + ), + ), + ] diff --git a/performance/migrations/0010_eqprsfileupload_task_id.py b/performance/migrations/0010_eqprsfileupload_task_id.py new file mode 100644 index 000000000..1ba48388b --- /dev/null +++ b/performance/migrations/0010_eqprsfileupload_task_id.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.28 on 2023-01-13 10:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0009_auto_20230113_1028"), + ] + + operations = [ + migrations.AddField( + model_name="eqprsfileupload", + name="task_id", + field=models.TextField(default="WASN'T SET WHEN FIELD WAS ADDED"), + preserve_default=False, + ), + ] diff --git a/performance/migrations/0011_auto_20230202_1043.py b/performance/migrations/0011_auto_20230202_1043.py new file mode 100644 index 000000000..74c376989 --- /dev/null +++ b/performance/migrations/0011_auto_20230202_1043.py @@ -0,0 +1,26 @@ +# Generated by Django 2.2.28 on 2023-02-02 10:43 + +import django.contrib.postgres.indexes +import django.contrib.postgres.search +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0010_eqprsfileupload_task_id"), + ] + + operations = [ + migrations.AddField( + model_name="indicator", + name="content_search", + field=django.contrib.postgres.search.SearchVectorField(null=True), + ), + migrations.AddIndex( + model_name="indicator", + index=django.contrib.postgres.indexes.GinIndex( + fields=["content_search"], name="performance_content_b5accd_gin" + ), + ), + ] diff --git a/performance/migrations/0012_auto_20230202_1121.py b/performance/migrations/0012_auto_20230202_1121.py new file mode 100644 index 000000000..ae98bdfc4 --- /dev/null +++ b/performance/migrations/0012_auto_20230202_1121.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2023-02-02 11:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0011_auto_20230202_1043"), + ] + + operations = [ + migrations.RenameField( + model_name="indicator", + old_name="annual_dpme_coordincator_comments", + new_name="annual_dpme_coordinator_comments", + ), + ] diff --git a/performance/migrations/0013__add_full_text_index_triggers.py b/performance/migrations/0013__add_full_text_index_triggers.py new file mode 100644 index 000000000..613f4751a --- /dev/null +++ b/performance/migrations/0013__add_full_text_index_triggers.py @@ -0,0 +1,49 @@ +# Generated by Django 2.2.28 on 2023-02-02 10:46 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0012_auto_20230202_1121"), + ] + + migration = """ + CREATE TRIGGER content_search_update BEFORE INSERT OR UPDATE + ON performance_indicator FOR EACH ROW EXECUTE PROCEDURE + tsvector_update_trigger( + content_search, 'pg_catalog.english', + q1_target, q1_actual_output, q1_deviation_reason, q1_corrective_action, + q1_national_comments, q1_otp_comments, q1_dpme_coordinator_comments, + q1_treasury_comments, + + q2_target, q2_actual_output, q2_deviation_reason, q2_corrective_action, + q2_national_comments, q2_otp_comments, q2_dpme_coordinator_comments, + q2_treasury_comments, + + q3_target, q3_actual_output, q3_deviation_reason, q3_corrective_action, + q3_national_comments, q3_otp_comments, q3_dpme_coordinator_comments, + q3_treasury_comments, + + q4_target, q4_actual_output, q4_deviation_reason, q4_corrective_action, + q4_national_comments, q4_otp_comments, q4_dpme_coordinator_comments, + q4_treasury_comments, + + annual_target, annual_aggregate_output, annual_pre_audit_output, + annual_deviation_reason, annual_corrective_action, + annual_national_comments, annual_otp_comments, annual_dpme_coordinator_comments, + annual_treasury_comments, annual_audited_output, + + sector, programme_name, subprogramme_name, frequency, type, subtype, + mtsf_outcome, cluster + ); + -- Force triggers to run and populate the content_search column. + UPDATE performance_indicator set ID = ID; + """ + + reverse_migration = """ + DROP TRIGGER content_search_update ON performance_indicator; + """ + + operations = [migrations.RunSQL(migration, reverse_migration)] diff --git a/performance/migrations/0014_eqprsdepartmentalias.py b/performance/migrations/0014_eqprsdepartmentalias.py new file mode 100644 index 000000000..ae42d3be9 --- /dev/null +++ b/performance/migrations/0014_eqprsdepartmentalias.py @@ -0,0 +1,37 @@ +# Generated by Django 2.2.28 on 2023-02-23 07:53 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("budgetportal", "0068_auto_20230220_0910"), + ("performance", "0013__add_full_text_index_triggers"), + ] + + operations = [ + migrations.CreateModel( + name="EQPRSDepartmentAlias", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("alias", models.TextField()), + ( + "department", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="budgetportal.Department", + ), + ), + ], + ), + ] diff --git a/performance/migrations/0015_auto_20230223_0755.py b/performance/migrations/0015_auto_20230223_0755.py new file mode 100644 index 000000000..558f15971 --- /dev/null +++ b/performance/migrations/0015_auto_20230223_0755.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.28 on 2023-02-23 07:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0014_eqprsdepartmentalias"), + ] + + operations = [ + migrations.AlterModelOptions( + name="eqprsdepartmentalias", + options={"verbose_name_plural": "Eqprs department aliases"}, + ), + ] diff --git a/performance/migrations/0016_auto_20230223_0804.py b/performance/migrations/0016_auto_20230223_0804.py new file mode 100644 index 000000000..8320f712b --- /dev/null +++ b/performance/migrations/0016_auto_20230223_0804.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2023-02-23 08:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("performance", "0015_auto_20230223_0755"), + ] + + operations = [ + migrations.AlterField( + model_name="eqprsdepartmentalias", + name="alias", + field=models.CharField(max_length=200), + ), + ] diff --git a/performance/migrations/__init__.py b/performance/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/performance/models.py b/performance/models.py new file mode 100644 index 000000000..5338c25a1 --- /dev/null +++ b/performance/models.py @@ -0,0 +1,130 @@ +from django.contrib.auth.models import User +from django.contrib.postgres.fields import JSONField +from django.contrib.postgres.search import SearchVectorField +from django.contrib.postgres.indexes import GinIndex +from django.db import models +from django_q.tasks import Task + +from budgetportal.models.government import Department + +import uuid + + +def eqprs_file_path(instance, filename): + return f"eqprs_uploads/{uuid.uuid4()}/{filename}" + + +class EQPRSFileUpload(models.Model): + user = models.ForeignKey(User, models.DO_NOTHING, blank=True) + task_id = models.TextField() + file = models.FileField(upload_to=eqprs_file_path) + # Plain text listing which departments could not be matched and were not imported + import_report = models.TextField() + num_imported = models.IntegerField( + null=True, default=0, verbose_name="Number of rows we could import" + ) # number of rows we could import + num_not_imported = models.IntegerField( + null=True, default=0, verbose_name="Number of rows we could not import" + ) # number of rows we could not import + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) + updated_at = models.DateTimeField(auto_now=True, blank=True, null=True) + + +class Frequency: + ANNUALLY = "annually" + QUARTERLY = "quarterly" + + +FREQUENCIES = ( + (Frequency.ANNUALLY, "Annually"), + (Frequency.QUARTERLY, "Quarterly"), +) + + +class Indicator(models.Model): + """The indicator values available for a indicator in a department in a financial year""" + + department = models.ForeignKey( + Department, on_delete=models.CASCADE, related_name="indicator_values" + ) + indicator_name = models.TextField() + + # OTP stands for Office of the Premier + # national_comments is provincial only, headed "National Oversight Comments" + # dpme_comments is national only, headed "DPME Coordinator Comments" + # treasury_comments is national only, headed "National Treasury Coordinator comments + + q1_target = models.TextField(blank=True) + q1_actual_output = models.TextField(blank=True) + q1_deviation_reason = models.TextField(blank=True) + q1_corrective_action = models.TextField(blank=True) + q1_national_comments = models.TextField(blank=True) + q1_otp_comments = models.TextField(blank=True) + q1_dpme_coordinator_comments = models.TextField(blank=True) + q1_treasury_comments = models.TextField(blank=True) + + q2_target = models.TextField(blank=True) + q2_actual_output = models.TextField(blank=True) + q2_deviation_reason = models.TextField(blank=True) + q2_corrective_action = models.TextField(blank=True) + q2_national_comments = models.TextField(blank=True) + q2_otp_comments = models.TextField(blank=True) + q2_dpme_coordinator_comments = models.TextField(blank=True) + q2_treasury_comments = models.TextField(blank=True) + + q3_target = models.TextField(blank=True) + q3_actual_output = models.TextField(blank=True) + q3_deviation_reason = models.TextField(blank=True) + q3_corrective_action = models.TextField(blank=True) + q3_national_comments = models.TextField(blank=True) + q3_otp_comments = models.TextField(blank=True) + q3_dpme_coordinator_comments = models.TextField(blank=True) + q3_treasury_comments = models.TextField(blank=True) + + q4_target = models.TextField(blank=True) + q4_actual_output = models.TextField(blank=True) + q4_deviation_reason = models.TextField(blank=True) + q4_corrective_action = models.TextField(blank=True) + q4_national_comments = models.TextField(blank=True) + q4_otp_comments = models.TextField(blank=True) + q4_dpme_coordinator_comments = models.TextField(blank=True) + q4_treasury_comments = models.TextField(blank=True) + + annual_target = models.TextField(blank=True) # AnnualTarget_Summary2 + annual_aggregate_output = models.TextField(blank=True) # Preliminary_Summary2 + annual_pre_audit_output = models.TextField(blank=True) # PrelimaryAudited_Summary2 + annual_deviation_reason = models.TextField(blank=True) + annual_corrective_action = models.TextField(blank=True) + annual_otp_comments = models.TextField(blank=True) + annual_national_comments = models.TextField(blank=True) + annual_dpme_coordinator_comments = models.TextField(blank=True) + annual_treasury_comments = models.TextField(blank=True) + annual_audited_output = models.TextField(blank=True) # ValidatedAuditedSummary2 + + sector = models.TextField(blank=True) + programme_name = models.TextField(blank=True) + subprogramme_name = models.TextField(blank=True) + frequency = models.CharField(max_length=9, choices=FREQUENCIES) + type = models.TextField(blank=True) + subtype = models.TextField(blank=True) + mtsf_outcome = models.TextField(blank=True) + cluster = models.TextField(blank=True) + uid = models.TextField(blank=True) + + source = models.ForeignKey( + EQPRSFileUpload, on_delete=models.CASCADE, related_name="indicator_values" + ) + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) + + content_search = SearchVectorField(null=True) + + class Meta: + indexes = [GinIndex(fields=["content_search"])] + + +class EQPRSDepartmentAlias(models.Model): + department = models.ForeignKey(Department, on_delete=models.CASCADE) + alias = models.CharField(max_length=200) + + class Meta: + verbose_name_plural = "Eqprs department aliases" diff --git a/performance/serializer.py b/performance/serializer.py new file mode 100644 index 000000000..9e0bfc681 --- /dev/null +++ b/performance/serializer.py @@ -0,0 +1,75 @@ +from rest_framework.serializers import ModelSerializer +from .models import Indicator +from budgetportal.models.government import Department, Government, Sphere, FinancialYear + + +class FinancialYearSerializer(ModelSerializer): + class Meta: + model = FinancialYear + fields = ("slug",) + + +class SphereSerializer(ModelSerializer): + financial_year = FinancialYearSerializer() + + class Meta: + model = Sphere + fields = ( + "financial_year", + "name", + ) + + +class GovernmentSerializer(ModelSerializer): + sphere = SphereSerializer() + + class Meta: + model = Government + fields = ( + "sphere", + "name", + ) + + +class DepartmentSerializer(ModelSerializer): + government = GovernmentSerializer() + + class Meta: + model = Department + fields = ( + "government", + "name", + ) + + +class IndicatorSerializer(ModelSerializer): + department = DepartmentSerializer() + + class Meta: + model = Indicator + exclude = ( + "source", + "content_search", + "uid", + "created_at", + "annual_otp_comments", + "annual_national_comments", + "annual_dpme_coordinator_comments", + "annual_treasury_comments", + "q1_national_comments", + "q1_otp_comments", + "q1_dpme_coordinator_comments", + "q1_treasury_comments", + "q2_national_comments", + "q2_otp_comments", + "q2_dpme_coordinator_comments", + "q2_treasury_comments", + "q3_national_comments", + "q3_otp_comments", + "q3_dpme_coordinator_comments", + "q3_treasury_comments", + "q4_national_comments", + "q4_otp_comments", + "q4_dpme_coordinator_comments", + "q4_treasury_comments", + ) diff --git a/performance/settings.py b/performance/settings.py new file mode 100644 index 000000000..d8f943175 --- /dev/null +++ b/performance/settings.py @@ -0,0 +1,7 @@ +REST_FRAMEWORK = { + "DEFAULT_RENDERER_CLASSES": ( + "rest_framework.renderers.JSONRenderer", + "rest_framework.renderers.BrowsableAPIRenderer", + "drf_excel.renderers.XLSXRenderer", + ) +} diff --git a/performance/template.xlsx b/performance/template.xlsx new file mode 100644 index 000000000..ade2a2aaf Binary files /dev/null and b/performance/template.xlsx differ diff --git a/performance/templates/performance/performance.html b/performance/templates/performance/performance.html new file mode 100644 index 000000000..c06861492 --- /dev/null +++ b/performance/templates/performance/performance.html @@ -0,0 +1,15 @@ +{% extends 'page-shell.html' %} +{% load define_action %} +{% block page_content %} + +
    +
    +

    Quarterly performance reporting (QPR) indicators

    + +

    Find the latest quarterly performance monitoring indicators, results, and explanations from national and provincial departments.

    + +

    How is performance measured in government? Quarterly and audited annual indicators is one of the tools to monitor implementation of department mandates.

    + {% include 'components/performance/Table/index.html' %} +
    +
    +{% endblock %} diff --git a/performance/tests/static/correct_data.csv b/performance/tests/static/correct_data.csv new file mode 100644 index 000000000..4a325e11c --- /dev/null +++ b/performance/tests/static/correct_data.csv @@ -0,0 +1,9 @@ +ReportTitle,Textbox95,Textbox80,Textbox81,Textbox82,Textbox83 +QPR for FY 2021-22 Provincial Institutions Oversight Performance Report as of ( FY 2021-22),"Location/s: Eastern Cape, Free State",InstitutionType: Provincial,Institution: Health,FinancialYear: FY 2021-22,"Report Printed On: Wednesday, September 28, 2022 11:11:10 AM" + +Sector,Institution,Programme,SubProgramme,Location,Frequency,Indicator,Type,SubType,Outcome,Cluster,Target_Q1,ActualOutput_Q1,ReasonforDeviation_Q1,CorrectiveAction_Q1,OTP_Q1,National_Q1,Target_Q2,ActualOutput_Q2,ReasonforDeviation_Q2,CorrectiveAction_Q2,OTP_Q2,National_Q2,Target_Q3,ActualOutput_Q3,ReasonforDeviation_Q3,CorrectiveAction_Q3,OTP_Q3,National_Q3,Target_Q4,ActualOutput_Q4,ReasonforDeviation_Q4,CorrectiveAction_Q4,OTP_Q4,National_Q4,AnnualTarget_Summary2,Preliminary_Summary2,PrelimaryAudited_Summary2,ReasonforDeviation_Summary,CorrectiveAction_Summary,OTP_Summary,National_Summary,ValidatedAudited_Summary2,UID +Health,Health,Eastern Cape,Programme 1: Administration,Sub-Programme 1.1: Office of the MEC,Quarterly,9.1.2 Number of statutory documents tabled at Legislature,Non-Standardized,Max,"Priority 3: Education, Skills And Health","The Social Protection, Community and Human Development cluster",0,0,There is no target for quarter one,,,,1,2,Target achieved,,,,2,2,Target achieved,,,,5,8,All statutory documents submitted.,,,,8,8,8,Target achieved ,,,,,291 +Health,Health,Eastern Cape,Programme 1: Administration,Sub-Programme 1.2: Management,Annually,6.4.1 Holistic Human Resources for Health (HRH) strategy approved ,Non-Standardized,Not Applicable,"Priority 3: Education, Skills And Health","The Social Protection, Community and Human Development cluster",,,,,,,,,,,,,,,,,,,,,,,,,Approved HRH Strategy,A draft plan for Human Resources for Health (HRH has been produced,Draft HRH Strategy,"The HRH 2030 Strategy Plan is in progress. +The Task Team has produced a draft HRH 2030 Plan by the end of the 4th quarter. + The Task Team requires the additional capacity of a qualified Technical Advisor to conclude this plan. +",,,,,291 diff --git a/performance/tests/static/data_for_deleting_indicators.csv b/performance/tests/static/data_for_deleting_indicators.csv new file mode 100644 index 000000000..9d02be107 --- /dev/null +++ b/performance/tests/static/data_for_deleting_indicators.csv @@ -0,0 +1,9 @@ +ReportTitle,Textbox95,Textbox80,Textbox81,Textbox82,Textbox83 +QPR for FY 2017-18 Provincial Institutions Oversight Performance Report as of ( FY 2017-18),"Location/s: Eastern Cape, Free State",InstitutionType: Provincial,Institution: Health,FinancialYear: FY 2017-18,"Report Printed On: Wednesday, September 28, 2022 11:11:10 AM" + +Sector,Institution,Programme,SubProgramme,Location,Frequency,Indicator,Type,SubType,Outcome,Cluster,Target_Q1,ActualOutput_Q1,ReasonforDeviation_Q1,CorrectiveAction_Q1,OTP_Q1,National_Q1,Target_Q2,ActualOutput_Q2,ReasonforDeviation_Q2,CorrectiveAction_Q2,OTP_Q2,National_Q2,Target_Q3,ActualOutput_Q3,ReasonforDeviation_Q3,CorrectiveAction_Q3,OTP_Q3,National_Q3,Target_Q4,ActualOutput_Q4,ReasonforDeviation_Q4,CorrectiveAction_Q4,OTP_Q4,National_Q4,AnnualTarget_Summary2,Preliminary_Summary2,PrelimaryAudited_Summary2,ReasonforDeviation_Summary,CorrectiveAction_Summary,OTP_Summary,National_Summary,ValidatedAudited_Summary2,UID +Health,Health,Eastern Cape,Programme 1: Administration,Sub-Programme 1.1: Office of the MEC,Quarterly,9.1.2 Number of statutory documents tabled at Legislature,Non-Standardized,Max,"Priority 3: Education, Skills And Health","The Social Protection, Community and Human Development cluster",0,0,There is no target for quarter one,,,,1,2,Target achieved,,,,2,2,Target achieved,,,,5,8,All statutory documents submitted.,,,,8,8,8,Target achieved ,,,,,291 +Health,Education,Western Cape,Programme 1: Administration,Sub-Programme 1.2: Management,Annually,6.4.1 Holistic Human Resources for Health (HRH) strategy approved ,Non-Standardized,Not Applicable,"Priority 3: Education, Skills And Health","The Social Protection, Community and Human Development cluster",,,,,,,,,,,,,,,,,,,,,,,,,Approved HRH Strategy,A draft plan for Human Resources for Health (HRH has been produced,Draft HRH Strategy,"The HRH 2030 Strategy Plan is in progress. +The Task Team has produced a draft HRH 2030 Plan by the end of the 4th quarter. + The Task Team requires the additional capacity of a qualified Technical Advisor to conclude this plan. +",,,,,291 diff --git a/performance/tests/static/department_name_containing_government.csv b/performance/tests/static/department_name_containing_government.csv new file mode 100644 index 000000000..2b53d67ba --- /dev/null +++ b/performance/tests/static/department_name_containing_government.csv @@ -0,0 +1,9 @@ +ReportTitle,Textbox95,Textbox80,Textbox81,Textbox82,Textbox83 +QPR for FY 2021-22 Provincial Institutions Oversight Performance Report as of ( FY 2021-22),"Location/s: Eastern Cape, Free State",InstitutionType: Provincial,Institution: Health,FinancialYear: FY 2021-22,"Report Printed On: Wednesday, September 28, 2022 11:11:10 AM" + +Sector,Institution,Programme,SubProgramme,Location,Frequency,Indicator,Type,SubType,Outcome,Cluster,Target_Q1,ActualOutput_Q1,ReasonforDeviation_Q1,CorrectiveAction_Q1,OTP_Q1,National_Q1,Target_Q2,ActualOutput_Q2,ReasonforDeviation_Q2,CorrectiveAction_Q2,OTP_Q2,National_Q2,Target_Q3,ActualOutput_Q3,ReasonforDeviation_Q3,CorrectiveAction_Q3,OTP_Q3,National_Q3,Target_Q4,ActualOutput_Q4,ReasonforDeviation_Q4,CorrectiveAction_Q4,OTP_Q4,National_Q4,AnnualTarget_Summary2,Preliminary_Summary2,PrelimaryAudited_Summary2,ReasonforDeviation_Summary,CorrectiveAction_Summary,OTP_Summary,National_Summary,ValidatedAudited_Summary2,UID +Health,Health,Eastern Cape,Programme 1: Administration,Sub-Programme 1.1: Office of the MEC,Quarterly,9.1.2 Number of statutory documents tabled at Legislature,Non-Standardized,Max,"Priority 3: Education, Skills And Health","The Social Protection, Community and Human Development cluster",0,0,There is no target for quarter one,,,,1,2,Target achieved,,,,2,2,Target achieved,,,,5,8,All statutory documents submitted.,,,,8,8,8,Target achieved ,,,,,291 +Health,Eastern Cape: Health,Eastern Cape,Programme 1: Administration,Sub-Programme 1.2: Management,Annually,6.4.1 Holistic Human Resources for Health (HRH) strategy approved ,Non-Standardized,Not Applicable,"Priority 3: Education, Skills And Health","The Social Protection, Community and Human Development cluster",,,,,,,,,,,,,,,,,,,,,,,,,Approved HRH Strategy,A draft plan for Human Resources for Health (HRH has been produced,Draft HRH Strategy,"The HRH 2030 Strategy Plan is in progress. +The Task Team has produced a draft HRH 2030 Plan by the end of the 4th quarter. + The Task Team requires the additional capacity of a qualified Technical Advisor to conclude this plan. +",,,,,291 diff --git a/performance/tests/static/national_data.csv b/performance/tests/static/national_data.csv new file mode 100644 index 000000000..2b5524672 --- /dev/null +++ b/performance/tests/static/national_data.csv @@ -0,0 +1,5 @@ +ReportTitle,Textbox95,Textbox80,Textbox81,Textbox82,Textbox83 +QPR for FY 2021-22 Provincial Institutions Oversight Performance Report as of ( FY 2021-22),"Location/s: Eastern Cape, Free State",InstitutionType: Provincial,Institution: Health,FinancialYear: FY 2021-22,"Report Printed On: Wednesday, September 28, 2022 11:11:10 AM" + +Sector,Institution,Programme,SubProgramme,Location,Frequency,Indicator,Type,SubType,Outcome,Cluster,Target_Q1,ActualOutput_Q1,ReasonforDeviation_Q1,CorrectiveAction_Q1,OTP_Q1,National_Q1,Target_Q2,ActualOutput_Q2,ReasonforDeviation_Q2,CorrectiveAction_Q2,OTP_Q2,National_Q2,Target_Q3,ActualOutput_Q3,ReasonforDeviation_Q3,CorrectiveAction_Q3,OTP_Q3,National_Q3,Target_Q4,ActualOutput_Q4,ReasonforDeviation_Q4,CorrectiveAction_Q4,OTP_Q4,National_Q4,AnnualTarget_Summary2,Preliminary_Summary2,PrelimaryAudited_Summary2,ReasonforDeviation_Summary,CorrectiveAction_Summary,OTP_Summary,National_Summary,ValidatedAudited_Summary2,UID +Health,Health,National,Programme 1: Administration,Sub-Programme 1.1: Office of the MEC,Quarterly,9.1.2 Number of statutory documents tabled at Legislature,Non-Standardized,Max,"Priority 3: Education, Skills And Health","The Social Protection, Community and Human Development cluster",0,0,There is no target for quarter one,,,,1,2,Target achieved,,,,2,2,Target achieved,,,,5,8,All statutory documents submitted.,,,,8,8,8,Target achieved ,,,,,291 \ No newline at end of file diff --git a/performance/tests/static/wrong_report_type.csv b/performance/tests/static/wrong_report_type.csv new file mode 100644 index 000000000..f3589f94e --- /dev/null +++ b/performance/tests/static/wrong_report_type.csv @@ -0,0 +1,9 @@ +ReportTitle,Textbox95,Textbox80,Textbox81,Textbox82,Textbox83 +QPR for FY 2021-22 Provincial This is not correct Report as of ( FY 2021-22),"Location/s: Eastern Cape, Free State",InstitutionType: Provincial,Institution: Health,FinancialYear: FY 2021-22,"Report Printed On: Wednesday, September 28, 2022 11:11:10 AM" + +Sector,Institution,Programme,SubProgramme,Location,Frequency,Indicator,Type,SubType,Outcome,Cluster,Target_Q1,ActualOutput_Q1,ReasonforDeviation_Q1,CorrectiveAction_Q1,OTP_Q1,National_Q1,Target_Q2,ActualOutput_Q2,ReasonforDeviation_Q2,CorrectiveAction_Q2,OTP_Q2,National_Q2,Target_Q3,ActualOutput_Q3,ReasonforDeviation_Q3,CorrectiveAction_Q3,OTP_Q3,National_Q3,Target_Q4,ActualOutput_Q4,ReasonforDeviation_Q4,CorrectiveAction_Q4,OTP_Q4,National_Q4,AnnualTarget_Summary2,Preliminary_Summary2,PrelimaryAudited_Summary2,ReasonforDeviation_Summary,CorrectiveAction_Summary,OTP_Summary,National_Summary,ValidatedAudited_Summary2,UID +Health,Health,Eastern Cape,Programme 1: Administration,Sub-Programme 1.1: Office of the MEC,Quarterly,9.1.2 Number of statutory documents tabled at Legislature,Non-Standardized,Max,"Priority 3: Education, Skills And Health","The Social Protection, Community and Human Development cluster",0,0,There is no target for quarter one,,,,1,2,Target achieved,,,,2,2,Target achieved,,,,5,8,All statutory documents submitted.,,,,8,8,8,Target achieved ,,,,,291 +Health,Health,Eastern Cape,Programme 1: Administration,Sub-Programme 1.2: Management,Annually,6.4.1 Holistic Human Resources for Health (HRH) strategy approved ,Non-Standardized,Not Applicable,"Priority 3: Education, Skills And Health","The Social Protection, Community and Human Development cluster",,,,,,,,,,,,,,,,,,,,,,,,,Approved HRH Strategy,A draft plan for Human Resources for Health (HRH has been produced,Draft HRH Strategy,"The HRH 2030 Strategy Plan is in progress. +The Task Team has produced a draft HRH 2030 Plan by the end of the 4th quarter. + The Task Team requires the additional capacity of a qualified Technical Advisor to conclude this plan. +",,,,,291 \ No newline at end of file diff --git a/performance/tests/test_eqprs_api.py b/performance/tests/test_eqprs_api.py new file mode 100644 index 000000000..c6b417992 --- /dev/null +++ b/performance/tests/test_eqprs_api.py @@ -0,0 +1,94 @@ +from django.urls import reverse +from rest_framework.test import APITestCase, APIRequestFactory +from rest_framework import status +from django.test import Client +from openpyxl import load_workbook + +import io + + +class indicator_API_Test(APITestCase): + list_url = "/performance/api/v1/eqprs/" + fixtures = ["test_api.json"] + + def test_api(self): + response = self.client.get(self.list_url) + self.assertEqual(response.status_code, 200) + response_payload = self.client.get(self.list_url).json() + + found = False + message1 = "1.2.1 Percentage of valid invoices paid within 30 days upon receipt by the department" + message2 = "1.1.1 Unqualified audit opinion" + if ( + response_payload["results"]["items"][0]["indicator_name"] == message2 + or response_payload["results"]["items"][1]["indicator_name"] == message1 + ): + found = True + + self.assertEqual(found, True) + self.assertTrue(found, True) + + def test_create(self): + response = self.client.post(self.list_url) + self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED) + + def test_update(self): + response = self.client.patch(self.list_url) + self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED) + + def test_text_search(self): + filter_url = self.list_url + "?page=1&q=Unqualified%20audit%20opinion" + response_payload = self.client.get(filter_url).json() + self.assertEqual(len(response_payload["results"]["items"]), 1) + + def test_frequency_search(self): + filter_url = self.list_url + "?page=1&frequency=Annually" + response_payload = self.client.get(filter_url).json() + self.assertEqual(len(response_payload["results"]["items"]), 1) + + def test_downloaded_file(self): + file_url = "/performance/performance-indicators.xlsx/" + response = self.client.get(file_url, stream=True) + file_content = response.getvalue() + file_obj = io.BytesIO(file_content) + wb = load_workbook(file_obj) + sh = wb["Sheet1"] + + self.assertEqual(sh["A1"].value, "Financial year") + self.assertEqual(sh["A2"].value, "2015-16") + self.assertEqual(sh["H3"].value, "Annually") + self.assertEqual(sh["A4"].value, None) + + def test_downloaded_file_with_filter(self): + file_url = "/performance/performance-indicators.xlsx/?frequency=Annually" + response = self.client.get(file_url, stream=True) + file_content = response.getvalue() + file_obj = io.BytesIO(file_content) + wb = load_workbook(file_obj) + sh = wb["Sheet1"] + + self.assertEqual(sh["A1"].value, "Financial year") + self.assertEqual(sh["H2"].value, "Annually") + self.assertEqual(sh["A3"].value, None) + + +class repetitive_API_Test(APITestCase): + list_url = "/performance/api/v1/eqprs/" + file_url = "/performance/performance-indicators.xlsx/" + fixtures = ["test_repetitive_data.json"] + + def test_response_with_repetitive_data(self): + api_response_payload = self.client.get(self.list_url).json() + + file_response = self.client.get(self.file_url, stream=True) + file_content = file_response.getvalue() + file_obj = io.BytesIO(file_content) + wb = load_workbook(file_obj) + + sh = wb["Sheet1"] + + api_result_length = len(api_response_payload["results"]["items"]) + file_result_length = sh.max_row + header_row_count = 1 + + self.assertEqual(file_result_length - header_row_count, api_result_length) diff --git a/performance/tests/test_eqprs_uploads.py b/performance/tests/test_eqprs_uploads.py new file mode 100644 index 000000000..09012f68a --- /dev/null +++ b/performance/tests/test_eqprs_uploads.py @@ -0,0 +1,367 @@ +from django.test import TestCase +from performance.admin import EQPRSFileUploadAdmin +from django.contrib.auth.models import User +from django.contrib.admin.sites import AdminSite +from django.contrib import admin +from performance.models import EQPRSFileUpload, Indicator, EQPRSDepartmentAlias +from budgetportal.models.government import Department, Government, Sphere, FinancialYear +from django.test import RequestFactory +from performance import models +from django.core.files import File +from unittest.mock import Mock + +import performance.admin +import os +import time + +USERNAME = "testuser" +EMAIL = "testuser@domain.com" +PASSWORD = "12345" + + +def get_mocked_request(superuser): + request = RequestFactory().get("/get/request") + request.method = "GET" + request.user = superuser + return request + + +class EQPRSFileUploadTestCase(TestCase): + def setUp(self): + self.superuser = User.objects.create_user( + username=USERNAME, + password=PASSWORD, + is_staff=True, + is_superuser=True, + is_active=True, + ) + file_path = os.path.abspath(("performance/tests/static/correct_data.csv")) + national_file_path = os.path.abspath( + ("performance/tests/static/national_data.csv") + ) + wrong_report_type_file_path = os.path.abspath( + ("performance/tests/static/wrong_report_type.csv") + ) + data_for_deleting_indicators_path = os.path.abspath( + ("performance/tests/static/data_for_deleting_indicators.csv") + ) + department_name_containing_government_path = os.path.abspath( + ("performance/tests/static/department_name_containing_government.csv") + ) + self.csv_file = File(open(file_path, "rb")) + self.national_file = File(open(national_file_path, "rb")) + self.wrong_report_type_file = File(open(wrong_report_type_file_path, "rb")) + self.data_for_deleting_indicators_file = File( + open(data_for_deleting_indicators_path, "rb") + ) + self.department_name_containing_government_file = File( + open(department_name_containing_government_path, "rb") + ) + self.mocked_request = get_mocked_request(self.superuser) + + def tearDown(self): + self.csv_file.close() + self.national_file.close() + self.wrong_report_type_file.close() + self.data_for_deleting_indicators_file.close() + + def test_report_name_validation(self): + test_element = EQPRSFileUpload.objects.create( + user=self.superuser, file=self.wrong_report_type_file + ) + performance.admin.save_imported_indicators(test_element.id) + test_element.refresh_from_db() + assert "Report type must be for one of" in test_element.import_report + assert ( + "* Provincial Institutions Oversight Performance Report" + in test_element.import_report + ) + assert ( + "* National Institutions Oversight Performance Report" + in test_element.import_report + ) + + def test_with_missing_department(self): + test_element = EQPRSFileUpload.objects.create( + user=self.superuser, file=self.csv_file + ) + fy = FinancialYear.objects.create(slug="2021-22") + sphere = Sphere.objects.create(name="Provincial", financial_year=fy) + government = Government.objects.create(name="Eastern Cape", sphere=sphere) + department = Department.objects.create( + name="HealthTest", government=government, vote_number=1 + ) + performance.admin.save_imported_indicators(test_element.id) + test_element.refresh_from_db() + assert test_element.num_not_imported == 2 + assert ( + "Department names that could not be matched on import :" + in test_element.import_report + ) + assert "* Health" in test_element.import_report + + def test_with_correct_csv(self): + fy = FinancialYear.objects.create(slug="2021-22") + sphere = Sphere.objects.create(name="Provincial", financial_year=fy) + government = Government.objects.create(name="Eastern Cape", sphere=sphere) + department = Department.objects.create( + name="Health", government=government, vote_number=1 + ) + + test_element = EQPRSFileUpload.objects.create( + user=self.superuser, file=self.csv_file + ) + performance.admin.save_imported_indicators(test_element.id) + test_element.refresh_from_db() + assert Indicator.objects.all().count() == 2 + + indicator = models.Indicator.objects.all().first() + assert test_element.import_report == "" + assert test_element.num_imported == 2 + assert ( + indicator.indicator_name + == "9.1.2 Number of statutory documents tabled at Legislature" + ) + assert indicator.sector == "Health" + assert indicator.programme_name == "Programme 1: Administration" + assert indicator.subprogramme_name == "Sub-Programme 1.1: Office of the MEC" + assert indicator.frequency == "quarterly" + assert indicator.type == "Non-Standardized" + assert indicator.subtype == "Max" + assert indicator.mtsf_outcome == "Priority 3: Education, Skills And Health" + assert ( + indicator.cluster + == "The Social Protection, Community and Human Development cluster" + ) + + assert indicator.q1_target == "0" + assert indicator.q1_actual_output == "0" + assert indicator.q1_deviation_reason == "There is no target for quarter one" + assert indicator.q1_corrective_action == "" + assert indicator.q1_national_comments == "" + assert indicator.q1_otp_comments == "" + assert indicator.q1_dpme_coordinator_comments == "" + assert indicator.q1_treasury_comments == "" + + assert indicator.q2_target == "1" + assert indicator.q2_actual_output == "2" + assert indicator.q2_deviation_reason == "Target achieved" + assert indicator.q2_corrective_action == "" + assert indicator.q2_national_comments == "" + assert indicator.q2_otp_comments == "" + assert indicator.q2_dpme_coordinator_comments == "" + assert indicator.q2_treasury_comments == "" + + assert indicator.q3_target == "2" + assert indicator.q3_actual_output == "2" + assert indicator.q3_deviation_reason == "Target achieved" + assert indicator.q3_corrective_action == "" + assert indicator.q3_national_comments == "" + assert indicator.q3_otp_comments == "" + assert indicator.q3_dpme_coordinator_comments == "" + assert indicator.q3_treasury_comments == "" + + assert indicator.q4_target == "5" + assert indicator.q4_actual_output == "8" + assert indicator.q4_deviation_reason == "All statutory documents submitted." + assert indicator.q4_corrective_action == "" + assert indicator.q4_national_comments == "" + assert indicator.q4_otp_comments == "" + assert indicator.q4_dpme_coordinator_comments == "" + assert indicator.q4_treasury_comments == "" + + assert indicator.annual_target == "8" + assert indicator.annual_aggregate_output == "" + assert indicator.annual_pre_audit_output == "8" + assert indicator.annual_deviation_reason == "Target achieved " + assert indicator.annual_corrective_action == "" + assert indicator.annual_otp_comments == "" + assert indicator.annual_national_comments == "" + assert indicator.annual_dpme_coordinator_comments == "" + assert indicator.annual_treasury_comments == "" + assert indicator.annual_audited_output == "" + + def test_task_creation(self): + fy = FinancialYear.objects.create(slug="2021-22") + sphere = Sphere.objects.create(name="Provincial", financial_year=fy) + government = Government.objects.create(name="Eastern Cape", sphere=sphere) + department = Department.objects.create( + name="Health", government=government, vote_number=1 + ) + + model_admin = EQPRSFileUploadAdmin( + model=EQPRSFileUpload, admin_site=AdminSite() + ) + model_admin.save_model( + obj=EQPRSFileUpload(file=self.csv_file), + request=Mock(user=self.superuser), + form=None, + change=None, + ) + + last_element = EQPRSFileUpload.objects.all().last() + assert last_element.task_id is not None + + def test_status_in_list_view(self): + assert "processing_completed" in EQPRSFileUploadAdmin.list_display + + fy = FinancialYear.objects.create(slug="2021-22") + sphere = Sphere.objects.create(name="Provincial", financial_year=fy) + government = Government.objects.create(name="Eastern Cape", sphere=sphere) + department = Department.objects.create( + name="Health", government=government, vote_number=1 + ) + + model_admin = EQPRSFileUploadAdmin( + model=EQPRSFileUpload, admin_site=AdminSite() + ) + model_admin.save_model( + obj=EQPRSFileUpload(file=self.csv_file), + request=Mock(user=self.superuser), + form=None, + change=None, + ) + + last_element = EQPRSFileUpload.objects.all().last() + assert model_admin.processing_completed(last_element) == True + + def test_with_national_government(self): + fy = FinancialYear.objects.create(slug="2021-22") + sphere = Sphere.objects.create(name="Provincial", financial_year=fy) + government = Government.objects.create(name="South Africa", sphere=sphere) + department = Department.objects.create( + name="Health", government=government, vote_number=1 + ) + + test_element = EQPRSFileUpload.objects.create( + user=self.superuser, file=self.national_file + ) + performance.admin.save_imported_indicators(test_element.id) + test_element.refresh_from_db() + + assert test_element.import_report == "" + assert test_element.num_imported == 1 + indicator = models.Indicator.objects.all().first() + assert indicator.programme_name == "Programme 1: Administration" + + def test_with_alias(self): + fy = FinancialYear.objects.create(slug="2021-22") + sphere = Sphere.objects.create(name="Provincial", financial_year=fy) + government = Government.objects.create(name="South Africa", sphere=sphere) + department = Department.objects.create( + name="Department to be found by its alias", + government=government, + vote_number=1, + ) + EQPRSDepartmentAlias.objects.create(department=department, alias="Health") + + test_element = EQPRSFileUpload.objects.create( + user=self.superuser, file=self.national_file + ) + performance.admin.save_imported_indicators(test_element.id) + test_element.refresh_from_db() + + assert test_element.import_report == "" + assert test_element.num_imported == 1 + indicator = models.Indicator.objects.all().first() + assert indicator.programme_name == "Programme 1: Administration" + assert indicator.department.name == "Department to be found by its alias" + + def test_deleting_old_indicators(self): + """ + test that + - if gov A and dept A is in the data, its old indicators for that + financial year are deleted. Any other financial years remain + - if Gov A, B and dept A, B is in the data, old indicators for Gov A + DeptA are deleted but Gov A dept B are not deleted + """ + + fy = FinancialYear.objects.create(slug="2017-18") + sphere = Sphere.objects.create(name="Provincial", financial_year=fy) + + government_1 = Government.objects.create(name="Eastern Cape", sphere=sphere) + department_1 = Department.objects.create( + name="Health", + government=government_1, + vote_number=1, + ) + Indicator.objects.create( + indicator_name="Test Indicator 1", + department=department_1, + source=EQPRSFileUpload.objects.create(user=self.superuser, file=None), + ) + + government_2 = Government.objects.create(name="Western Cape", sphere=sphere) + department_2 = Department.objects.create( + name="Education", + government=government_2, + vote_number=1, + ) + Indicator.objects.create( + indicator_name="Test Indicator 2", + department=department_2, + source=EQPRSFileUpload.objects.create(user=self.superuser, file=None), + ) + + department_3 = Department.objects.create( + name="Health", + government=government_2, + vote_number=2, + ) + Indicator.objects.create( + indicator_name="Test Indicator 3", + department=department_3, + source=EQPRSFileUpload.objects.create(user=self.superuser, file=None), + ) + + assert Indicator.objects.all().count() == 3 + + test_element = EQPRSFileUpload.objects.create( + user=self.superuser, file=self.data_for_deleting_indicators_file + ) + performance.admin.save_imported_indicators(test_element.id) + test_element.refresh_from_db() + + # ids 1 & 2 are deleted + assert Indicator.objects.all().count() == 3 + + # id 3 is remaining + indicator_3 = Indicator.objects.get(id=3) + assert indicator_3.indicator_name == "Test Indicator 3" + assert indicator_3.department.name == "Health" + assert indicator_3.department.government.name == "Western Cape" + assert indicator_3.department.government.sphere.financial_year.slug == "2017-18" + + # new objects(with id 4 & 5) are created + indicator_4 = Indicator.objects.get(id=4) + assert ( + indicator_4.indicator_name + == "9.1.2 Number of statutory documents tabled at Legislature" + ) + assert indicator_4.department.name == "Health" + assert indicator_4.department.government.name == "Eastern Cape" + assert indicator_4.department.government.sphere.financial_year.slug == "2017-18" + + indicator_5 = Indicator.objects.get(id=5) + assert ( + indicator_5.indicator_name + == "6.4.1 Holistic Human Resources for Health (HRH) strategy approved " + ) + assert indicator_5.department.name == "Education" + assert indicator_5.department.government.name == "Western Cape" + assert indicator_5.department.government.sphere.financial_year.slug == "2017-18" + + def test_department_name_containing_government(self): + fy = FinancialYear.objects.create(slug="2021-22") + sphere = Sphere.objects.create(name="Provincial", financial_year=fy) + government = Government.objects.create(name="Eastern Cape", sphere=sphere) + department = Department.objects.create( + name="Health", government=government, vote_number=1 + ) + + test_element = EQPRSFileUpload.objects.create( + user=self.superuser, file=self.department_name_containing_government_file + ) + performance.admin.save_imported_indicators(test_element.id) + test_element.refresh_from_db() + assert Indicator.objects.all().count() == 2 diff --git a/performance/urls.py b/performance/urls.py new file mode 100644 index 000000000..9dff46f91 --- /dev/null +++ b/performance/urls.py @@ -0,0 +1,17 @@ +from django.contrib import admin +from django.conf.urls import url +from rest_framework.routers import DefaultRouter +from django.urls import path, include +from performance import views + + +urlpatterns = [ + # Performance + path("", views.performance_tabular_view, name="performance"), + path(r"api/v1/eqprs/", views.IndicatorListView.as_view()), + path( + r"performance-indicators.xlsx/", + views.IndicatorXLSXListView.as_view(), + name="performance-indicators-xlsx", + ), +] diff --git a/performance/views.py b/performance/views.py new file mode 100644 index 000000000..91fdae55c --- /dev/null +++ b/performance/views.py @@ -0,0 +1,163 @@ +from django.shortcuts import render +from .models import Indicator +from .serializer import IndicatorSerializer +from rest_framework import generics +from django.db.models import Count, Q +from rest_framework.pagination import PageNumberPagination +from budgetportal.models import MainMenuItem +from django.contrib.postgres.search import SearchQuery +from drf_excel.mixins import XLSXFileMixin +from django.http import StreamingHttpResponse + +import xlsx_streaming + +FIELD_MAP = { + "department_name": "department__name", + "financial_year_slug": "department__government__sphere__financial_year__slug", + "government_name": "department__government__name", + "sphere_name": "department__government__sphere__name", + "frequency": "frequency", + "mtsf_outcome": "mtsf_outcome", + "sector": "sector", +} + +XLSX_COLUMNS = [ + "department__government__sphere__financial_year__slug", + "department__government__sphere__name", + "department__government__name", + "department__name", + "programme_name", + "subprogramme_name", + "indicator_name", + "frequency", + "q1_target", + "q1_actual_output", + "q1_deviation_reason", + "q1_corrective_action", + "q2_target", + "q2_actual_output", + "q2_deviation_reason", + "q2_corrective_action", + "q3_target", + "q3_actual_output", + "q3_deviation_reason", + "q3_corrective_action", + "q4_target", + "q4_actual_output", + "q4_deviation_reason", + "q4_corrective_action", + "annual_target", + "annual_aggregate_output", + "annual_pre_audit_output", + "annual_deviation_reason", + "annual_corrective_action", + "annual_audited_output", + "sector", + "type", + "subtype", + "mtsf_outcome", + "cluster", +] + + +def performance_tabular_view(request): + context = { + "navbar": MainMenuItem.objects.prefetch_related("children").all(), + "title": "Quarterly performance reporting (QPR) indicators", + "description": "Find the latest quarterly performance monitoring indicators, results, and explanations from national and provincial departments. How is performance measured in government? Quarterly and audited annual indicators is one of the tools to monitor implementation of department mandates.", + } + return render(request, "performance/performance.html", context) + + +def text_search(qs, text): + if len(text) == 0: + return qs + + return qs.filter( + Q(content_search=SearchQuery(text)) + | Q(content_search__icontains=text) + | Q(indicator_name__icontains=text) + ) + + +def add_filters(qs, params): + query_dict = {} + for k, v in FIELD_MAP.items(): + if v in params: + query_dict[v] = params[v] + + return qs.filter(**query_dict) + + +def get_filtered_queryset(queryset, request): + search_query = request.GET.get("q", "") + + filtered_queryset = queryset.select_related( + "department", + "department__government", + "department__government__sphere", + "department__government__sphere__financial_year", + ) + filtered_queryset = text_search(filtered_queryset, search_query) + filtered_queryset = add_filters(filtered_queryset, request.GET) + + return filtered_queryset + + +class IndicatorListView(generics.ListAPIView): + serializer_class = IndicatorSerializer + queryset = Indicator.objects.all() + pagination_class = PageNumberPagination + + def list(self, request): + queryset = get_filtered_queryset(self.get_queryset(), request) + + facets = self.get_facets(queryset) + + queryset = self.paginate_queryset(queryset) + serializer_class = self.get_serializer_class() + serializer = serializer_class(queryset, many=True) + data = { + "items": serializer.data, + "facets": facets, + } + return self.get_paginated_response(data) + + def get_facets(self, qs): + def facet_query(field): + return qs.values(field).annotate(count=Count(field)) + + return { + "department_name": facet_query("department__name"), + "financial_year_slug": facet_query( + "department__government__sphere__financial_year__slug" + ), + "government_name": facet_query("department__government__name"), + "sphere_name": facet_query("department__government__sphere__name"), + "frequency": facet_query("frequency"), + "mtsf_outcome": facet_query("mtsf_outcome"), + "sector": facet_query("sector"), + } + + +class IndicatorXLSXListView(XLSXFileMixin, generics.ListAPIView): + pagination_class = None + template_filename = "performance/template.xlsx" + filename = "eqprs-indicators.xlsx" + queryset = Indicator.objects.all() + + def list(self, request, *args, **kwargs): + excel_data = get_filtered_queryset(self.get_queryset(), request) + + with open(self.template_filename, "rb") as template: + stream = xlsx_streaming.stream_queryset_as_xlsx( + self.filter_queryset(excel_data).values_list(*XLSX_COLUMNS), + xlsx_template=template, + batch_size=50, + ) + response = StreamingHttpResponse( + stream, + content_type="application/vnd.xlsxformats-officedocument.spreadsheetml.sheet", + ) + response["Content-Disposition"] = f"attachment; filename={self.filename}" + return response diff --git a/requirements-test.txt b/requirements-test.txt index 244241f28..e69de29bb 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +0,0 @@ -codecov==2.0.15 -mock==4.0.1 -selenium==3.141.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c10a8bf0c..367853d85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,31 @@ arrow==0.15.5 backports.csv==1.0.7 backports.functools-lru-cache==1.6.1 +black==22.3.0 blessed==1.17.2 boto3==1.12.21 cachetools==4.0.0 -certifi==2019.11.28 +certifi==2022.12.7 chardet==3.0.4 ckanapi==4.3 defusedxml==0.6.0 diff-match-patch==20181111 dj-database-url==0.5.0 -Django==2.2.20 +Django==2.2.28 django-admin-sortable==2.2.3 django-adminplus==0.5 django-allauth==0.41.0 django-autoslug==1.9.6 django-ckeditor==5.9.0 +django-constance==2.9.1 django-debug-toolbar==2.2 -django-extensions==2.2.8 +django-extensions==3.1.5 django-environ==0.4.5 django-filter==2.2.0 django-haystack==2.8.1 django-import-export==2.0.2 django-js-asset==1.2.2 -django-markdownify==0.8.0 +django-markdownify==0.9.2 django-partial-index==0.6.0 django-picklefield==2.1.1 django-pipeline==1.7.0 @@ -34,7 +36,9 @@ djangorestframework-csv==2.1.0 django-storages==1.9.1 docopt==0.6.2 drf-haystack==1.8.9 +drf-excel==1.0.0 et-xmlfile==1.0.1 +frictionless[json]==4.40.8 futures==3.1.1 gevent==1.4.0 greenlet==0.4.15 @@ -48,7 +52,7 @@ openpyxl==3.0.3 pathlib==1.0.1 psycopg2==2.8.4 pyaml==19.12.0 -pyScss==1.3.5 +pyScss==1.3.7 pysolr==3.8.1 python-dateutil==2.8.1 python-slugify==4.0.0 @@ -66,7 +70,10 @@ Unidecode==1.1.1 urllib3==1.25.8 wagtail==2.7.4 wcwidth==0.1.8 -Werkzeug==1.0.0 +Werkzeug==2.0.3 whitenoise==5.0.1 xlrd==1.2.0 +xlsx-streaming==1.1.0 xlwt==1.3.0 +mock==4.0.1 +selenium==3.141.0 diff --git a/runtime.txt b/runtime.txt index 6919bf9ed..91b17a136 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.7.6 +python-3.7.16 diff --git a/webpack.config.js b/webpack.config.js index 224fb879c..f4f801e62 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -27,7 +27,9 @@ module.exports = { { test: /\.jsx?$/, loader: 'babel-loader', - query: { compact: false }, + options: { + presets: ['react'] + } }, { @@ -52,5 +54,5 @@ module.exports = { plugins: [ new MiniCssExtractPlugin({filename: 'styles.bundle.css'}), - ], + ] }; diff --git a/yarn.lock b/yarn.lock index 80e11e259..9359af09d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -914,6 +914,13 @@ dependencies: regenerator-runtime "^0.13.2" +"@babel/runtime@^7.3.1", "@babel/runtime@^7.4.4", "@babel/runtime@^7.5.5", "@babel/runtime@^7.8.3", "@babel/runtime@^7.8.7": + version "7.20.13" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.20.13.tgz#7055ab8a7cff2b8f6058bf6ae45ff84ad2aded4b" + integrity sha512-gt3PKXs0DBoL9xCvOIIZ2NEqAGZqHjAnmVbfQtB620V0uReIQutpel14KcneZuer7UioY8ALKZ7iocavvzTNFA== + dependencies: + regenerator-runtime "^0.13.11" + "@babel/template@^7.1.0", "@babel/template@^7.2.2", "@babel/template@^7.4.0", "@babel/template@^7.4.4": version "7.4.4" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.4.4.tgz#f4b88d1225689a08f5bc3a17483545be9e4ed237" @@ -1012,6 +1019,11 @@ resolved "https://registry.yarnpkg.com/@emotion/hash/-/hash-0.6.6.tgz#62266c5f0eac6941fece302abad69f2ee7e25e44" integrity sha512-ojhgxzUHZ7am3D2jHkMzPpsBAiB005GF5YU4ea+8DNPybMk01JJUM9V9YRlF/GE95tcOm8DxQvWA2jq19bGalQ== +"@emotion/hash@^0.8.0": + version "0.8.0" + resolved "https://registry.yarnpkg.com/@emotion/hash/-/hash-0.8.0.tgz#bbbff68978fefdbe68ccb533bc8cbe1d1afb5413" + integrity sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow== + "@emotion/is-prop-valid@0.7.3", "@emotion/is-prop-valid@^0.7.3": version "0.7.3" resolved "https://registry.yarnpkg.com/@emotion/is-prop-valid/-/is-prop-valid-0.7.3.tgz#a6bf4fa5387cbba59d44e698a4680f481a8da6cc" @@ -1292,6 +1304,24 @@ recompose "0.28.0 - 0.30.0" warning "^4.0.1" +"@material-ui/core@^4.12.4": + version "4.12.4" + resolved "https://registry.yarnpkg.com/@material-ui/core/-/core-4.12.4.tgz#4ac17488e8fcaf55eb6a7f5efb2a131e10138a73" + integrity sha512-tr7xekNlM9LjA6pagJmL8QCgZXaubWUwkJnoYcMKd4gw/t4XiyvnTkjdGrUVicyB2BsdaAv1tvow45bPM4sSwQ== + dependencies: + "@babel/runtime" "^7.4.4" + "@material-ui/styles" "^4.11.5" + "@material-ui/system" "^4.12.2" + "@material-ui/types" "5.1.0" + "@material-ui/utils" "^4.11.3" + "@types/react-transition-group" "^4.2.0" + clsx "^1.0.4" + hoist-non-react-statics "^3.3.2" + popper.js "1.16.1-lts" + prop-types "^15.7.2" + react-is "^16.8.0 || ^17.0.0" + react-transition-group "^4.4.0" + "@material-ui/icons@^3.0.2": version "3.0.2" resolved "https://registry.yarnpkg.com/@material-ui/icons/-/icons-3.0.2.tgz#d67a6dd1ec8312d3a88ec97944a63daeef24fe10" @@ -1300,6 +1330,13 @@ "@babel/runtime" "^7.2.0" recompose "0.28.0 - 0.30.0" +"@material-ui/icons@^4.11.3": + version "4.11.3" + resolved "https://registry.yarnpkg.com/@material-ui/icons/-/icons-4.11.3.tgz#b0693709f9b161ce9ccde276a770d968484ecff1" + integrity sha512-IKHlyx6LDh8n19vzwH5RtHIOHl9Tu90aAAxcbWME6kp4dmvODM3UvOHJeMIDzUbd4muuJKHmlNoBN+mDY4XkBA== + dependencies: + "@babel/runtime" "^7.4.4" + "@material-ui/lab@^3.0.0-alpha.30": version "3.0.0-alpha.30" resolved "https://registry.yarnpkg.com/@material-ui/lab/-/lab-3.0.0-alpha.30.tgz#c6c64d0ff2b28410a09e4009f3677499461f3df8" @@ -1311,6 +1348,28 @@ keycode "^2.1.9" prop-types "^15.6.0" +"@material-ui/styles@^4.11.5": + version "4.11.5" + resolved "https://registry.yarnpkg.com/@material-ui/styles/-/styles-4.11.5.tgz#19f84457df3aafd956ac863dbe156b1d88e2bbfb" + integrity sha512-o/41ot5JJiUsIETME9wVLAJrmIWL3j0R0Bj2kCOLbSfqEkKf0fmaPt+5vtblUh5eXr2S+J/8J3DaCb10+CzPGA== + dependencies: + "@babel/runtime" "^7.4.4" + "@emotion/hash" "^0.8.0" + "@material-ui/types" "5.1.0" + "@material-ui/utils" "^4.11.3" + clsx "^1.0.4" + csstype "^2.5.2" + hoist-non-react-statics "^3.3.2" + jss "^10.5.1" + jss-plugin-camel-case "^10.5.1" + jss-plugin-default-unit "^10.5.1" + jss-plugin-global "^10.5.1" + jss-plugin-nested "^10.5.1" + jss-plugin-props-sort "^10.5.1" + jss-plugin-rule-value-function "^10.5.1" + jss-plugin-vendor-prefixer "^10.5.1" + prop-types "^15.7.2" + "@material-ui/system@^3.0.0-alpha.0": version "3.0.0-alpha.2" resolved "https://registry.yarnpkg.com/@material-ui/system/-/system-3.0.0-alpha.2.tgz#096e80c8bb0f70aea435b9e38ea7749ee77b4e46" @@ -1321,6 +1380,21 @@ prop-types "^15.6.0" warning "^4.0.1" +"@material-ui/system@^4.12.2": + version "4.12.2" + resolved "https://registry.yarnpkg.com/@material-ui/system/-/system-4.12.2.tgz#f5c389adf3fce4146edd489bf4082d461d86aa8b" + integrity sha512-6CSKu2MtmiJgcCGf6nBQpM8fLkuB9F55EKfbdTC80NND5wpTmKzwdhLYLH3zL4cLlK0gVaaltW7/wMuyTnN0Lw== + dependencies: + "@babel/runtime" "^7.4.4" + "@material-ui/utils" "^4.11.3" + csstype "^2.5.2" + prop-types "^15.7.2" + +"@material-ui/types@5.1.0": + version "5.1.0" + resolved "https://registry.yarnpkg.com/@material-ui/types/-/types-5.1.0.tgz#efa1c7a0b0eaa4c7c87ac0390445f0f88b0d88f2" + integrity sha512-7cqRjrY50b8QzRSYyhSpx4WRw2YuO0KKIGQEVk5J8uoz2BanawykgZGoWEqKm7pVIbzFDN0SpPcVV4IhOFkl8A== + "@material-ui/utils@^3.0.0-alpha.2": version "3.0.0-alpha.3" resolved "https://registry.yarnpkg.com/@material-ui/utils/-/utils-3.0.0-alpha.3.tgz#836c62ea46f5ffc6f0b5ea05ab814704a86908b1" @@ -1330,6 +1404,15 @@ prop-types "^15.6.0" react-is "^16.6.3" +"@material-ui/utils@^4.11.3": + version "4.11.3" + resolved "https://registry.yarnpkg.com/@material-ui/utils/-/utils-4.11.3.tgz#232bd86c4ea81dab714f21edad70b7fdf0253942" + integrity sha512-ZuQPV4rBK/V1j2dIkSSEcH5uT6AaHuKWFfotADHsC0wVL1NLd2WkFCm4ZZbX33iO4ydl6V0GPngKm8HZQ2oujg== + dependencies: + "@babel/runtime" "^7.4.4" + prop-types "^15.7.2" + react-is "^16.8.0 || ^17.0.0" + "@mrmlnc/readdir-enhanced@^2.2.1": version "2.2.1" resolved "https://registry.yarnpkg.com/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz#524af240d1a360527b730475ecfa1344aa540dde" @@ -1946,6 +2029,13 @@ dependencies: "@types/react" "*" +"@types/react-transition-group@^4.2.0": + version "4.4.5" + resolved "https://registry.yarnpkg.com/@types/react-transition-group/-/react-transition-group-4.4.5.tgz#aae20dcf773c5aa275d5b9f7cdbca638abc5e416" + integrity sha512-juKD/eiSM3/xZYzjuzH6ZwpP+/lejltmiS3QEzV/vmb/Q8+HfDmxu+Baga8UEMGBqV88Nbg4l2hY/K2DkyaLLA== + dependencies: + "@types/react" "*" + "@types/react@*": version "16.8.17" resolved "https://registry.yarnpkg.com/@types/react/-/react-16.8.17.tgz#f287b76a5badb93bc9aa3f54521a3eb53d6c2374" @@ -4928,6 +5018,11 @@ clone@^1.0.2: resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e" integrity sha1-2jCcwmPfFZlMaIypAheco8fNfH4= +clsx@^1.0.4: + version "1.2.1" + resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.2.1.tgz#0ddc4a20a549b59c93a4116bb26f5294ca17dc12" + integrity sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg== + co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" @@ -5632,6 +5727,14 @@ css-vendor@^0.3.8: dependencies: is-in-browser "^1.0.2" +css-vendor@^2.0.8: + version "2.0.8" + resolved "https://registry.yarnpkg.com/css-vendor/-/css-vendor-2.0.8.tgz#e47f91d3bd3117d49180a3c935e62e3d9f7f449d" + integrity sha512-x9Aq0XTInxrkuFeHKbYC7zWY8ai7qJ04Kxd9MnvbC1uO5DagxoHQjm4JvG+vCdXOoFtCjbL2XSZfxmoYa9uQVQ== + dependencies: + "@babel/runtime" "^7.8.3" + is-in-browser "^1.0.2" + css-what@2.1, css-what@^2.1.2: version "2.1.3" resolved "https://registry.yarnpkg.com/css-what/-/css-what-2.1.3.tgz#a6d7604573365fe74686c3f311c56513d88285f2" @@ -5812,6 +5915,11 @@ csstype@^2.0.0, csstype@^2.2.0, csstype@^2.5.2, csstype@^2.5.7: resolved "https://registry.yarnpkg.com/csstype/-/csstype-2.6.4.tgz#d585a6062096e324e7187f80e04f92bd0f00e37f" integrity sha512-lAJUJP3M6HxFXbqtGRc0iZrdyeN+WzOWeY0q/VnFzI+kqVrYIzC7bWlKqCW7oCIdzoPkvfp82EVvrTlQ8zsWQg== +csstype@^3.0.2: + version "3.1.1" + resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.1.tgz#841b532c45c758ee546a11d5bd7b7b473c8c30b9" + integrity sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw== + currently-unhandled@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea" @@ -6045,9 +6153,9 @@ decimal.js@^10.2.0: integrity sha512-KaL7+6Fw6i5A2XSnsbhm/6B+NuEA7TZ4vqxnd5tXz9sbKtrN9Srj8ab4vKVdK8YAqZO9P1kg45Y6YLoduPf+kw== decode-uri-component@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" - integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU= + version "0.2.2" + resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.2.tgz#e69dbe25d37941171dd540e024c444cd5188e1e9" + integrity sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ== deep-equal@^1.0.1: version "1.0.1" @@ -6303,6 +6411,14 @@ dom-helpers@^3.2.1, dom-helpers@^3.4.0: dependencies: "@babel/runtime" "^7.1.2" +dom-helpers@^5.0.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/dom-helpers/-/dom-helpers-5.2.1.tgz#d9400536b2bf8225ad98fe052e029451ac40e902" + integrity sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA== + dependencies: + "@babel/runtime" "^7.8.7" + csstype "^3.0.2" + dom-serializer@0: version "0.1.1" resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0" @@ -8286,6 +8402,13 @@ hoist-non-react-statics@^3.2.1, hoist-non-react-statics@^3.3.0: dependencies: react-is "^16.7.0" +hoist-non-react-statics@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45" + integrity sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw== + dependencies: + react-is "^16.7.0" + home-or-tmp@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/home-or-tmp/-/home-or-tmp-2.0.0.tgz#e36c3f2d2cae7d746a857e38d18d5f32a7882db8" @@ -8508,6 +8631,11 @@ hyphenate-style-name@^1.0.0, hyphenate-style-name@^1.0.2: resolved "https://registry.yarnpkg.com/hyphenate-style-name/-/hyphenate-style-name-1.0.3.tgz#097bb7fa0b8f1a9cf0bd5c734cf95899981a9b48" integrity sha512-EcuixamT82oplpoJ2XU4pDtKGWQ7b00CD9f1ug9IaQ3p1bkHMiKCZ9ut9QDI6qsa6cpUuB+A/I+zLtdNK4n2DQ== +hyphenate-style-name@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/hyphenate-style-name/-/hyphenate-style-name-1.0.4.tgz#691879af8e220aea5750e8827db4ef62a54e361d" + integrity sha512-ygGZLjmXfPHj+ZWh6LwbC37l43MhfztxetbFCoYTM2VjkIUpeHgSNn7QIyVFj7YQ1Wl9Cbw5sholVJPzWvC2MQ== + iconv-lite@0.4.23: version "0.4.23" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63" @@ -9933,6 +10061,66 @@ jss-nested@^6.0.1: dependencies: warning "^3.0.0" +jss-plugin-camel-case@^10.5.1: + version "10.9.2" + resolved "https://registry.yarnpkg.com/jss-plugin-camel-case/-/jss-plugin-camel-case-10.9.2.tgz#76dddfa32f9e62d17daa4e3504991fd0933b89e1" + integrity sha512-wgBPlL3WS0WDJ1lPJcgjux/SHnDuu7opmgQKSraKs4z8dCCyYMx9IDPFKBXQ8Q5dVYij1FFV0WdxyhuOOAXuTg== + dependencies: + "@babel/runtime" "^7.3.1" + hyphenate-style-name "^1.0.3" + jss "10.9.2" + +jss-plugin-default-unit@^10.5.1: + version "10.9.2" + resolved "https://registry.yarnpkg.com/jss-plugin-default-unit/-/jss-plugin-default-unit-10.9.2.tgz#3e7f4a1506b18d8fe231554fd982439feb2a9c53" + integrity sha512-pYg0QX3bBEFtTnmeSI3l7ad1vtHU42YEEpgW7pmIh+9pkWNWb5dwS/4onSfAaI0kq+dOZHzz4dWe+8vWnanoSg== + dependencies: + "@babel/runtime" "^7.3.1" + jss "10.9.2" + +jss-plugin-global@^10.5.1: + version "10.9.2" + resolved "https://registry.yarnpkg.com/jss-plugin-global/-/jss-plugin-global-10.9.2.tgz#e7f2ad4a5e8e674fb703b04b57a570b8c3e5c2c2" + integrity sha512-GcX0aE8Ef6AtlasVrafg1DItlL/tWHoC4cGir4r3gegbWwF5ZOBYhx04gurPvWHC8F873aEGqge7C17xpwmp2g== + dependencies: + "@babel/runtime" "^7.3.1" + jss "10.9.2" + +jss-plugin-nested@^10.5.1: + version "10.9.2" + resolved "https://registry.yarnpkg.com/jss-plugin-nested/-/jss-plugin-nested-10.9.2.tgz#3aa2502816089ecf3981e1a07c49b276d67dca63" + integrity sha512-VgiOWIC6bvgDaAL97XCxGD0BxOKM0K0zeB/ECyNaVF6FqvdGB9KBBWRdy2STYAss4VVA7i5TbxFZN+WSX1kfQA== + dependencies: + "@babel/runtime" "^7.3.1" + jss "10.9.2" + tiny-warning "^1.0.2" + +jss-plugin-props-sort@^10.5.1: + version "10.9.2" + resolved "https://registry.yarnpkg.com/jss-plugin-props-sort/-/jss-plugin-props-sort-10.9.2.tgz#645f6c8f179309667b3e6212f66b59a32fb3f01f" + integrity sha512-AP1AyUTbi2szylgr+O0OB7gkIxEGzySLITZ2GpsaoX72YMCGI2jYAc+WUhPfvUnZYiauF4zTnN4V4TGuvFjJlw== + dependencies: + "@babel/runtime" "^7.3.1" + jss "10.9.2" + +jss-plugin-rule-value-function@^10.5.1: + version "10.9.2" + resolved "https://registry.yarnpkg.com/jss-plugin-rule-value-function/-/jss-plugin-rule-value-function-10.9.2.tgz#9afe07596e477123cbf11120776be6a64494541f" + integrity sha512-vf5ms8zvLFMub6swbNxvzsurHfUZ5Shy5aJB2gIpY6WNA3uLinEcxYyraQXItRHi5ivXGqYciFDRM2ZoVoRZ4Q== + dependencies: + "@babel/runtime" "^7.3.1" + jss "10.9.2" + tiny-warning "^1.0.2" + +jss-plugin-vendor-prefixer@^10.5.1: + version "10.9.2" + resolved "https://registry.yarnpkg.com/jss-plugin-vendor-prefixer/-/jss-plugin-vendor-prefixer-10.9.2.tgz#410a0f3b9f8dbbfba58f4d329134df4849aa1237" + integrity sha512-SxcEoH+Rttf9fEv6KkiPzLdXRmI6waOTcMkbbEFgdZLDYNIP9UKNHFy6thhbRKqv0XMQZdrEsbDyV464zE/dUA== + dependencies: + "@babel/runtime" "^7.3.1" + css-vendor "^2.0.8" + jss "10.9.2" + jss-props-sort@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/jss-props-sort/-/jss-props-sort-6.0.0.tgz#9105101a3b5071fab61e2d85ea74cc22e9b16323" @@ -9945,6 +10133,16 @@ jss-vendor-prefixer@^7.0.0: dependencies: css-vendor "^0.3.8" +jss@10.9.2, jss@^10.5.1: + version "10.9.2" + resolved "https://registry.yarnpkg.com/jss/-/jss-10.9.2.tgz#9379be1f195ef98011dfd31f9448251bd61b95a9" + integrity sha512-b8G6rWpYLR4teTUbGd4I4EsnWjg7MN0Q5bSsjKhVkJVjhQDy2KzkbD2AW3TuT0RYZVmZZHKIrXDn6kjU14qkUg== + dependencies: + "@babel/runtime" "^7.3.1" + csstype "^3.0.2" + is-in-browser "^1.1.3" + tiny-warning "^1.0.2" + jss@^9.8.7: version "9.8.7" resolved "https://registry.yarnpkg.com/jss/-/jss-9.8.7.tgz#ed9763fc0f2f0260fc8260dac657af61e622ce05" @@ -10222,7 +10420,7 @@ lodash.camelcase@^4.3.0: lodash.debounce@^4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" - integrity sha1-gteb/zCmfEAF/9XiUVMArZyk168= + integrity sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow== lodash.deburr@^4.1.0: version "4.1.0" @@ -10232,7 +10430,7 @@ lodash.deburr@^4.1.0: lodash.flow@^3.1.0: version "3.5.0" resolved "https://registry.yarnpkg.com/lodash.flow/-/lodash.flow-3.5.0.tgz#87bf40292b8cf83e4e8ce1a3ae4209e20071675a" - integrity sha1-h79AKSuM+D5OjOGjrkIJ4gBxZ1o= + integrity sha512-ff3BX/tSioo+XojX4MOsOMhJw0nZoUEF011LX8g8d3gvjVbxd89cCio4BCXronjxcTUIJUoqKEUA+n4CqvvRPw== lodash.get@^4.4.2: version "4.4.2" @@ -11932,6 +12130,11 @@ popmotion@^8.6.2: stylefire "^4.1.3" tslib "^1.9.1" +popper.js@1.16.1-lts: + version "1.16.1-lts" + resolved "https://registry.yarnpkg.com/popper.js/-/popper.js-1.16.1-lts.tgz#cf6847b807da3799d80ee3d6d2f90df8a3f50b05" + integrity sha512-Kjw8nKRl1m+VrSFCoVGPph93W/qrSO7ZkqPpTf7F4bk/sqcfWK019dWBUpE/fBOsOQY1dks/Bmcbfn1heM/IsA== + popper.js@^1.14.1, popper.js@^1.14.4: version "1.15.0" resolved "https://registry.yarnpkg.com/popper.js/-/popper.js-1.15.0.tgz#5560b99bbad7647e9faa475c6b8056621f5a4ff2" @@ -12951,28 +13154,6 @@ postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.14, postcss@^7.0.2, postcss@^7.0.4, source-map "^0.6.1" supports-color "^6.1.0" -preact-css-transition-group@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/preact-css-transition-group/-/preact-css-transition-group-1.3.0.tgz#06fe468b26f7802e95b829a762db0bc199aef399" - integrity sha1-Bv5Giyb3gC6VuCmnYtsLwZmu85k= - -preact-render-to-string@^3.8.2: - version "3.8.2" - resolved "https://registry.yarnpkg.com/preact-render-to-string/-/preact-render-to-string-3.8.2.tgz#bd72964d705a57da3a9e72098acaa073dd3ceff9" - integrity sha512-przuZPajiurStGgxMoJP0EJeC4xj5CgHv+M7GfF3YxAdhGgEWAkhOSE0xympAFN20uMayntBZpttIZqqLl77fw== - dependencies: - pretty-format "^3.5.1" - -preact-transition-group@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/preact-transition-group/-/preact-transition-group-1.1.1.tgz#f0a49327ea515ece34ea2be864c4a7d29e5d6e10" - integrity sha1-8KSTJ+pRXs406ivoZMSn0p5dbhA= - -preact@^8.3.1: - version "8.4.2" - resolved "https://registry.yarnpkg.com/preact/-/preact-8.4.2.tgz#1263b974a17d1ea80b66590e41ef786ced5d6a23" - integrity sha512-TsINETWiisfB6RTk0wh3/mvxbGRvx+ljeBccZ4Z6MPFKgu/KFGyf2Bmw3Z/jlXhL5JlNKY6QAbA9PVyzIy9//A== - prelude-ls@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" @@ -13018,11 +13199,6 @@ pretty-format@^24.8.0: ansi-styles "^3.2.0" react-is "^16.8.4" -pretty-format@^3.5.1: - version "3.8.0" - resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-3.8.0.tgz#bfbed56d5e9a776645f4b1ff7aa1a3ac4fa3c385" - integrity sha1-v77VbV6ad2ZF9LH/eqGjrE+jw4U= - pretty-hrtime@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz#b7e3ea42435a4c9b2759d99e0f201eb195802ee1" @@ -13563,11 +13739,21 @@ react-is@^16.6.0, react-is@^16.6.3, react-is@^16.7.0, react-is@^16.8.1, react-is resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.8.6.tgz#5bbc1e2d29141c9fbdfed456343fe2bc430a6a16" integrity sha512-aUk3bHfZ2bRSVFFbbeVS4i+lNPZr3/WM5jT2J5omUVV1zzcs1nAaf3l51ctA5FFvCRbhrH0bdAsRRQddFJZPtA== +"react-is@^16.8.0 || ^17.0.0": + version "17.0.2" + resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" + integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== + react-lifecycles-compat@^3.0.0, react-lifecycles-compat@^3.0.2, react-lifecycles-compat@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz#4f1a273afdfc8f3488a8c516bfda78f872352362" integrity sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA== +react-lines-ellipsis@^0.15.3: + version "0.15.3" + resolved "https://registry.yarnpkg.com/react-lines-ellipsis/-/react-lines-ellipsis-0.15.3.tgz#eb081c12e58170b09c5bd9db2fdbde9b5a3ea6c0" + integrity sha512-+kIcUZJPI3QqazZbO0eeEUmfRjmYKjCRzSmXQL4otrJzwOU9MjvASGQd4SELOOBSy6oiygjfDuCocGMssX0oUQ== + react-markdown@^4.0.6: version "4.0.8" resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-4.0.8.tgz#e3621b5becaac82a651008d7bc8390d3e4e438c0" @@ -13750,6 +13936,16 @@ react-transition-group@^2.2.1, react-transition-group@^2.5.0: prop-types "^15.6.2" react-lifecycles-compat "^3.0.4" +react-transition-group@^4.4.0: + version "4.4.5" + resolved "https://registry.yarnpkg.com/react-transition-group/-/react-transition-group-4.4.5.tgz#e53d4e3f3344da8521489fbef8f2581d42becdd1" + integrity sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g== + dependencies: + "@babel/runtime" "^7.5.5" + dom-helpers "^5.0.1" + loose-envify "^1.4.0" + prop-types "^15.6.2" + react-url-query@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/react-url-query/-/react-url-query-1.4.0.tgz#e0e639e25ec63e28c548140f4b12fba4c10cfcb2" @@ -14062,6 +14258,11 @@ regenerator-runtime@^0.12.0, regenerator-runtime@^0.12.1: resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.12.1.tgz#fa1a71544764c036f8c49b13a08b2594c9f8a0de" integrity sha512-odxIc1/vDlo4iZcfXqRYFj0vpXFNoGdKMAUieAlFYO6m/nl5e9KR/beGf41z4a1FI+aQgtjhuaSlDxQ0hmkrHg== +regenerator-runtime@^0.13.11: + version "0.13.11" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" + integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== + regenerator-runtime@^0.13.2: version "0.13.2" resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.2.tgz#32e59c9a6fb9b1a4aff09b4930ca2d4477343447" @@ -16064,6 +16265,11 @@ tiny-warning@^1.0.0: resolved "https://registry.yarnpkg.com/tiny-warning/-/tiny-warning-1.0.2.tgz#1dfae771ee1a04396bdfde27a3adcebc6b648b28" integrity sha512-rru86D9CpQRLvsFG5XFdy0KdLAvjdQDyZCsRcuu60WtzFylDM3eAWSxEVz5kzL2Gp544XiUvPbVKtOA/txLi9Q== +tiny-warning@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/tiny-warning/-/tiny-warning-1.0.3.tgz#94a30db453df4c643d0fd566060d60a875d84754" + integrity sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA== + tinycolor2@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/tinycolor2/-/tinycolor2-1.4.1.tgz#f4fad333447bc0b07d4dc8e9209d8f39a8ac77e8"