Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix more sqlalchemy #1417

Open
wants to merge 111 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 83 commits
Commits
Show all changes
111 commits
Select commit Hold shift + click to select a range
6d8fdab
fix more sqlalchemy
Nov 14, 2024
7b83ea3
fix more
Nov 14, 2024
310386a
hmmm
Nov 14, 2024
5276ba4
hmmm
Nov 14, 2024
441b75d
hmmm
Nov 14, 2024
b708498
hmmm
Nov 14, 2024
f16c814
revert one test
Nov 14, 2024
0177fdc
hmmm
Nov 14, 2024
8c7cb44
more tests
Nov 15, 2024
b0bf042
more
Nov 15, 2024
704fff7
more
Nov 15, 2024
8f211cc
more
Nov 15, 2024
d1ebcba
more
Nov 15, 2024
1311598
more
Nov 15, 2024
3673a92
fix more
Nov 15, 2024
ee00100
fix tests
Nov 15, 2024
5c6a5f9
fix tests
Nov 15, 2024
5f4da49
more
Nov 15, 2024
98ee86f
more
Nov 15, 2024
703a29f
more
Nov 15, 2024
1f0a64d
more
Nov 15, 2024
555cf5d
more
Nov 15, 2024
3eadfb2
fix style
Nov 15, 2024
bc4d4c9
more
Nov 15, 2024
5b667a1
more
Nov 18, 2024
6c44f81
more
Nov 18, 2024
2eb692a
more
Nov 18, 2024
4ef1847
fix permission_dao
Nov 18, 2024
3168f28
more
Nov 18, 2024
4f3b99b
more
Nov 18, 2024
c77382c
more
Nov 18, 2024
da57887
fix
Nov 18, 2024
2c3c107
revert bad changes
Nov 18, 2024
ce42803
fix
Nov 18, 2024
92bf9c5
more
Nov 18, 2024
25d2901
try fixing pagination
Nov 18, 2024
4539e5c
try fixing pagination
Nov 18, 2024
71e4794
try fixing pagination
Nov 18, 2024
552d464
try fixing pagination
Nov 18, 2024
e7abb06
try fixing pagination
Nov 18, 2024
83a7df6
try fixing pagination
Nov 18, 2024
c3de127
try fixing pagination
Nov 18, 2024
d513c5e
debug
Nov 18, 2024
14bfcd3
debug
Nov 18, 2024
d2d5fa7
fix
Nov 18, 2024
c92a760
fix
Nov 18, 2024
4078a7e
fix
Nov 18, 2024
53f3948
fix
Nov 18, 2024
ce5b9cf
fix
Nov 18, 2024
38e7672
fix
Nov 18, 2024
43a1969
fix
Nov 18, 2024
2abb14d
fix
Nov 18, 2024
84ebf11
fix
Nov 18, 2024
27c9885
fix
Nov 18, 2024
f9cf3f3
fix scheduled tasks
Nov 19, 2024
b33e2ca
fix scheduled tasks
Nov 19, 2024
9d257eb
fix scheduled tasks
Nov 19, 2024
c45a5a3
fix
Nov 19, 2024
8945f84
fix
Nov 19, 2024
4d544ef
fix
Nov 19, 2024
bfcc8ac
fix
Nov 19, 2024
67aa1e6
fix
Nov 19, 2024
0d44f29
fix
Nov 19, 2024
32e73f6
revert
Nov 19, 2024
63ff83b
fix
Nov 19, 2024
a53c067
uploads
Nov 19, 2024
0c9995d
uploads
Nov 19, 2024
b388d9f
fix notifications
Nov 19, 2024
76dc06c
fix notifications
Nov 19, 2024
8b74448
fix notifications
Nov 19, 2024
331f0eb
fix notifications
Nov 19, 2024
fd7b3b9
fix notifications
Nov 19, 2024
a3658ce
fix notifications
Nov 19, 2024
4a03f5b
remove print statements
Nov 19, 2024
1a1de39
change querie to stmt
Nov 21, 2024
d283613
code review feedback and merge from main
Nov 21, 2024
6e23d0b
merge from main
Nov 22, 2024
9e5b21e
merge from main
Nov 26, 2024
8cd0bd8
code review feedback
Dec 19, 2024
096ec68
code review feedback
Dec 19, 2024
e4782e4
code review feedback
Dec 19, 2024
f4ce3d1
code review feedback
Dec 19, 2024
6591693
code review feedback
Dec 19, 2024
3388371
fix filter_bys
Dec 19, 2024
83193c2
fix fragile filter approach
Dec 19, 2024
db16f94
noqa the x == False for sqlalchemy
Dec 19, 2024
772f78d
noqa the x == False for sqlalchemy
Dec 19, 2024
5f9b4bc
noqa the x == False for sqlalchemy
Dec 19, 2024
67d8974
noqa the x == False for sqlalchemy
Dec 19, 2024
440bf85
noqa the x == False for sqlalchemy
Dec 19, 2024
c29fb78
noqa the x == False for sqlalchemy
Dec 19, 2024
9954ac4
noqa the x == False for sqlalchemy
Dec 19, 2024
605782c
noqa the x == False for sqlalchemy
Dec 19, 2024
bb1d81b
noqa the x == False for sqlalchemy
Dec 19, 2024
5a94229
noqa the x == False for sqlalchemy
Dec 19, 2024
8f572bb
noqa the x == False for sqlalchemy
Dec 19, 2024
f809a06
noqa the x == False for sqlalchemy
Dec 19, 2024
a77343e
noqa the x == False for sqlalchemy
Dec 19, 2024
3c0621f
more code review feedback
Dec 20, 2024
d8b7b06
merge from main
Dec 23, 2024
4ac7c74
merge from main
Dec 30, 2024
942b4a3
fix
Dec 30, 2024
82aebcd
fix
Dec 30, 2024
16286eb
fix
Dec 30, 2024
feeef72
fix
Dec 30, 2024
2878fb0
fix enum
Dec 30, 2024
bf497d8
try again
Dec 30, 2024
3da4755
try again
Dec 30, 2024
4e3f899
try again
Dec 30, 2024
71f682a
try again
Dec 30, 2024
c7cb377
try again
Dec 30, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .ds.baseline
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@
"filename": "tests/app/service/test_rest.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 1284,
"line_number": 1285,
"is_secret": false
}
],
Expand Down Expand Up @@ -341,15 +341,15 @@
"filename": "tests/app/user/test_rest.py",
"hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
"is_verified": false,
"line_number": 108,
"line_number": 110,
"is_secret": false
},
{
"type": "Secret Keyword",
"filename": "tests/app/user/test_rest.py",
"hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
"is_verified": false,
"line_number": 826,
"line_number": 858,
"is_secret": false
}
],
Expand Down Expand Up @@ -384,5 +384,5 @@
}
]
},
"generated_at": "2024-10-31T21:25:32Z"
"generated_at": "2024-11-15T18:43:06Z"
}
3 changes: 2 additions & 1 deletion app/billing/rest.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from flask import Blueprint, jsonify, request

from app import db
from app.billing.billing_schemas import (
create_or_update_free_sms_fragment_limit_schema,
serialize_ft_billing_remove_emails,
Expand Down Expand Up @@ -60,7 +61,7 @@ def get_free_sms_fragment_limit(service_id):
)

if annual_billing is None:
service = Service.query.get(service_id)
service = db.session.get(Service, service_id)
# An entry does not exist in annual_billing table for that service and year.
# Set the annual billing to the default free allowance based on the organization type of the service.

Expand Down
27 changes: 14 additions & 13 deletions app/celery/scheduled_tasks.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
from datetime import timedelta

from flask import current_app
from sqlalchemy import between
from sqlalchemy import between, select, union, update
from sqlalchemy.exc import SQLAlchemyError

from app import notify_celery, zendesk_client
from app import db, notify_celery, zendesk_client
from app.celery.tasks import (
get_recipient_csv_and_template_and_sender_id,
process_incomplete_jobs,
Expand All @@ -18,7 +18,6 @@
from app.dao.invited_user_dao import expire_invitations_created_more_than_two_days_ago
from app.dao.jobs_dao import (
dao_set_scheduled_jobs_to_pending,
dao_update_job,
find_jobs_with_missing_rows,
find_missing_row_for_job,
)
Expand Down Expand Up @@ -105,30 +104,32 @@ def check_job_status():
thirty_minutes_ago = utc_now() - timedelta(minutes=30)
thirty_five_minutes_ago = utc_now() - timedelta(minutes=35)

incomplete_in_progress_jobs = Job.query.filter(
incomplete_in_progress_jobs = select(Job).filter(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be a .filter() rather than a .where()?

Job.job_status == JobStatus.IN_PROGRESS,
between(Job.processing_started, thirty_five_minutes_ago, thirty_minutes_ago),
)
incomplete_pending_jobs = Job.query.filter(
incomplete_pending_jobs = select(Job).filter(
Job.job_status == JobStatus.PENDING,
Job.scheduled_for.isnot(None),
between(Job.scheduled_for, thirty_five_minutes_ago, thirty_minutes_ago),
)

jobs_not_complete_after_30_minutes = (
incomplete_in_progress_jobs.union(incomplete_pending_jobs)
.order_by(Job.processing_started, Job.scheduled_for)
.all()
)
jobs_not_complete_after_30_minutes = db.session.execute(
union(incomplete_in_progress_jobs, incomplete_pending_jobs).order_by(
Job.processing_started, Job.scheduled_for
)
).all()

# temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
# if they haven't been re-processed in time.
job_ids = []
for job in jobs_not_complete_after_30_minutes:
job.job_status = JobStatus.ERROR
dao_update_job(job)
job_ids.append(str(job.id))
db.session.execute(
update(Job).where(Job.id == job.id).values(job_status=JobStatus.ERROR)
)
db.session.commit()

job_ids.append(str(job.id))
if job_ids:
current_app.logger.info("Job(s) {} have not completed.".format(job_ids))
process_incomplete_jobs.apply_async([job_ids], queue=QueueNames.JOBS)
Expand Down
64 changes: 50 additions & 14 deletions app/dao/api_key_dao.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import uuid
from datetime import timedelta

from sqlalchemy import func, or_
from sqlalchemy import func, or_, select

from app import db
from app.dao.dao_utils import autocommit, version_class
Expand All @@ -23,31 +23,61 @@ def save_model_api_key(api_key):
@autocommit
@version_class(ApiKey)
def expire_api_key(service_id, api_key_id):
api_key = ApiKey.query.filter_by(id=api_key_id, service_id=service_id).one()
api_key = (
db.session.execute(
select(ApiKey).where(
ApiKey.id == api_key_id, ApiKey.service_id == service_id
)
)
.scalars()
xlorepdarkhelm marked this conversation as resolved.
Show resolved Hide resolved
.one()
)
api_key.expiry_date = utc_now()
db.session.add(api_key)


def get_model_api_keys(service_id, id=None):
if id:
return ApiKey.query.filter_by(
id=id, service_id=service_id, expiry_date=None
).one()
return (
db.session.execute(
select(ApiKey).where(
ApiKey.id == id,
ApiKey.service_id == service_id,
ApiKey.expiry_date == None, # noqa
)
)
.scalars()
xlorepdarkhelm marked this conversation as resolved.
Show resolved Hide resolved
.one()
)
seven_days_ago = utc_now() - timedelta(days=7)
return ApiKey.query.filter(
or_(
ApiKey.expiry_date == None, # noqa
func.date(ApiKey.expiry_date) > seven_days_ago, # noqa
),
ApiKey.service_id == service_id,
).all()
return (
db.session.execute(
select(ApiKey).where(
or_(
ApiKey.expiry_date == None, # noqa
func.date(ApiKey.expiry_date) > seven_days_ago, # noqa
),
ApiKey.service_id == service_id,
)
)
.scalars()
.all()
)


def get_unsigned_secrets(service_id):
"""
This method can only be exposed to the Authentication of the api calls.
"""
api_keys = ApiKey.query.filter_by(service_id=service_id, expiry_date=None).all()
api_keys = (
db.session.execute(
select(ApiKey).where(
ApiKey.service_id == service_id, ApiKey.expiry_date == None # noqa
)
)
.scalars()
.all()
)
keys = [x.secret for x in api_keys]
return keys

Expand All @@ -56,5 +86,11 @@ def get_unsigned_secret(key_id):
"""
This method can only be exposed to the Authentication of the api calls.
"""
api_key = ApiKey.query.filter_by(id=key_id, expiry_date=None).one()
api_key = (
db.session.execute(
select(ApiKey).where(ApiKey.id == key_id, ApiKey.expiry_date == None) # noqa
)
.scalars()
.one()
)
return api_key.secret
16 changes: 13 additions & 3 deletions app/dao/email_branding_dao.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,28 @@
from sqlalchemy import select

from app import db
from app.dao.dao_utils import autocommit
from app.models import EmailBranding


def dao_get_email_branding_options():
return EmailBranding.query.all()
return db.session.execute(select(EmailBranding)).scalars().all()


def dao_get_email_branding_by_id(email_branding_id):
return EmailBranding.query.filter_by(id=email_branding_id).one()
return (
db.session.execute(select(EmailBranding).filter_by(id=email_branding_id))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

.filter_by()

.scalars()
.one()
)


def dao_get_email_branding_by_name(email_branding_name):
return EmailBranding.query.filter_by(name=email_branding_name).first()
return (
db.session.execute(select(EmailBranding).filter_by(name=email_branding_name))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

.filter_by()

.scalars()
.first()
)


@autocommit
Expand Down
Loading
Loading