Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

qe: partially refactor relation aggregation selection #4658

Merged
merged 31 commits into from
Jan 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
f4af429
Refactor relation aggregation selection
aqrln Jan 11, 2024
c6b71a0
build user_selection and field_selection in query graph builder
aqrln Jan 18, 2024
d9087c8
reduce the scope, merge user_selection/full_selection back together
aqrln Jan 19, 2024
ca1824f
remove unused extract_aggregation_rows_from_scalars
aqrln Jan 19, 2024
f2b246d
fix typo
aqrln Jan 19, 2024
1e94819
Rename full_selection back to selected_fields
aqrln Jan 19, 2024
bc1c258
rust-analyzer missed a field?
aqrln Jan 19, 2024
5f383fd
fix compile errors in query_graph_builder
aqrln Jan 19, 2024
6099b6c
update serializer
aqrln Jan 19, 2024
3b96775
Remove some dead code
aqrln Jan 19, 2024
6dd6cc5
remove some commented out code
aqrln Jan 19, 2024
aba9c88
remove unused import
aqrln Jan 19, 2024
8172d3c
adjust wording
aqrln Jan 19, 2024
971535d
rewrite a match shorter
aqrln Jan 19, 2024
8c3b503
Fix 10 tests
aqrln Jan 19, 2024
aa92a2d
Refactor the last fix
aqrln Jan 19, 2024
849cb13
remove unused fn
aqrln Jan 19, 2024
3efb101
fix 15 tests
aqrln Jan 19, 2024
dcf070d
fix field order mismatch
aqrln Jan 19, 2024
7d56663
cleanup unused code
aqrln Jan 19, 2024
d2b6c1a
rename scalars to records
aqrln Jan 19, 2024
8caf0d7
pass an iterator without collecting
aqrln Jan 19, 2024
68eca7b
fix a todo!() in mongodb connector
aqrln Jan 19, 2024
cd48073
fix benchmark regression
aqrln Jan 19, 2024
9b11ed5
use same param name as on main
aqrln Jan 19, 2024
4b55d2d
resolve todo comment
aqrln Jan 19, 2024
dacdb34
do more work outside of loop
aqrln Jan 19, 2024
8d4a588
Fix incorrect id fields in empty nested results with MongoDB
aqrln Jan 20, 2024
d96e721
remove some unnecessary type annotations on lines changed anyway
aqrln Jan 20, 2024
163e33d
remove obsolete todo comment
aqrln Jan 23, 2024
ef0e83b
add a short comment
aqrln Jan 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use mongodb::{
bson::{self, extjson},
error::{CommandError, Error as DriverError, TRANSIENT_TRANSACTION_ERROR},
};
use query_structure::{CompositeFieldRef, Field, ScalarFieldRef, SelectedField};
use query_structure::{CompositeFieldRef, Field, ScalarFieldRef, SelectedField, VirtualSelection};
use regex::Regex;
use thiserror::Error;
use user_facing_errors::query_engine::DatabaseConstraint;
Expand Down Expand Up @@ -274,6 +274,7 @@ pub trait DecorateErrorWithFieldInformationExtension {
fn decorate_with_scalar_field_info(self, sf: &ScalarFieldRef) -> Self;
fn decorate_with_field_name(self, field_name: &str) -> Self;
fn decorate_with_composite_field_info(self, cf: &CompositeFieldRef) -> Self;
fn decorate_with_virtual_field_info(self, vs: &VirtualSelection) -> Self;
}

impl<T> DecorateErrorWithFieldInformationExtension for crate::Result<T> {
Expand All @@ -286,6 +287,7 @@ impl<T> DecorateErrorWithFieldInformationExtension for crate::Result<T> {
SelectedField::Scalar(sf) => self.decorate_with_scalar_field_info(sf),
SelectedField::Composite(composite_sel) => self.decorate_with_composite_field_info(&composite_sel.field),
SelectedField::Relation(_) => unreachable!(),
SelectedField::Virtual(vs) => self.decorate_with_virtual_field_info(vs),
}
}

Expand All @@ -300,4 +302,8 @@ impl<T> DecorateErrorWithFieldInformationExtension for crate::Result<T> {
fn decorate_with_composite_field_info(self, cf: &CompositeFieldRef) -> Self {
self.map_err(|err| err.decorate_with_field_name(cf.name()))
}

fn decorate_with_virtual_field_info(self, vs: &VirtualSelection) -> Self {
self.map_err(|err| err.decorate_with_field_name(&vs.db_alias()))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@ use crate::{
};
use async_trait::async_trait;
use connector_interface::{
Connection, ConnectionLike, ReadOperations, RelAggregationSelection, Transaction, UpdateType, WriteArgs,
WriteOperations,
Connection, ConnectionLike, ReadOperations, Transaction, UpdateType, WriteArgs, WriteOperations,
};
use mongodb::{ClientSession, Database};
use query_structure::{prelude::*, RelationLoadStrategy, SelectionResult};
Expand Down Expand Up @@ -234,7 +233,6 @@ impl ReadOperations for MongoDbConnection {
model: &Model,
filter: &query_structure::Filter,
selected_fields: &FieldSelection,
aggr_selections: &[RelAggregationSelection],
_relation_load_strategy: RelationLoadStrategy,
_trace_id: Option<String>,
) -> connector_interface::Result<Option<SingleRecord>> {
Expand All @@ -244,7 +242,6 @@ impl ReadOperations for MongoDbConnection {
model,
filter,
selected_fields,
aggr_selections,
))
.await
}
Expand All @@ -254,7 +251,6 @@ impl ReadOperations for MongoDbConnection {
model: &Model,
query_arguments: query_structure::QueryArguments,
selected_fields: &FieldSelection,
aggregation_selections: &[RelAggregationSelection],
_relation_load_strategy: RelationLoadStrategy,
_trace_id: Option<String>,
) -> connector_interface::Result<ManyRecords> {
Expand All @@ -264,7 +260,6 @@ impl ReadOperations for MongoDbConnection {
model,
query_arguments,
selected_fields,
aggregation_selections,
))
.await
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@ use crate::{
error::MongoError,
root_queries::{aggregate, read, write},
};
use connector_interface::{
ConnectionLike, ReadOperations, RelAggregationSelection, Transaction, UpdateType, WriteOperations,
};
use connector_interface::{ConnectionLike, ReadOperations, Transaction, UpdateType, WriteOperations};
use mongodb::options::{Acknowledgment, ReadConcern, TransactionOptions, WriteConcern};
use query_engine_metrics::{decrement_gauge, increment_gauge, metrics, PRISMA_CLIENT_QUERIES_ACTIVE};
use query_structure::{RelationLoadStrategy, SelectionResult};
Expand Down Expand Up @@ -264,7 +262,6 @@ impl<'conn> ReadOperations for MongoDbTransaction<'conn> {
model: &Model,
filter: &query_structure::Filter,
selected_fields: &FieldSelection,
aggr_selections: &[RelAggregationSelection],
_relation_load_strategy: RelationLoadStrategy,
_trace_id: Option<String>,
) -> connector_interface::Result<Option<SingleRecord>> {
Expand All @@ -274,7 +271,6 @@ impl<'conn> ReadOperations for MongoDbTransaction<'conn> {
model,
filter,
selected_fields,
aggr_selections,
))
.await
}
Expand All @@ -284,7 +280,6 @@ impl<'conn> ReadOperations for MongoDbTransaction<'conn> {
model: &Model,
query_arguments: query_structure::QueryArguments,
selected_fields: &FieldSelection,
aggregation_selections: &[RelAggregationSelection],
_relation_load_strategy: RelationLoadStrategy,
_trace_id: Option<String>,
) -> connector_interface::Result<ManyRecords> {
Expand All @@ -294,7 +289,6 @@ impl<'conn> ReadOperations for MongoDbTransaction<'conn> {
model,
query_arguments,
selected_fields,
aggregation_selections,
))
.await
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use connector_interface::{AggregationSelection, RelAggregationSelection};
use connector_interface::AggregationSelection;
use indexmap::IndexMap;
use query_structure::{
ast::FieldArity, DefaultKind, FieldSelection, PrismaValue, ScalarFieldRef, SelectedField, TypeIdentifier,
Expand Down Expand Up @@ -48,18 +48,12 @@ impl CompositeOutputMeta {
}
}

pub fn from_selected_fields(
selected_fields: &FieldSelection,
aggregation_selections: &[RelAggregationSelection],
) -> OutputMetaMapping {
let selections: Vec<&SelectedField> = selected_fields.selections().collect();
from_selections(&selections, aggregation_selections)
pub fn from_selected_fields(selected_fields: &FieldSelection) -> OutputMetaMapping {
let selections: Vec<_> = selected_fields.selections().collect();
from_selections(&selections)
}

pub fn from_selections(
selected_fields: &[&SelectedField],
aggregation_selections: &[RelAggregationSelection],
) -> OutputMetaMapping {
pub fn from_selections(selected_fields: &[&SelectedField]) -> OutputMetaMapping {
let mut map = OutputMetaMapping::new();

for selection in selected_fields {
Expand All @@ -70,7 +64,7 @@ pub fn from_selections(

SelectedField::Composite(cs) => {
let selections: Vec<&SelectedField> = cs.selections.iter().collect();
let inner = from_selections(&selections, &[]);
let inner = from_selections(&selections);

map.insert(
cs.field.db_name().to_owned(),
Expand All @@ -80,12 +74,22 @@ pub fn from_selections(
}),
);
}

SelectedField::Relation(_) => unreachable!(),
}
}

for selection in aggregation_selections {
map.insert(selection.db_alias(), from_rel_aggregation_selection(selection));
SelectedField::Virtual(vs) => {
let (ident, arity) = vs.type_identifier_with_arity();

map.insert(
vs.db_alias(),
OutputMeta::Scalar(ScalarOutputMeta {
ident,
default: None,
list: matches!(arity, FieldArity::List),
}),
);
}
}
}

map
Expand Down Expand Up @@ -126,18 +130,6 @@ pub fn from_aggregation_selection(selection: &AggregationSelection) -> OutputMet
map
}

/// Mapping for one specific relation aggregation selection.
/// DB alias -> OutputMeta
pub fn from_rel_aggregation_selection(aggr_selection: &RelAggregationSelection) -> OutputMeta {
let (ident, arity) = aggr_selection.type_identifier_with_arity();

OutputMeta::Scalar(ScalarOutputMeta {
ident,
default: None,
list: matches!(arity, FieldArity::List),
})
}

impl From<ScalarOutputMeta> for OutputMeta {
fn from(s: ScalarOutputMeta) -> Self {
Self::Scalar(s)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,10 @@ fn path_prefixed_selection(doc: &mut Document, parent_paths: Vec<String>, select
parent_paths.push(cs.field.db_name().to_owned());
path_prefixed_selection(doc, parent_paths, cs.selections);
}

query_structure::SelectedField::Relation(_) => unreachable!(),

query_structure::SelectedField::Virtual(_) => {}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ use crate::{
root_queries::observing,
vacuum_cursor, BsonTransform, IntoBson,
};
use connector_interface::{AggregationSelection, RelAggregationSelection};
use connector_interface::AggregationSelection;
use itertools::Itertools;
use mongodb::{
bson::{doc, Document},
options::AggregateOptions,
ClientSession, Collection,
};
use query_structure::{FieldSelection, Filter, Model, QueryArguments, ScalarFieldRef};
use query_structure::{FieldSelection, Filter, Model, QueryArguments, ScalarFieldRef, VirtualSelection};
use std::convert::TryFrom;

// Mongo Driver broke usage of the simple API, can't be used by us anymore.
Expand Down Expand Up @@ -355,13 +355,13 @@ impl MongoReadQueryBuilder {
}

/// Adds the necessary joins and the associated selections to the projection
pub fn with_aggregation_selections(
pub fn with_virtual_fields<'a>(
mut self,
aggregation_selections: &[RelAggregationSelection],
virtual_selections: impl Iterator<Item = &'a VirtualSelection>,
) -> crate::Result<Self> {
for aggr in aggregation_selections {
for aggr in virtual_selections {
let join = match aggr {
RelAggregationSelection::Count(rf, filter) => {
VirtualSelection::RelationCount(rf, filter) => {
let filter = filter
.as_ref()
.map(|f| MongoFilterVisitor::new(FilterPrefix::default(), false).visit(f.clone()))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ use crate::{
error::DecorateErrorWithFieldInformationExtension, output_meta, query_builder::MongoReadQueryBuilder,
query_strings::Find, vacuum_cursor, IntoBson,
};
use connector_interface::RelAggregationSelection;
use mongodb::{bson::doc, options::FindOptions, ClientSession, Database};
use query_structure::*;
use tracing::{info_span, Instrument};
Expand All @@ -15,7 +14,6 @@ pub async fn get_single_record<'conn>(
model: &Model,
filter: &Filter,
selected_fields: &FieldSelection,
aggregation_selections: &[RelAggregationSelection],
) -> crate::Result<Option<SingleRecord>> {
let coll = database.collection(model.db_name());

Expand All @@ -25,22 +23,19 @@ pub async fn get_single_record<'conn>(
"db.statement" = &format_args!("db.{}.findOne(*)", coll.name())
);

let meta_mapping = output_meta::from_selected_fields(selected_fields, aggregation_selections);
let meta_mapping = output_meta::from_selected_fields(selected_fields);
let query_arguments: QueryArguments = (model.clone(), filter.clone()).into();
let query = MongoReadQueryBuilder::from_args(query_arguments)?
.with_model_projection(selected_fields.clone())?
.with_aggregation_selections(aggregation_selections)?
.with_virtual_fields(selected_fields.virtuals())?
.build()?;

let docs = query.execute(coll, session).instrument(span).await?;

if docs.is_empty() {
Ok(None)
} else {
let field_names: Vec<_> = selected_fields
.db_names()
.chain(aggregation_selections.iter().map(|aggr_sel| aggr_sel.db_alias()))
.collect();
let field_names: Vec<_> = selected_fields.db_names().collect();
let doc = docs.into_iter().next().unwrap();
let record = document_to_record(doc, &field_names, &meta_mapping)?;

Expand All @@ -61,7 +56,6 @@ pub async fn get_many_records<'conn>(
model: &Model,
query_arguments: QueryArguments,
selected_fields: &FieldSelection,
aggregation_selections: &[RelAggregationSelection],
) -> crate::Result<ManyRecords> {
let coll = database.collection(model.db_name());

Expand All @@ -72,12 +66,9 @@ pub async fn get_many_records<'conn>(
);

let reverse_order = query_arguments.take.map(|t| t < 0).unwrap_or(false);
let field_names: Vec<_> = selected_fields
.db_names()
.chain(aggregation_selections.iter().map(|aggr_sel| aggr_sel.db_alias()))
.collect();
let field_names: Vec<_> = selected_fields.db_names().collect();

let meta_mapping = output_meta::from_selected_fields(selected_fields, aggregation_selections);
let meta_mapping = output_meta::from_selected_fields(selected_fields);
let mut records = ManyRecords::new(field_names.clone());

if let Some(0) = query_arguments.take {
Expand All @@ -86,7 +77,7 @@ pub async fn get_many_records<'conn>(

let query = MongoReadQueryBuilder::from_args(query_arguments)?
.with_model_projection(selected_fields.clone())?
.with_aggregation_selections(aggregation_selections)?
.with_virtual_fields(selected_fields.virtuals())?
.build()?;

let docs = query.execute(coll, session).instrument(span).await?;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ pub async fn delete_record<'conn>(
cause: "Record to delete does not exist.".to_owned(),
})?;

let meta_mapping = output_meta::from_selected_fields(&selected_fields, &[]);
let meta_mapping = output_meta::from_selected_fields(&selected_fields);
let field_names: Vec<_> = selected_fields.db_names().collect();
let record = document_to_record(document, &field_names, &meta_mapping)?;
Ok(SingleRecord { record, field_names })
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ impl IntoBson for (&SelectedField, PrismaValue) {
SelectedField::Scalar(sf) => (sf, value).into_bson(),
SelectedField::Composite(_) => todo!(), // [Composites] todo
SelectedField::Relation(_) => unreachable!(),
SelectedField::Virtual(_) => unreachable!(),
}
}
}
Expand Down
Loading