From 5ec7b4bcd7e2e0698736cd374f2196682227c913 Mon Sep 17 00:00:00 2001
From: Debdatta Kunda <87335885+kundadebdatta@users.noreply.github.com>
Date: Wed, 19 Jul 2023 16:26:04 -0700
Subject: [PATCH 01/20] Code changes to update release note. (#3996)
---
changelog.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/changelog.md b/changelog.md
index 481cde52db..ab733a4137 100644
--- a/changelog.md
+++ b/changelog.md
@@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
#### Added
- [3668](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3668) Query : Adds string comparison alternative when converting LINQ to SQL (Thanks [@ernesto1596](https://github.com/ernesto1596))
- [3834](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3834) Query : Adds support for newtonsoft member access via ExtensionData (Thanks [@onionhammer](https://github.com/onionhammer))
+- [3939](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3939) CreateAndInitializeAsync: Adds Code to Optimize Rntbd Open Connection Logic to Open Connections in Parallel
### [3.35.1-preview](https://www.nuget.org/packages/Microsoft.Azure.Cosmos/3.35.1-preview) - 2023-06-27
### [3.35.1](https://www.nuget.org/packages/Microsoft.Azure.Cosmos/3.35.1) - 2023-06-27
From 08981bf2fd87f110ca4a1e2af5f82974857c1aaf Mon Sep 17 00:00:00 2001
From: vipulvishal-ms <110802706+vipulvishal-ms@users.noreply.github.com>
Date: Sat, 29 Jul 2023 02:33:38 +0530
Subject: [PATCH 02/20] Client Encryption: Adds fix for supporting Prefix
Partition Key (Hierarchical partitioning) (#3979)
* Hirarchical pk bug fix
* Hirarchical pk bug fix
* Hirarchical pk bug fix
* Hirarchical pk bug fix
* Hirarchical pk bug fix
* testing new version
* adding more tests
* adding more tests
* adding more tests
* code review changes
* test fix
* test fix
* test fix
* test fix
---------
Co-authored-by: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com>
---
Directory.Build.props | 4 +-
.../changelog.md | 10 +
.../src/EncryptionContainer.cs | 17 +-
.../Microsoft.Azure.Cosmos.Encryption.csproj | 4 +-
.../tests/EmulatorTests/MdeEncryptionTests.cs | 388 +++++++++++++++++-
5 files changed, 413 insertions(+), 10 deletions(-)
diff --git a/Directory.Build.props b/Directory.Build.props
index b9171d6307..9daa1e7e9f 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -4,8 +4,8 @@
3.35.2
preview
3.31.3
- 2.0.2
- 2.0.2
+ 2.0.3
+ 2.0.3
preview
1.0.0-preview06
1.1.0-preview3
diff --git a/Microsoft.Azure.Cosmos.Encryption/changelog.md b/Microsoft.Azure.Cosmos.Encryption/changelog.md
index a8edf3441d..2f50fcbdf3 100644
--- a/Microsoft.Azure.Cosmos.Encryption/changelog.md
+++ b/Microsoft.Azure.Cosmos.Encryption/changelog.md
@@ -3,6 +3,16 @@ Preview features are treated as a separate branch and will not be included in th
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+### [2.0.3](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption/2.0.3) - 2023-07-12
+
+#### Added
+- [#3979](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3979) Adds fix for supporting Prefix Partition Key (Hierarchical partitioning).
+
+### [2.0.3-preview](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption/2.0.3-preview) - 2023-07-12
+
+#### Added
+- [#3979](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3979) Adds fix for supporting Prefix Partition Key (Hierarchical partitioning).
+
### [2.0.2](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption/2.0.2) - 2023-06-01
#### Added
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs
index d32bfa1c94..0bd342f0ef 100644
--- a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs
+++ b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs
@@ -957,15 +957,22 @@ internal async Task CheckIfIdIsEncryptedAndGetEncryptedIdAsync(
JArray jArray = JArray.Parse(partitionKey.ToString());
#if ENCRYPTIONPREVIEW
- if (jArray.Count > 1)
+ if (encryptionSettings.PartitionKeyPaths.Count > 1)
{
- int i = 0;
+ int counter = 0;
PartitionKeyBuilder partitionKeyBuilder = new PartitionKeyBuilder();
+ if (jArray.Count() > encryptionSettings.PartitionKeyPaths.Count())
+ {
+ throw new NotSupportedException($"The number of partition keys passed in the query exceeds the number of keys initialized on the container. Container Id : {this.Id}");
+ }
bool isPkEncrypted = false;
+
// partitionKeyBuilder expects the paths and values to be in same order.
- foreach (string path in encryptionSettings.PartitionKeyPaths)
+ for(counter = 0; counter < jArray.Count(); counter++)
{
+ string path = encryptionSettings.PartitionKeyPaths[counter];
+
// case: partition key path is /a/b/c and the client encryption policy has /a in path.
// hence encrypt the partition key value with using its top level path /a since /c would have been encrypted in the document using /a's policy.
string partitionKeyPath = path.Split('/')[1];
@@ -975,12 +982,12 @@ internal async Task CheckIfIdIsEncryptedAndGetEncryptedIdAsync(
if (encryptionSettingForProperty == null)
{
- partitionKeyBuilder.Add(jArray[i++].ToString());
+ partitionKeyBuilder.Add(jArray[counter].ToString());
continue;
}
isPkEncrypted = true;
- Stream valueStream = EncryptionProcessor.BaseSerializer.ToStream(jArray[i++]);
+ Stream valueStream = EncryptionProcessor.BaseSerializer.ToStream(jArray[counter]);
Stream encryptedPartitionKey = await EncryptionProcessor.EncryptValueStreamAsync(
valueStreamToEncrypt: valueStream,
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj b/Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj
index 94e93aa6fe..f7baa75f60 100644
--- a/Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj
+++ b/Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj
@@ -28,11 +28,11 @@
-
+
-
+
diff --git a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs
index 65efd425aa..3489cdf02e 100644
--- a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs
+++ b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs
@@ -189,6 +189,9 @@ public void TestInitialize()
{
// Reset static cache TTL
Microsoft.Data.Encryption.Cryptography.ProtectedDataEncryptionKey.TimeToLive = TimeSpan.FromHours(2);
+ // flag to disable https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3951
+ // need to be removed after the fix
+ Environment.SetEnvironmentVariable("AZURE_COSMOS_REPLICA_VALIDATION_ENABLED", "False");
}
private static async Task CreateClientEncryptionKeyAsync(string cekId, Cosmos.EncryptionKeyWrapMetadata encryptionKeyWrapMetadata)
@@ -2303,7 +2306,7 @@ public async Task ValidatePkAndIdEncryptionSupport()
VerifyExpectedDocResponse(testDoc, readResponse.Resource);
#if ENCRYPTIONTESTPREVIEW
- // hierarchical
+ // hierarchical pk container test
cepWithPKIdPath1 = new ClientEncryptionIncludedPath()
{
Path = "/Sensitive_LongFormat",
@@ -2352,9 +2355,320 @@ public async Task ValidatePkAndIdEncryptionSupport()
Assert.AreEqual(HttpStatusCode.OK, readResponse.StatusCode);
VerifyExpectedDocResponse(testDoc, readResponse.Resource);
+
+ // test to validate query with one partition key (topmost) in hierarchical pk container of 3 keys
+ QueryRequestOptions queryRequestOptions = new QueryRequestOptions
+ {
+ PartitionKey = new PartitionKeyBuilder().Add(testDoc.Sensitive_StringFormat).Build()
+ };
+
+ using FeedIterator setIterator = encryptionContainer.GetItemQueryIterator("select * from c", requestOptions: queryRequestOptions);
+
+ while (setIterator.HasMoreResults)
+ {
+ FeedResponse response = await setIterator.ReadNextAsync().ConfigureAwait(false);
+ Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
+ VerifyExpectedDocResponse(testDoc, response.First());
+ }
+
+ // test to validate query with one partition key (topmost) in hierarchical pk container of 3 keys with where clause on topmost pk
+ QueryDefinition queryDefinition = encryptionContainer.CreateQueryDefinition("SELECT * FROM c WHERE c.Sensitive_StringFormat = @Sensitive_StringFormat");
+
+ await queryDefinition.AddParameterAsync("@Sensitive_StringFormat", testDoc.Sensitive_StringFormat, "/Sensitive_StringFormat");
+
+ FeedIterator setIteratorWithFilter = encryptionContainer.GetItemQueryIterator(queryDefinition, requestOptions: queryRequestOptions);
+
+ while (setIteratorWithFilter.HasMoreResults)
+ {
+ FeedResponse response = await setIteratorWithFilter.ReadNextAsync().ConfigureAwait(false);
+ Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
+ VerifyExpectedDocResponse(testDoc, response.First());
+ }
+
+ // test to validate query with one partition key (2nd topmost) in hierarchical pk container of 3 keys with where clause on topmost pk
+ // this shold give 0 items as PK is set wrongly
+ queryRequestOptions = new QueryRequestOptions
+ {
+ PartitionKey = new PartitionKeyBuilder().Add(testDoc.Sensitive_NestedObjectFormatL1.Sensitive_NestedObjectFormatL2.Sensitive_StringFormatL2).Build()
+ };
+
+ setIteratorWithFilter = encryptionContainer.GetItemQueryIterator(queryDefinition, requestOptions: queryRequestOptions);
+
+ while (setIteratorWithFilter.HasMoreResults)
+ {
+ FeedResponse response = await setIteratorWithFilter.ReadNextAsync().ConfigureAwait(false);
+ Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
+ Assert.AreEqual(0, response.Count());
+ }
#endif
}
+#if ENCRYPTIONTESTPREVIEW
+ [TestMethod]
+ public async Task TestHirarchicalPkWithFullAndPartialKey()
+ {
+ HirarchicalPkTestDoc testDoc = HirarchicalPkTestDoc.Create();
+
+ ClientEncryptionIncludedPath cepWithPKIdPath1 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/State",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ ClientEncryptionIncludedPath cepWithPKIdPath2 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/City",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ ClientEncryptionIncludedPath cepWithPKIdPath3 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/ZipCode",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ Collection paths = new Collection { cepWithPKIdPath1, cepWithPKIdPath2, cepWithPKIdPath3 };
+
+ ClientEncryptionPolicy clientEncryptionPolicy= new ClientEncryptionPolicy(paths, 2);
+
+ ContainerProperties containerProperties = new ContainerProperties()
+ {
+ Id = "HierarchicalPkContainerWith3Pk",
+ PartitionKeyPaths = new List { "/State", "/City", "/ZipCode" },
+ ClientEncryptionPolicy = clientEncryptionPolicy
+ };
+
+ Container encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);
+ await encryptionContainer.InitializeEncryptionAsync();
+
+ PartitionKey hirarchicalPk = new PartitionKeyBuilder()
+ .Add(testDoc.State)
+ .Add(testDoc.City)
+ .Add(testDoc.ZipCode)
+ .Build();
+
+ ItemResponse createResponse = await encryptionContainer.CreateItemAsync(
+ testDoc,
+ partitionKey: hirarchicalPk);
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
+
+ // read back
+ ItemResponse readResponse = await encryptionContainer.ReadItemAsync(
+ testDoc.Id,
+ hirarchicalPk);
+
+ Assert.AreEqual(HttpStatusCode.OK, readResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, readResponse.Resource);
+
+ PartitionKey partialHirarchicalPk = new PartitionKeyBuilder()
+ .Add(testDoc.State)
+ .Add(testDoc.City)
+ .Build();
+
+ QueryRequestOptions queryRequestOptions = new QueryRequestOptions
+ {
+ PartitionKey = partialHirarchicalPk
+ };
+
+ using FeedIterator setIterator = encryptionContainer.GetItemQueryIterator("select * from c", requestOptions: queryRequestOptions);
+
+ while (setIterator.HasMoreResults)
+ {
+ FeedResponse response = await setIterator.ReadNextAsync().ConfigureAwait(false);
+ Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
+ VerifyExpectedDocResponse(testDoc, response.First());
+ }
+
+ QueryDefinition withEncryptedParameter = encryptionContainer.CreateQueryDefinition(
+ "SELECT * FROM c WHERE c.City = @cityInput AND c.State = @stateInput");
+
+ await withEncryptedParameter.AddParameterAsync(
+ "@cityInput",
+ testDoc.City,
+ "/City");
+
+ await withEncryptedParameter.AddParameterAsync(
+ "@stateInput",
+ testDoc.State,
+ "/State");
+
+ // query with partial HirarchicalPk state and city
+ FeedIterator queryResponseIterator;
+ queryResponseIterator = encryptionContainer.GetItemQueryIterator(withEncryptedParameter, requestOptions: queryRequestOptions);
+
+ while (queryResponseIterator.HasMoreResults)
+ {
+ FeedResponse response = await queryResponseIterator.ReadNextAsync().ConfigureAwait(false);
+ Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
+ VerifyExpectedDocResponse(testDoc, response.First());
+ }
+
+ partialHirarchicalPk = new PartitionKeyBuilder()
+ .Add(testDoc.State)
+ .Build();
+
+ queryRequestOptions = new QueryRequestOptions
+ {
+ PartitionKey = partialHirarchicalPk
+ };
+
+ // query with partial HirarchicalPk state
+ queryResponseIterator = encryptionContainer.GetItemQueryIterator(withEncryptedParameter, requestOptions: queryRequestOptions);
+
+ while (queryResponseIterator.HasMoreResults)
+ {
+ FeedResponse response = await queryResponseIterator.ReadNextAsync().ConfigureAwait(false);
+ Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
+ VerifyExpectedDocResponse(testDoc, response.First());
+ }
+
+ partialHirarchicalPk = new PartitionKeyBuilder()
+ .Add(testDoc.ZipCode)
+ .Build();
+
+ queryRequestOptions = new QueryRequestOptions
+ {
+ PartitionKey = partialHirarchicalPk
+ };
+
+ // query with partial HirarchicalPk zipCode.
+ // Since zipCode is 3rd in HirarchicalPk set. Query will get 0 response.
+ queryResponseIterator = encryptionContainer.GetItemQueryIterator(withEncryptedParameter, requestOptions: queryRequestOptions);
+
+ while (queryResponseIterator.HasMoreResults)
+ {
+ FeedResponse response = await queryResponseIterator.ReadNextAsync().ConfigureAwait(false);
+ Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
+ Assert.AreEqual(0, response.Count());
+ }
+
+ // query with no HirarchicalPk set.
+ queryResponseIterator = encryptionContainer.GetItemQueryIterator(withEncryptedParameter);
+
+ while (queryResponseIterator.HasMoreResults)
+ {
+ FeedResponse response = await queryResponseIterator.ReadNextAsync().ConfigureAwait(false);
+ Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
+ VerifyExpectedDocResponse(testDoc, response.First());
+ }
+
+ partialHirarchicalPk = new PartitionKeyBuilder()
+ .Add(testDoc.State)
+ .Add(testDoc.City)
+ .Add(testDoc.ZipCode)
+ .Add("Extra Value")
+ .Build();
+
+ queryRequestOptions = new QueryRequestOptions
+ {
+ PartitionKey = partialHirarchicalPk
+ };
+ // query with more PKs greater than number of PK feilds set in the container settings.
+ try
+ {
+ queryResponseIterator = encryptionContainer.GetItemQueryIterator(withEncryptedParameter, requestOptions: queryRequestOptions);
+ while (queryResponseIterator.HasMoreResults)
+ {
+ FeedResponse response = await queryResponseIterator.ReadNextAsync().ConfigureAwait(false);
+ Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
+ Assert.AreEqual(0, response.Count());
+ }
+ }
+ catch (Exception ex)
+ {
+ Assert.IsTrue(ex is NotSupportedException);
+ if (ex is NotSupportedException notSupportedException)
+ Assert.IsTrue(notSupportedException.Message.Contains("The number of partition keys passed in the query exceeds the number of keys initialized on the container"));
+ }
+ }
+
+ [TestMethod]
+ public async Task TestHirarchicalPkWithOnlyOneKey()
+ {
+ HirarchicalPkTestDoc testDoc = HirarchicalPkTestDoc.Create();
+
+ ClientEncryptionIncludedPath cepWithPKIdPath1 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/State",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ ClientEncryptionIncludedPath cepWithPKIdPath2 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/City",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ ClientEncryptionIncludedPath cepWithPKIdPath3 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/ZipCode",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ Collection paths = new Collection { cepWithPKIdPath1, cepWithPKIdPath2, cepWithPKIdPath3 };
+
+ ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2);
+
+ ContainerProperties containerProperties = new ContainerProperties()
+ {
+ Id = "HierarchicalPkContainerWithOnePk",
+ PartitionKeyPaths = new List { "/State" },
+ ClientEncryptionPolicy = clientEncryptionPolicy
+ };
+
+ Container encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);
+ await encryptionContainer.InitializeEncryptionAsync();
+
+ PartitionKey hirarchicalPk = new PartitionKeyBuilder()
+ .Add(testDoc.State)
+ .Build();
+
+ ItemResponse createResponse = await encryptionContainer.CreateItemAsync(
+ testDoc,
+ partitionKey: hirarchicalPk);
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
+
+ // read back
+ ItemResponse readResponse = await encryptionContainer.ReadItemAsync(
+ testDoc.Id,
+ hirarchicalPk);
+
+ Assert.AreEqual(HttpStatusCode.OK, readResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, readResponse.Resource);
+
+ PartitionKey fullHirarchicalPk = new PartitionKeyBuilder()
+ .Add(testDoc.State)
+ .Build();
+
+ QueryRequestOptions queryRequestOptions = new QueryRequestOptions
+ {
+ PartitionKey = fullHirarchicalPk
+ };
+
+ using FeedIterator setIterator = encryptionContainer.GetItemQueryIterator("select * from c", requestOptions: queryRequestOptions);
+
+ while (setIterator.HasMoreResults)
+ {
+ FeedResponse response = await setIterator.ReadNextAsync().ConfigureAwait(false);
+ Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
+ VerifyExpectedDocResponse(testDoc, response.First());
+ }
+ }
+#endif
[TestMethod]
public async Task EncryptionStreamIteratorValidation()
{
@@ -3204,6 +3518,14 @@ private static async Task> MdeDeleteItemAsync(
return deleteResponse;
}
+ private static void VerifyExpectedDocResponse(HirarchicalPkTestDoc expectedDoc, HirarchicalPkTestDoc verifyDoc)
+ {
+ Assert.AreEqual(expectedDoc.Id, verifyDoc.Id);
+ Assert.AreEqual(expectedDoc.State, verifyDoc.State);
+ Assert.AreEqual(expectedDoc.City, verifyDoc.City);
+ Assert.AreEqual(expectedDoc.ZipCode, verifyDoc.ZipCode);
+ }
+
private static void VerifyExpectedDocResponse(TestDoc expectedDoc, TestDoc verifyDoc)
{
Assert.AreEqual(expectedDoc.Id, verifyDoc.Id);
@@ -3716,6 +4038,70 @@ public Stream ToStream()
}
}
+ public class HirarchicalPkTestDoc
+ {
+ [JsonProperty("id")]
+ public string Id { get; set; }
+
+ public string PK { get; set; }
+
+ public string State { get; set; }
+
+ public string City { get; set; }
+
+ public string ZipCode { get; set; }
+
+ public HirarchicalPkTestDoc()
+ {
+ }
+ public HirarchicalPkTestDoc(HirarchicalPkTestDoc other)
+ {
+ this.Id = other.Id;
+ this.PK = other.PK;
+ this.State = other.State;
+ this.City = other.City;
+ this.ZipCode = other.ZipCode;
+ }
+
+ public override bool Equals(object obj)
+ {
+ return obj is HirarchicalPkTestDoc doc
+ && this.Id == doc.Id
+ && this.PK == doc.PK
+ && this.State == doc.State
+ && this.City == doc.City
+ && this.ZipCode == doc.ZipCode;
+ }
+
+ public override int GetHashCode()
+ {
+ int hashCode = 1652434776;
+ hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Id);
+ hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.PK);
+ hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.State);
+ hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.City);
+ hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.ZipCode);
+ return hashCode;
+ }
+
+ public static HirarchicalPkTestDoc Create(string partitionKey = null)
+ {
+ return new HirarchicalPkTestDoc()
+ {
+ Id = Guid.NewGuid().ToString(),
+ PK = partitionKey ?? Guid.NewGuid().ToString(),
+ State = Guid.NewGuid().ToString(),
+ City = Guid.NewGuid().ToString(),
+ ZipCode = Guid.NewGuid().ToString()
+ };
+ }
+
+ public Stream ToStream()
+ {
+ return TestCommon.ToStream(this);
+ }
+ }
+
internal class TestKeyEncryptionKey : IKeyEncryptionKey
{
private static readonly Dictionary keyinfo = new Dictionary
From ebd1b91a26547721ef232ef597d878144889b80b Mon Sep 17 00:00:00 2001
From: akotalwar <94020786+akotalwar@users.noreply.github.com>
Date: Tue, 1 Aug 2023 08:28:49 -0700
Subject: [PATCH 03/20] Query: Refactors changelog.md with Optimistic Direct
Execution recommendation (#4004)
* Update changelog.md
This is a recommendation for customers if they would like to use the ODE features.
* Updated release notes for ODE
---
changelog.md | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/changelog.md b/changelog.md
index ab733a4137..a6f9e65864 100644
--- a/changelog.md
+++ b/changelog.md
@@ -44,6 +44,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- [3836](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3836) Integrated cache: Adds BypassIntegratedCache to DedicatedGatewayRequestOptions
- [3909](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3909) Query: Adds EnableOptimisticDirectExecution in QueryRequestOptions enabled by default
+Recommendation for customers regarding Optimistic Direct Execution:
+
+Starting Version 3.35.0, the Preview SDK enables the ODE feature by default. This can potentially cause a new type of continuation token to be generated. Such a token is not recognized by the older SDKs by design and this could result in a Malformed Continuation Token Exception.
+If you have a scenario where tokens generated from the newer SDKs are used by an older SDK, we recommend a 2 step approach to upgrade:
+
+- Upgrade to the new SDK and disable ODE, both together as part of a single deployment. Wait for all nodes to upgrade.
+ - In order to disable ODE, set EnableOptimisticDirectExecution to false in the QueryRequestOptions.
+- Enable ODE as part of second deployment for all nodes.
+
### [3.35.0](https://www.nuget.org/packages/Microsoft.Azure.Cosmos/3.35.0) - 2023-06-19
#### Fixed
From 8c4f99f35e326b61ecca210988334d00629b6ee7 Mon Sep 17 00:00:00 2001
From: akotalwar <94020786+akotalwar@users.noreply.github.com>
Date: Sat, 5 Aug 2023 13:01:59 -0700
Subject: [PATCH 04/20] [Internal] Query: Adds performance testing for
OptimisticDirectExecution pipeline (#3839)
* Infrastructure for performance testing with ODE pipeline.
* Resolve comments
* Removed randomization from data creation process
* Fixed comments
* Removed Query and EnableODE from QueryStatisticsMetrics, as they do not relate to query statistics.
* Removed try catch to make CreateItemAsync call always succeed
* Removed one liner functions
* Removed code from MetricsSerializer and QueryStatisticsDatumVisitor files
* Fixed comments
* Removed request Charge check
* Bug in Debug Assert
* Test
* Bug in debug assert fix
* Fixed second bug in Metrics Accumalator class
* Added ignore flag to ode perf tests so that they do not run on every loop build
* Added comment explaining the Ignore flag.
---
.../ContentSerializationPerformanceTests.cs | 18 +-
.../QueryPerfTest/MetricsAccumulator.cs | 8 +-
...timisticDirectExecutionPerformanceTests.cs | 469 ++++++++++++++++++
.../QueryPerfTest/QueryStatisticsMetrics.cs | 6 +
.../settings.json | 18 +-
5 files changed, 496 insertions(+), 23 deletions(-)
create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs
diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs
index 759e63b345..8ff5660c6f 100644
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/ContentSerializationPerformanceTests.cs
@@ -33,15 +33,15 @@ public ContentSerializationPerformanceTests()
this.queryStatisticsDatumVisitor = new();
this.endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"];
this.authKey = Utils.ConfigurationManager.AppSettings["MasterKey"];
- this.cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.CosmosDatabaseId"];
- this.containerId = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.ContainerId"];
- this.contentSerialization = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.ContentSerialization"];
- this.query = Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.Query"];
- this.numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.NumberOfIterations"]);
- this.warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.WarmupIterations"]);
- this.MaxConcurrency = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.MaxConcurrency"]);
- this.MaxItemCount = int.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.MaxItemCount"]);
- this.useStronglyTypedIterator = bool.Parse(Utils.ConfigurationManager.AppSettings["ContentSerializationPerformanceTests.UseStronglyTypedIterator"]);
+ this.cosmosDatabaseId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.CosmosDatabaseId"];
+ this.containerId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.ContainerId"];
+ this.contentSerialization = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.ContentSerialization"];
+ this.query = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.Query"];
+ this.numberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.NumberOfIterations"]);
+ this.warmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.WarmupIterations"]);
+ this.MaxConcurrency = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.MaxConcurrency"]);
+ this.MaxItemCount = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.MaxItemCount"]);
+ this.useStronglyTypedIterator = bool.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.UseStronglyTypedIterator"]);
}
[TestMethod]
diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs
index 56fc16b683..a7e6032001 100644
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/MetricsAccumulator.cs
@@ -53,18 +53,16 @@ public void ReadFromTrace(FeedResponse Response, QueryStatisticsDatumVisit
{
if (storeResponse.StoreResult.StatusCode == StatusCodes.Ok)
{
- backendAndClientMetrics.Add(Tuple.Create(retrieveCosmosElementTraces[k], backendMetrics[j], transitMetrics[i]));
+ backendAndClientMetrics.Add(Tuple.Create(retrieveCosmosElementTraces[k], backendMetrics[j], node));
j++;
k++;
}
else
{
//We add null values to the tuple since status codes other than Ok will not have data for 'Query Metrics' and 'Get Cosmos Element Response'
- backendAndClientMetrics.Add(Tuple.Create(null, null, transitMetrics[i]));
+ backendAndClientMetrics.Add(Tuple.Create(null, null, node));
}
}
-
- i++;
}
Debug.Assert(i == transitMetrics.Count, "All 'transit metrics' must be grouped.");
@@ -76,7 +74,7 @@ public void ReadFromTrace(FeedResponse Response, QueryStatisticsDatumVisit
{
if (metrics.Item2 != null)
{
- Debug.Assert(metrics.Item1 == null, "'Get Cosmos Element Response' is null");
+ Debug.Assert(metrics.Item1 != null, "'Get Cosmos Element Response' is null");
queryStatisticsDatumVisitor.AddGetCosmosElementResponseTime(metrics.Item1.Duration.TotalMilliseconds);
foreach (KeyValuePair kvp in metrics.Item2.Data)
{
diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs
new file mode 100644
index 0000000000..756357e611
--- /dev/null
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/OptimisticDirectExecutionPerformanceTests.cs
@@ -0,0 +1,469 @@
+namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
+{
+ using System;
+ using System.Collections.Generic;
+ using System.IO;
+ using System.Linq;
+ using System.Text;
+ using System.Threading.Tasks;
+ using Microsoft.Azure.Cosmos;
+ using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+ [TestClass]
+ public class OptimisticDirectExecutionPerformanceTests
+ {
+ private Container Container;
+ private const string RawDataFileName = "OptimisticDirectExecutionPerformanceTestsRawData.csv";
+ private const string AggregateDataFileName = "OptimisticDirectExecutionPerformanceTestsAggregatedData.csv";
+ private const string PrintQueryMetrics = "QueryMetrics";
+ private static readonly string RawDataPath = Path.GetFullPath(RawDataFileName);
+ private static readonly string AggregateDataPath = Path.GetFullPath(AggregateDataFileName);
+ private static readonly string Endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"];
+ private static readonly string AuthKey = Utils.ConfigurationManager.AppSettings["MasterKey"];
+ private static readonly string CosmosDatabaseId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.CosmosDatabaseId"];
+ private static readonly string ContainerId = Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.ContainerId"];
+ private static readonly PartitionKey PartitionKeyValue = new PartitionKey("Andersen");
+ private static readonly int NumberOfIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.NumberOfIterations"]);
+ private static readonly int WarmupIterations = int.Parse(Utils.ConfigurationManager.AppSettings["QueryPerformanceTests.WarmupIterations"]);
+
+ [TestInitialize]
+ public async Task InitializeTest()
+ {
+ CosmosClientOptions clientOptions = new CosmosClientOptions()
+ {
+ ConnectionMode = ConnectionMode.Direct,
+ };
+
+ CosmosClient cosmosClient = new CosmosClient(Endpoint, AuthKey, clientOptions);
+ Database database = await cosmosClient.CreateDatabaseAsync(CosmosDatabaseId);
+ this.Container = await database.CreateContainerAsync(ContainerId, partitionKeyPath: "/name");
+ await this.AddItemsToContainerAsync(this.Container);
+
+ if (File.Exists(RawDataPath))
+ {
+ File.Delete(RawDataPath);
+ }
+
+ if (File.Exists(AggregateDataPath))
+ {
+ File.Delete(AggregateDataPath);
+ }
+ }
+
+ [TestCleanup]
+ public async Task CleanupAsync()
+ {
+ if (this.Container != null)
+ {
+ await this.Container.Database.DeleteAsync();
+ this.Container = null;
+ }
+ }
+
+ private async Task AddItemsToContainerAsync(Container container)
+ {
+ int totalItems = 5000;
+ string[] cityOptions = new string[] { "Seattle", "Chicago", "NYC", "SF" };
+ int numberOfRecipeints = cityOptions.Length;
+ List recipientList = new List(numberOfRecipeints);
+
+ for (int j = 0; j < numberOfRecipeints; j++)
+ {
+ RecipientList recipient = new RecipientList()
+ {
+ Name = "John",
+ City = cityOptions[j],
+ };
+
+ recipientList.Add(recipient);
+ }
+
+ // Create a family object for the Andersen family
+ foreach (int i in Enumerable.Range(0, totalItems))
+ {
+ States andersenFamily = new States
+ {
+ Id = i.ToString(),
+ Name = i < (totalItems/2) ? "Andersen" : "Smith",
+ City = cityOptions[i%cityOptions.Length],
+ PostalCode = (i * 10).ToString(),
+ Region = "Northwest",
+ UserDefinedID = i % 10,
+ RecipientList = recipientList
+ };
+
+ ItemResponse andersenFamilyResponse = await container.CreateItemAsync(andersenFamily, new PartitionKey(andersenFamily.Name));
+ Console.WriteLine("Created item in database with id: {0} Operation consumed {1} RUs.\n", andersenFamilyResponse.Resource.Id, andersenFamilyResponse.RequestCharge);
+ }
+ }
+
+ //Set Ode perf tests to ignore so that they dont run on every loop build.
+ //Ignore flag can be removed when checking for Ode performance.
+ [Ignore]
+ [TestMethod]
+ [Owner("akotalwar")]
+ public async Task RunAsync()
+ {
+ string highPrepTimeSumQuery = CreateHighPrepTimeSumQuery();
+ string highPrepTimeConditionalQuery = CreateHighPrepTimeConditionalQuery();
+ List> globalCustomOdeStatisticsList = new List>();
+
+ List odeTestCases = new List()
+ {
+ //Simple Query
+ CreateInput("SELECT * FROM c", PartitionKeyValue, false, -1, 2500),
+ CreateInput("SELECT * FROM c", PartitionKeyValue, true, -1, 2500),
+
+ //TOP
+ CreateInput("SELECT TOP 1000 c.id FROM c", PartitionKeyValue, false, -1, 1000),
+ CreateInput("SELECT TOP 1000 c.id FROM c", PartitionKeyValue, true, -1, 1000),
+
+ //Filter
+ CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", PartitionKeyValue, false, -1, 1250),
+ CreateInput("SELECT c.id FROM c WHERE c.city IN ('Seattle', 'NYC')", PartitionKeyValue, true, -1, 1250),
+
+ //DISTINCT + Filter
+ CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", PartitionKeyValue, false, -1, 3),
+ CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 OFFSET 1 LIMIT 3", PartitionKeyValue, true, -1, 3),
+
+ CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", PartitionKeyValue, false, -1, 2),
+ CreateInput("SELECT DISTINCT c.city FROM c WHERE STARTSWITH(c.city, 'S')", PartitionKeyValue, true, -1, 2),
+
+ //JOIN
+ CreateInput("SELECT root.id " +
+ "FROM root " +
+ "JOIN root.id a " +
+ "JOIN root.id b " +
+ "JOIN root.id c " +
+ "WHERE root.id = '1' OR a.id in (1,2,3,4,5,6,7,8,9,10) " +
+ "OR b.id in (1,2,3,4,5,6,7,8,9,10) " +
+ "OR c.id in (1,2,3,4,5,6,7,8,9,10)", PartitionKeyValue, false, -1, 1),
+ CreateInput("SELECT root.id " +
+ "FROM root " +
+ "JOIN root.id a " +
+ "JOIN root.id b " +
+ "JOIN root.id c " +
+ "WHERE root.id = '1' OR a.id in (1,2,3,4,5,6,7,8,9,10) " +
+ "OR b.id in (1,2,3,4,5,6,7,8,9,10) " +
+ "OR c.id in (1,2,3,4,5,6,7,8,9,10)", PartitionKeyValue, true, -1, 1),
+
+ //High Prep Time
+ CreateInput(highPrepTimeSumQuery, PartitionKeyValue, false, -1, 2500),
+ CreateInput(highPrepTimeSumQuery, PartitionKeyValue, true, -1, 2500),
+
+ CreateInput(highPrepTimeConditionalQuery, PartitionKeyValue, false, -1, 1750),
+ CreateInput(highPrepTimeConditionalQuery, PartitionKeyValue, true, -1, 1750),
+
+ //Order By
+ CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", PartitionKeyValue, false, -1, 2500),
+ CreateInput("SELECT * FROM c ORDER BY c.userDefinedId DESC", PartitionKeyValue, true, -1, 2500),
+
+ CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", PartitionKeyValue, false, -1, 2500),
+ CreateInput("SELECT c.id FROM c ORDER BY c.postalcode DESC", PartitionKeyValue, true, -1, 2500),
+
+ //Order By + TOP
+ CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", PartitionKeyValue, false, -1, 5),
+ CreateInput("SELECT TOP 5 c.id FROM c ORDER BY c.userDefinedId", PartitionKeyValue, true, -1, 5),
+
+ //Order By + DISTINCT
+ CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", PartitionKeyValue, false, -1, 2500),
+ CreateInput("SELECT DISTINCT c.id FROM c ORDER BY c.city DESC", PartitionKeyValue, true, -1, 2500),
+
+ //Order By + DISTINCT + Filter
+ CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", PartitionKeyValue, false, -1, 4),
+ CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId > 5 ORDER BY c.userDefinedId", PartitionKeyValue, true, -1, 4),
+
+ CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", PartitionKeyValue, false, -1, 6),
+ CreateInput("SELECT DISTINCT c.userDefinedId FROM c WHERE c.userDefinedId BETWEEN 0 AND 5 ORDER BY c.id DESC", PartitionKeyValue, true, -1, 6),
+
+ //Group By
+ CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, false, -1, 2500),
+ CreateInput("SELECT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, true, -1, 2500),
+
+ CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", PartitionKeyValue, false, -1, 2),
+ CreateInput("SELECT Count(1) AS count, Sum(ARRAY_LENGTH(c.recipientList)) AS sum FROM c WHERE c.city IN ('Seattle', 'SF') GROUP BY c.city", PartitionKeyValue, true, -1, 2),
+
+ CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", PartitionKeyValue, false, -1, 4),
+ CreateInput("SELECT c.city, AVG(ARRAY_LENGTH(c.recipientList)) FROM c GROUP BY c.city", PartitionKeyValue, true, -1, 4),
+
+ //Group By + OFFSET
+ CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", PartitionKeyValue, false, -1, 3),
+ CreateInput("SELECT c.id FROM c GROUP BY c.id OFFSET 5 LIMIT 3", PartitionKeyValue, true, -1, 3),
+
+ //Group By + TOP
+ CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", PartitionKeyValue, false, -1, 25),
+ CreateInput("SELECT TOP 25 c.id FROM c GROUP BY c.id", PartitionKeyValue, true, -1, 25),
+
+ //Group By + DISTINCT
+ CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", PartitionKeyValue, false, -1, 2500),
+ CreateInput("SELECT DISTINCT c.id FROM c GROUP BY c.id", PartitionKeyValue, true, -1, 2500),
+
+ CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, false, -1, 2500),
+ CreateInput("SELECT DISTINCT c.postalcode FROM c GROUP BY c.postalcode", PartitionKeyValue, true, -1, 2500),
+ };
+
+ foreach (DirectExecutionTestCase testCase in odeTestCases)
+ {
+ globalCustomOdeStatisticsList.AddRange(await this.RunQueryAsync(testCase));
+ }
+
+ using (StreamWriter writer = new StreamWriter(new FileStream(RawDataPath, FileMode.Append, FileAccess.Write)))
+ {
+ SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, NumberOfIterations, rawData: true);
+ }
+
+ using (StreamWriter writer = new StreamWriter(new FileStream(AggregateDataPath, FileMode.Append, FileAccess.Write)))
+ {
+ SerializeODEQueryMetrics(writer, globalCustomOdeStatisticsList, NumberOfIterations, rawData: false);
+ }
+ }
+
+ private async Task>> RunQueryAsync(DirectExecutionTestCase queryInput)
+ {
+ List> odeQueryStatisticsList = new List>();
+ QueryRequestOptions requestOptions = new QueryRequestOptions()
+ {
+ MaxItemCount = queryInput.PageSizeOption,
+ EnableOptimisticDirectExecution = queryInput.EnableOptimisticDirectExecution,
+ PartitionKey = queryInput.PartitionKey,
+ };
+
+ for (int i = 0; i < NumberOfIterations + WarmupIterations; i++)
+ {
+ bool isWarmUpIteration = i < WarmupIterations;
+ using (FeedIterator iterator = this.Container.GetItemQueryIterator(
+ queryText: queryInput.Query,
+ requestOptions: requestOptions))
+ {
+ if (isWarmUpIteration)
+ {
+ while (iterator.HasMoreResults)
+ {
+ await iterator.ReadNextAsync();
+ }
+ }
+ else
+ {
+ odeQueryStatisticsList.Add(await this.GetIteratorStatistics(iterator, queryInput));
+ }
+ }
+ }
+
+ return odeQueryStatisticsList;
+ }
+
+ private async Task> GetIteratorStatistics(FeedIterator feedIterator, DirectExecutionTestCase queryInput)
+ {
+ MetricsAccumulator metricsAccumulator = new MetricsAccumulator();
+ Guid correlatedActivityId = Guid.NewGuid();
+ FeedResponse response;
+ int totalDocumentCount = 0;
+ string query;
+ bool enableOde;
+ List odeQueryStatisticsList = new List();
+
+ while (feedIterator.HasMoreResults)
+ {
+ QueryStatisticsDatumVisitor queryStatisticsDatumVisitor = new QueryStatisticsDatumVisitor();
+ System.Diagnostics.Stopwatch totalTime = new System.Diagnostics.Stopwatch();
+ System.Diagnostics.Stopwatch traceTime = new System.Diagnostics.Stopwatch();
+
+ totalTime.Start();
+ response = await feedIterator.ReadNextAsync();
+ traceTime.Start();
+ if (response.RequestCharge != 0)
+ {
+ metricsAccumulator.ReadFromTrace(response, queryStatisticsDatumVisitor);
+ }
+
+ traceTime.Stop();
+ totalTime.Stop();
+
+ query = queryInput.Query;
+ enableOde = queryInput.EnableOptimisticDirectExecution;
+ queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - traceTime.ElapsedMilliseconds);
+ queryStatisticsDatumVisitor.PopulateMetrics();
+
+ QueryStatisticsMetrics queryStatistics = queryStatisticsDatumVisitor.QueryMetricsList[0];
+ queryStatistics.RUCharge = response.RequestCharge;
+ queryStatistics.CorrelatedActivityId = correlatedActivityId;
+
+ // Each roundtrip is a new item in the list
+ odeQueryStatisticsList.Add(new OdeQueryStatistics
+ {
+ Query = query,
+ EnableOde = enableOde,
+ QueryStatisticsMetrics = queryStatistics
+ });
+
+ totalDocumentCount += response.Count;
+ }
+
+ Assert.AreEqual(queryInput.ExpectedResultCount, totalDocumentCount);
+
+ return odeQueryStatisticsList;
+ }
+
+ private static string CreateHighPrepTimeSumQuery()
+ {
+ int exprCount = 9999;
+ StringBuilder sb = new StringBuilder();
+ sb.Append("SELECT r.id FROM root r WHERE ");
+ for (int i = 0; i < exprCount; i++)
+ {
+ sb.Append(i == 0 ? "1" : "+1");
+ }
+
+ return sb.Append(" = " + exprCount + " ORDER BY r.id ASC").ToString();
+ }
+
+ private static string CreateHighPrepTimeConditionalQuery()
+ {
+ int exprCount = 999;
+ StringBuilder sb = new StringBuilder();
+ string[] cityOptions = new string[] { "Seattle", "Chicago", "NYC", "SF" };
+
+ sb.Append("SELECT * FROM root r WHERE ");
+ for (int nIdx = 0; nIdx < exprCount; nIdx++)
+ {
+ if (nIdx > 0)
+ {
+ sb.Append(" OR ");
+ }
+
+ sb.Append($"r.userDefinedId > {nIdx} AND r.city = '{cityOptions[nIdx % cityOptions.Length]}'");
+ }
+
+ return sb.ToString();
+ }
+
+ private static void SerializeODEQueryMetrics(TextWriter textWriter, List> customOdeStatisticsList, int numberOfIterations, bool rawData)
+ {
+ if (rawData)
+ {
+ SerializeODERawDataQueryMetrics(textWriter, customOdeStatisticsList);
+ }
+ else
+ {
+ SerializeODEProcessedDataQueryMetrics(textWriter, customOdeStatisticsList, numberOfIterations);
+ }
+ }
+
+ private static void SerializeODERawDataQueryMetrics(TextWriter textWriter, List> globalOdeQueryStatisticsList)
+ {
+ textWriter.WriteLine();
+ textWriter.WriteLine(PrintQueryMetrics);
+ textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"", "Query", "ODE", "RUCharge", "BackendTime", "TransitTime", "ClientTime", "EndToEndTime");
+ textWriter.WriteLine();
+
+ foreach (List queryStatisticsList in globalOdeQueryStatisticsList)
+ {
+ double totalClientTime = 0;
+ double totalBackendTime = 0;
+ double totalEndToEndTime = 0;
+ double totalTransitTime = 0;
+ double totalRU = 0;
+ string query = "";
+ bool ode = false;
+
+ foreach (OdeQueryStatistics queryStatistics in queryStatisticsList)
+ {
+ QueryStatisticsMetrics metrics = queryStatistics.QueryStatisticsMetrics;
+ double transitTime = metrics.Created + metrics.ChannelAcquisitionStarted + metrics.Pipelined + metrics.Received + metrics.Completed;
+ double backendTime = metrics.TotalQueryExecutionTime;
+
+ totalClientTime += metrics.EndToEndTime - (backendTime + transitTime);
+ totalBackendTime += backendTime;
+ totalEndToEndTime += metrics.EndToEndTime;
+ totalTransitTime += transitTime;
+ totalRU += metrics.RUCharge;
+ query = queryStatistics.Query;
+ ode = queryStatistics.EnableOde;
+ }
+
+ textWriter.WriteLine($"{query},{ode},{totalRU},{totalBackendTime},{totalTransitTime},{totalClientTime},{totalEndToEndTime}");
+ }
+ }
+
+ private static void SerializeODEProcessedDataQueryMetrics(TextWriter textWriter, List> globalOdeQueryStatisticsList, int numberOfIterations)
+ {
+ textWriter.WriteLine();
+ textWriter.WriteLine(PrintQueryMetrics);
+ textWriter.Write("\"{0}\",\"{1}\",\"{2}\",\"{3}\"", "Query", "ODE", "RUCharge", "EndToEndTime");
+ textWriter.WriteLine();
+
+ string prevQuery = globalOdeQueryStatisticsList[0][0].Query;
+ bool prevOde = globalOdeQueryStatisticsList[0][0].EnableOde;
+ double totalEndToEndTime = 0;
+ double totalRU = 0;
+
+ foreach (List odeQueryStatisticsList in globalOdeQueryStatisticsList)
+ {
+ if (odeQueryStatisticsList[0].Query == prevQuery && odeQueryStatisticsList[0].EnableOde == prevOde)
+ {
+ foreach (OdeQueryStatistics odeQueryStatistics in odeQueryStatisticsList)
+ {
+ QueryStatisticsMetrics metrics = odeQueryStatistics.QueryStatisticsMetrics;
+ totalEndToEndTime += metrics.EndToEndTime;
+ totalRU += metrics.RUCharge;
+ }
+ }
+ else
+ {
+ textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}");
+
+ foreach (OdeQueryStatistics odeQueryStatistics in odeQueryStatisticsList)
+ {
+ QueryStatisticsMetrics metrics = odeQueryStatistics.QueryStatisticsMetrics;
+ totalEndToEndTime = metrics.EndToEndTime;
+ totalRU = metrics.RUCharge;
+ prevQuery = odeQueryStatistics.Query;
+ prevOde = odeQueryStatistics.EnableOde;
+ }
+ }
+ }
+
+ textWriter.WriteLine($"{prevQuery},{prevOde},{totalRU / numberOfIterations},{totalEndToEndTime / numberOfIterations}");
+ }
+
+ private class OdeQueryStatistics
+ {
+ public string Query { get; set; }
+ public bool EnableOde { get; set; }
+ public QueryStatisticsMetrics QueryStatisticsMetrics { get; set; }
+ }
+
+ private static DirectExecutionTestCase CreateInput(
+ string query,
+ PartitionKey? partitionKey,
+ bool enableOptimisticDirectExecution,
+ int pageSizeOption,
+ int expectedResultCount)
+ {
+ return new DirectExecutionTestCase(query, partitionKey, enableOptimisticDirectExecution, pageSizeOption, expectedResultCount);
+ }
+
+ private readonly struct DirectExecutionTestCase
+ {
+ public string Query { get; }
+ public PartitionKey? PartitionKey { get; }
+ public bool EnableOptimisticDirectExecution { get; }
+ public int PageSizeOption { get; }
+ public int ExpectedResultCount { get; }
+
+ public DirectExecutionTestCase(
+ string query,
+ PartitionKey? partitionKey,
+ bool enableOptimisticDirectExecution,
+ int pageSizeOption,
+ int expectedResultCount)
+ {
+ this.Query = query;
+ this.PartitionKey = partitionKey;
+ this.EnableOptimisticDirectExecution = enableOptimisticDirectExecution;
+ this.PageSizeOption = pageSizeOption;
+ this.ExpectedResultCount = expectedResultCount;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs
index 70ed146f4c..d72193fc43 100644
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/QueryPerfTest/QueryStatisticsMetrics.cs
@@ -1,7 +1,11 @@
namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
{
+ using System;
+
internal class QueryStatisticsMetrics
{
+ public Guid CorrelatedActivityId { get; set; }
+
public double EndToEndTime { get; set; }
public double PocoTime { get; set; }
@@ -32,6 +36,8 @@ internal class QueryStatisticsMetrics
public double Received { get; set; }
+ public double RUCharge { get; set; }
+
public double Completed { get; set; }
public double BadRequestCreated { get; set; }
diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json
index 51ecc8a707..611ba8c677 100644
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/settings.json
@@ -11,14 +11,14 @@
"MasterKey": "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
"ServerStalenessIntervalInSeconds": "5",
"MasterStalenessIntervalInSeconds": "1",
- "ContentSerializationPerformanceTests.CosmosDatabaseId": "db",
- "ContentSerializationPerformanceTests.ContainerId": "container",
- "ContentSerializationPerformanceTests.ContentSerialization": "JsonText",
- "ContentSerializationPerformanceTests.Query": "SELECT TOP 1 c.region FROM c",
- "ContentSerializationPerformanceTests.MaxConcurrency": "-1",
- "ContentSerializationPerformanceTests.MaxItemCount": "-1",
- "ContentSerializationPerformanceTests.NumberOfIterations": "1",
- "ContentSerializationPerformanceTests.WarmupIterations": "0",
- "ContentSerializationPerformanceTests.UseStronglyTypedIterator": "true"
+ "QueryPerformanceTests.CosmosDatabaseId": "db",
+ "QueryPerformanceTests.ContainerId": "container",
+ "QueryPerformanceTests.ContentSerialization": "JsonText",
+ "QueryPerformanceTests.Query": "SELECT TOP 1 c.region FROM c",
+ "QueryPerformanceTests.MaxConcurrency": "-1",
+ "QueryPerformanceTests.MaxItemCount": "-1",
+ "QueryPerformanceTests.NumberOfIterations": "1",
+ "QueryPerformanceTests.WarmupIterations": "0",
+ "QueryPerformanceTests.UseStronglyTypedIterator": "true"
}
}
\ No newline at end of file
From c46614b9311365f203d489394cf475e212ca202f Mon Sep 17 00:00:00 2001
From: akotalwar <94020786+akotalwar@users.noreply.github.com>
Date: Sat, 5 Aug 2023 13:44:45 -0700
Subject: [PATCH 05/20] Query: Adds ODE continuation token support for non-ODE
pipelines (#4009)
* Added code to throw exception if ODE continuation token goes into non ODE pipeline
* Removed count variable
* Updated test name
* Removed ODE continuation token logic from caller class
* Simplified code
* Fixed comments
* Updated continuation token cast
* Removed const string for continuation token
* Added Ignore flag for test
* Added baseline test
* Updated baseline test
---
.../CosmosQueryExecutionContextFactory.cs | 12 ++++-
...imisticDirectExecutionContinuationToken.cs | 16 +++---
.../CosmosBasicQueryTests.cs | 53 ++++++++++++++++++-
...misticDirectExecutionQueryBaselineTests.cs | 44 +++++++++++++++
4 files changed, 117 insertions(+), 8 deletions(-)
diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs
index 47aa1c4490..43ebf6a2d8 100644
--- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs
+++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs
@@ -752,7 +752,17 @@ private static Documents.PartitionKeyDefinition GetPartitionKeyDefinition(InputP
ContainerQueryProperties containerQueryProperties,
ITrace trace)
{
- if (!inputParameters.EnableOptimisticDirectExecution) return null;
+ if (!inputParameters.EnableOptimisticDirectExecution)
+ {
+ if (inputParameters.InitialUserContinuationToken != null
+ && OptimisticDirectExecutionContinuationToken.IsOptimisticDirectExecutionContinuationToken(inputParameters.InitialUserContinuationToken))
+ {
+ throw new MalformedContinuationTokenException($"The continuation token supplied requires the Optimistic Direct Execution flag to be enabled in QueryRequestOptions for the query execution to resume. " +
+ $"{inputParameters.InitialUserContinuationToken}");
+ }
+
+ return null;
+ }
Debug.Assert(containerQueryProperties.ResourceId != null, "CosmosQueryExecutionContextFactory Assert!", "Container ResourceId cannot be null!");
diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/OptimisticDirectExecution/OptimisticDirectExecutionContinuationToken.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/OptimisticDirectExecution/OptimisticDirectExecutionContinuationToken.cs
index 59f87295f3..76b2531f01 100644
--- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/OptimisticDirectExecution/OptimisticDirectExecutionContinuationToken.cs
+++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/OptimisticDirectExecution/OptimisticDirectExecutionContinuationToken.cs
@@ -4,9 +4,7 @@
namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.OptimisticDirectExecutionQuery
{
- using System;
using System.Collections.Generic;
- using Microsoft.Azure.Cosmos.ChangeFeed;
using Microsoft.Azure.Cosmos.CosmosElements;
using Microsoft.Azure.Cosmos.Query.Core.Exceptions;
using Microsoft.Azure.Cosmos.Query.Core.Monads;
@@ -30,6 +28,12 @@ public OptimisticDirectExecutionContinuationToken(ParallelContinuationToken toke
public Range Range => this.Token.Range;
+ public static bool IsOptimisticDirectExecutionContinuationToken(CosmosElement continuationToken)
+ {
+ CosmosObject cosmosObjectContinuationToken = continuationToken as CosmosObject;
+ return !(cosmosObjectContinuationToken == null || !cosmosObjectContinuationToken.ContainsKey(OptimisticDirectExecutionToken));
+ }
+
public static CosmosElement ToCosmosElement(OptimisticDirectExecutionContinuationToken continuationToken)
{
CosmosElement inner = ParallelContinuationToken.ToCosmosElement(continuationToken.Token);
@@ -42,14 +46,14 @@ public static CosmosElement ToCosmosElement(OptimisticDirectExecutionContinuatio
public static TryCatch TryCreateFromCosmosElement(CosmosElement cosmosElement)
{
- CosmosObject cosmosObjectContinuationToken = cosmosElement as CosmosObject;
- if (cosmosObjectContinuationToken == null || !cosmosObjectContinuationToken.ContainsKey(OptimisticDirectExecutionToken))
+ if (!IsOptimisticDirectExecutionContinuationToken(cosmosElement))
{
return TryCatch.FromException(
- new MalformedContinuationTokenException(
- message: $"Malformed Continuation Token: Expected OptimisticDirectExecutionToken\r\n"));
+ new MalformedContinuationTokenException(
+ message: $"Malformed Continuation Token: Expected OptimisticDirectExecutionToken\r\n"));
}
+ CosmosObject cosmosObjectContinuationToken = (CosmosObject)cosmosElement;
TryCatch inner = ParallelContinuationToken.TryCreateFromCosmosElement(cosmosObjectContinuationToken[OptimisticDirectExecutionToken]);
return inner.Succeeded ?
diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosBasicQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosBasicQueryTests.cs
index 25ac9a705f..bfd6366104 100644
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosBasicQueryTests.cs
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosBasicQueryTests.cs
@@ -15,7 +15,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
using Cosmos.Scripts;
using Microsoft.Azure.Cosmos.Fluent;
using Microsoft.Azure.Cosmos.Linq;
- using Microsoft.Azure.Cosmos.Query.Core;
using Microsoft.Azure.Cosmos.Tracing;
using Microsoft.Azure.Documents.Collections;
using Microsoft.VisualStudio.TestTools.UnitTesting;
@@ -786,6 +785,58 @@ public async Task QueryActivityIdWithContinuationTokenAndTraceTest()
}
+ //TODO: Remove Ignore flag once emulator is updated to 0415
+ [Ignore]
+ [TestMethod]
+ public async Task TesOdeTokenCompatibilityWithNonOdePipeline()
+ {
+ string query = "select top 200 * from c";
+ CosmosClient client = DirectCosmosClient;
+ Container container = client.GetContainer(DatabaseId, ContainerId);
+
+ // Create items
+ for (int i = 0; i < 500; i++)
+ {
+ await container.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity());
+ }
+
+ QueryRequestOptions queryRequestOptions = new QueryRequestOptions
+ {
+ MaxItemCount = 50,
+ EnableOptimisticDirectExecution = true
+ };
+
+ FeedIteratorInternal feedIterator =
+ (FeedIteratorInternal)container.GetItemQueryStreamIterator(
+ query,
+ null,
+ queryRequestOptions);
+
+ ResponseMessage responseMessage = await feedIterator.ReadNextAsync(CancellationToken.None);
+ string continuationToken = responseMessage.ContinuationToken;
+
+ QueryRequestOptions newQueryRequestOptions = new QueryRequestOptions
+ {
+ MaxItemCount = 50,
+ EnableOptimisticDirectExecution = false
+ };
+
+ // use Continuation Token to create new iterator and use same trace
+ FeedIterator feedIteratorNew =
+ container.GetItemQueryStreamIterator(
+ query,
+ continuationToken,
+ newQueryRequestOptions);
+
+ while (feedIteratorNew.HasMoreResults)
+ {
+ responseMessage = await feedIteratorNew.ReadNextAsync(CancellationToken.None);
+ }
+
+ string expectedErrorMessage = "The continuation token supplied requires the Optimistic Direct Execution flag to be enabled in QueryRequestOptions for the query execution to resume. ";
+ Assert.IsTrue(responseMessage.CosmosException.ToString().Contains(expectedErrorMessage));
+ }
+
private class CustomHandler : RequestHandler
{
string correlatedActivityId;
diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/OptimisticDirectExecutionQueryBaselineTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/OptimisticDirectExecutionQueryBaselineTests.cs
index 81c765adc4..5c0e66f1a6 100644
--- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/OptimisticDirectExecutionQueryBaselineTests.cs
+++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/OptimisticDirectExecutionQueryBaselineTests.cs
@@ -19,6 +19,7 @@
using Microsoft.Azure.Cosmos.Query.Core.Pipeline;
using Microsoft.Azure.Cosmos.Query.Core.Pipeline.CrossPartition.OrderBy;
using Microsoft.Azure.Cosmos.Query.Core.Pipeline.CrossPartition.Parallel;
+ using Microsoft.Azure.Cosmos.Query.Core.Pipeline.OptimisticDirectExecutionQuery;
using Microsoft.Azure.Cosmos.Query.Core.Pipeline.Pagination;
using Microsoft.Azure.Cosmos.Query.Core.QueryClient;
using Microsoft.Azure.Cosmos.Query.Core.QueryPlan;
@@ -190,6 +191,49 @@ public async Task TestPipelineForBackendDocumentsOnSinglePartitionAsync()
Assert.AreEqual(100, documentCountInSinglePartition);
}
+ [TestMethod]
+ public async Task TestOdeTokenWithSpecializedPipeline()
+ {
+ int numItems = 100;
+ ParallelContinuationToken parallelContinuationToken = new ParallelContinuationToken(
+ token: Guid.NewGuid().ToString(),
+ range: new Documents.Routing.Range("A", "B", true, false));
+
+ OptimisticDirectExecutionContinuationToken optimisticDirectExecutionContinuationToken = new OptimisticDirectExecutionContinuationToken(parallelContinuationToken);
+ CosmosElement cosmosElementContinuationToken = OptimisticDirectExecutionContinuationToken.ToCosmosElement(optimisticDirectExecutionContinuationToken);
+
+ OptimisticDirectExecutionTestInput input = CreateInput(
+ description: @"Single Partition Key and Value Field",
+ query: "SELECT VALUE COUNT(1) FROM c",
+ expectedOptimisticDirectExecution: false,
+ partitionKeyPath: @"/pk",
+ partitionKeyValue: "a",
+ continuationToken: cosmosElementContinuationToken);
+
+ DocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems, multiPartition: false);
+ QueryRequestOptions queryRequestOptions = GetQueryRequestOptions(enableOptimisticDirectExecution: input.ExpectedOptimisticDirectExecution);
+ (CosmosQueryExecutionContextFactory.InputParameters inputParameters, CosmosQueryContextCore cosmosQueryContextCore) = CreateInputParamsAndQueryContext(input, queryRequestOptions);
+ IQueryPipelineStage queryPipelineStage = CosmosQueryExecutionContextFactory.Create(
+ documentContainer,
+ cosmosQueryContextCore,
+ inputParameters,
+ NoOpTrace.Singleton);
+
+ string expectedErrorMessage = "The continuation token supplied requires the Optimistic Direct Execution flag to be enabled in QueryRequestOptions for the query execution to resume. ";
+
+ while (await queryPipelineStage.MoveNextAsync(NoOpTrace.Singleton))
+ {
+ if (queryPipelineStage.Current.Failed)
+ {
+ Assert.IsTrue(queryPipelineStage.Current.InnerMostException.ToString().Contains(expectedErrorMessage));
+ return;
+ }
+
+ Assert.IsFalse(true);
+ break;
+ }
+ }
+
[TestMethod]
public async Task TestQueriesWhichNeverRequireDistribution()
{
From e05e3eb23c4afd241dc24f3a8872f3ab5b9f431f Mon Sep 17 00:00:00 2001
From: Debdatta Kunda <87335885+kundadebdatta@users.noreply.github.com>
Date: Tue, 8 Aug 2023 14:39:37 -0700
Subject: [PATCH 06/20] Code changes to disable replica validation in preview
package. (#4019)
---
Microsoft.Azure.Cosmos/src/Util/ConfigurationManager.cs | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/Microsoft.Azure.Cosmos/src/Util/ConfigurationManager.cs b/Microsoft.Azure.Cosmos/src/Util/ConfigurationManager.cs
index 216e1295b5..748f81833a 100644
--- a/Microsoft.Azure.Cosmos/src/Util/ConfigurationManager.cs
+++ b/Microsoft.Azure.Cosmos/src/Util/ConfigurationManager.cs
@@ -38,9 +38,7 @@ public static bool IsReplicaAddressValidationEnabled(
ConnectionPolicy connectionPolicy)
{
bool replicaValidationDefaultValue = false;
-#if PREVIEW
- replicaValidationDefaultValue = true;
-#endif
+
if (connectionPolicy != null
&& connectionPolicy.EnableAdvancedReplicaSelectionForTcp.HasValue)
{
From 27b935072764a901b3eed4b038aff7956f2d297e Mon Sep 17 00:00:00 2001
From: Debdatta Kunda <87335885+kundadebdatta@users.noreply.github.com>
Date: Tue, 8 Aug 2023 21:08:02 -0700
Subject: [PATCH 07/20] 3.35.3: Adds new SDK versions and contract files
(#4024)
* Updated change log and bumped up the patch version.
* Code changes to update the change log message.
---
Directory.Build.props | 4 +-
.../contracts/API_3.35.3-preview.txt | 1563 +++++++++++++++++
.../contracts/API_3.35.3.txt | 1502 ++++++++++++++++
changelog.md | 9 +
4 files changed, 3076 insertions(+), 2 deletions(-)
create mode 100644 Microsoft.Azure.Cosmos/contracts/API_3.35.3-preview.txt
create mode 100644 Microsoft.Azure.Cosmos/contracts/API_3.35.3.txt
diff --git a/Directory.Build.props b/Directory.Build.props
index 9daa1e7e9f..d3ceab0046 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -1,7 +1,7 @@
- 3.35.2
- 3.35.2
+ 3.35.3
+ 3.35.3
preview
3.31.3
2.0.3
diff --git a/Microsoft.Azure.Cosmos/contracts/API_3.35.3-preview.txt b/Microsoft.Azure.Cosmos/contracts/API_3.35.3-preview.txt
new file mode 100644
index 0000000000..6c989053fd
--- /dev/null
+++ b/Microsoft.Azure.Cosmos/contracts/API_3.35.3-preview.txt
@@ -0,0 +1,1563 @@
+namespace Microsoft.Azure.Cosmos
+{
+ public class AccountConsistency
+ {
+ public AccountConsistency();
+ public ConsistencyLevel DefaultConsistencyLevel { get; }
+ public int MaxStalenessIntervalInSeconds { get; }
+ public int MaxStalenessPrefix { get; }
+ }
+ public class AccountProperties
+ {
+ public AccountConsistency Consistency { get; }
+ public string ETag { get; }
+ public string Id { get; }
+ public IEnumerable ReadableRegions { get; }
+ public IEnumerable WritableRegions { get; }
+ }
+ public class AccountRegion
+ {
+ public AccountRegion();
+ public string Endpoint { get; }
+ public string Name { get; }
+ }
+ public sealed class BoundingBoxProperties
+ {
+ public BoundingBoxProperties();
+ public double Xmax { get; set; }
+ public double Xmin { get; set; }
+ public double Ymax { get; set; }
+ public double Ymin { get; set; }
+ }
+ public abstract class ChangeFeedEstimator
+ {
+ protected ChangeFeedEstimator();
+ public abstract FeedIterator GetCurrentStateIterator(ChangeFeedEstimatorRequestOptions changeFeedEstimatorRequestOptions=null);
+ }
+ public sealed class ChangeFeedEstimatorRequestOptions
+ {
+ public ChangeFeedEstimatorRequestOptions();
+ public Nullable MaxItemCount { get; set; }
+ }
+ public class ChangeFeedItemChange
+ {
+ public ChangeFeedItemChange();
+ public T Current { get; set; }
+ public ChangeFeedMetadata Metadata { get; set; }
+ public T Previous { get; set; }
+ }
+ public class ChangeFeedMetadata
+ {
+ public ChangeFeedMetadata(DateTime conflictResolutionTimestamp, long lsn, ChangeFeedOperationType operationType, long previousLsn);
+ public DateTime ConflictResolutionTimestamp { get; }
+ public bool IsTimeToLiveExpired { get; }
+ public long Lsn { get; }
+ public ChangeFeedOperationType OperationType { get; }
+ public long PreviousLsn { get; }
+ }
+ public abstract class ChangeFeedMode
+ {
+ public static ChangeFeedMode AllVersionsAndDeletes { get; }
+ public static ChangeFeedMode Incremental { get; }
+ public static ChangeFeedMode LatestVersion { get; }
+ }
+ public enum ChangeFeedOperationType
+ {
+ Create = 0,
+ Delete = 2,
+ Replace = 1,
+ }
+ public sealed class ChangeFeedPolicy
+ {
+ public ChangeFeedPolicy();
+ public static TimeSpan FullFidelityNoRetention { get; }
+ public TimeSpan FullFidelityRetention { get; set; }
+ }
+ public abstract class ChangeFeedProcessor
+ {
+ protected ChangeFeedProcessor();
+ public abstract Task StartAsync();
+ public abstract Task StopAsync();
+ }
+ public class ChangeFeedProcessorBuilder
+ {
+ public ChangeFeedProcessor Build();
+ public ChangeFeedProcessorBuilder WithErrorNotification(Container.ChangeFeedMonitorErrorDelegate errorDelegate);
+ public ChangeFeedProcessorBuilder WithInstanceName(string instanceName);
+ public ChangeFeedProcessorBuilder WithLeaseAcquireNotification(Container.ChangeFeedMonitorLeaseAcquireDelegate acquireDelegate);
+ public ChangeFeedProcessorBuilder WithLeaseConfiguration(Nullable acquireInterval=default(Nullable), Nullable expirationInterval=default(Nullable), Nullable renewInterval=default(Nullable));
+ public ChangeFeedProcessorBuilder WithLeaseContainer(Container leaseContainer);
+ public ChangeFeedProcessorBuilder WithLeaseReleaseNotification(Container.ChangeFeedMonitorLeaseReleaseDelegate releaseDelegate);
+ public ChangeFeedProcessorBuilder WithMaxItems(int maxItemCount);
+ public ChangeFeedProcessorBuilder WithPollInterval(TimeSpan pollInterval);
+ public ChangeFeedProcessorBuilder WithStartTime(DateTime startTime);
+ }
+ public abstract class ChangeFeedProcessorContext
+ {
+ protected ChangeFeedProcessorContext();
+ public abstract CosmosDiagnostics Diagnostics { get; }
+ public abstract Headers Headers { get; }
+ public abstract string LeaseToken { get; }
+ }
+ public sealed class ChangeFeedProcessorState
+ {
+ public ChangeFeedProcessorState(string leaseToken, long estimatedLag, string instanceName);
+ public long EstimatedLag { get; }
+ public string InstanceName { get; }
+ public string LeaseToken { get; }
+ }
+ public class ChangeFeedProcessorUserException : Exception
+ {
+ public ChangeFeedProcessorUserException(Exception originalException, ChangeFeedProcessorContext context);
+ protected ChangeFeedProcessorUserException(SerializationInfo info, StreamingContext context);
+ public ChangeFeedProcessorContext ChangeFeedProcessorContext { get; }
+ public override void GetObjectData(SerializationInfo info, StreamingContext context);
+ }
+ public sealed class ChangeFeedRequestOptions : RequestOptions
+ {
+ public ChangeFeedRequestOptions();
+ public new string IfMatchEtag { get; set; }
+ public new string IfNoneMatchEtag { get; set; }
+ public Nullable PageSizeHint { get; set; }
+ }
+ public abstract class ChangeFeedStartFrom
+ {
+ public static ChangeFeedStartFrom Beginning();
+ public static ChangeFeedStartFrom Beginning(FeedRange feedRange);
+ public static ChangeFeedStartFrom ContinuationToken(string continuationToken);
+ public static ChangeFeedStartFrom Now();
+ public static ChangeFeedStartFrom Now(FeedRange feedRange);
+ public static ChangeFeedStartFrom Time(DateTime dateTimeUtc);
+ public static ChangeFeedStartFrom Time(DateTime dateTimeUtc, FeedRange feedRange);
+ }
+ public sealed class ClientEncryptionIncludedPath
+ {
+ public ClientEncryptionIncludedPath();
+ public string ClientEncryptionKeyId { get; set; }
+ public string EncryptionAlgorithm { get; set; }
+ public string EncryptionType { get; set; }
+ public string Path { get; set; }
+ }
+ public abstract class ClientEncryptionKey
+ {
+ protected ClientEncryptionKey();
+ public abstract string Id { get; }
+ public abstract Task ReadAsync(RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task ReplaceAsync(ClientEncryptionKeyProperties clientEncryptionKeyProperties, RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ }
+ public class ClientEncryptionKeyProperties : IEquatable
+ {
+ protected ClientEncryptionKeyProperties();
+ public ClientEncryptionKeyProperties(string id, string encryptionAlgorithm, byte[] wrappedDataEncryptionKey, EncryptionKeyWrapMetadata encryptionKeyWrapMetadata);
+ public Nullable CreatedTime { get; }
+ public string EncryptionAlgorithm { get; }
+ public EncryptionKeyWrapMetadata EncryptionKeyWrapMetadata { get; }
+ public string ETag { get; }
+ public string Id { get; }
+ public Nullable LastModified { get; }
+ public virtual string SelfLink { get; }
+ public byte[] WrappedDataEncryptionKey { get; }
+ public bool Equals(ClientEncryptionKeyProperties other);
+ public override bool Equals(object obj);
+ public override int GetHashCode();
+ }
+ public class ClientEncryptionKeyResponse : Response
+ {
+ protected ClientEncryptionKeyResponse();
+ public override string ActivityId { get; }
+ public virtual ClientEncryptionKey ClientEncryptionKey { get; }
+ public override CosmosDiagnostics Diagnostics { get; }
+ public override string ETag { get; }
+ public override Headers Headers { get; }
+ public override double RequestCharge { get; }
+ public override ClientEncryptionKeyProperties Resource { get; }
+ public override HttpStatusCode StatusCode { get; }
+ public static implicit operator ClientEncryptionKey (ClientEncryptionKeyResponse response);
+ }
+ public sealed class ClientEncryptionPolicy
+ {
+ public ClientEncryptionPolicy(IEnumerable includedPaths);
+ public ClientEncryptionPolicy(IEnumerable includedPaths, int policyFormatVersion);
+ public IEnumerable IncludedPaths { get; }
+ public int PolicyFormatVersion { get; }
+ }
+ public sealed class CompositePath
+ {
+ public CompositePath();
+ public CompositePathSortOrder Order { get; set; }
+ public string Path { get; set; }
+ }
+ public enum CompositePathSortOrder
+ {
+ Ascending = 0,
+ Descending = 1,
+ }
+ public sealed class ComputedProperty
+ {
+ public ComputedProperty();
+ public string Name { get; set; }
+ public string Query { get; set; }
+ }
+ public class ConflictProperties
+ {
+ public ConflictProperties();
+ public string Id { get; }
+ public OperationKind OperationKind { get; }
+ public string SelfLink { get; }
+ }
+ public enum ConflictResolutionMode
+ {
+ Custom = 1,
+ LastWriterWins = 0,
+ }
+ public class ConflictResolutionPolicy
+ {
+ public ConflictResolutionPolicy();
+ public ConflictResolutionMode Mode { get; set; }
+ public string ResolutionPath { get; set; }
+ public string ResolutionProcedure { get; set; }
+ }
+ public abstract class Conflicts
+ {
+ protected Conflicts();
+ public abstract Task DeleteAsync(ConflictProperties conflict, PartitionKey partitionKey, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract FeedIterator GetConflictQueryIterator(QueryDefinition queryDefinition, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public abstract FeedIterator GetConflictQueryIterator(string queryText=null, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public abstract FeedIterator GetConflictQueryStreamIterator(QueryDefinition queryDefinition, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public abstract FeedIterator GetConflictQueryStreamIterator(string queryText=null, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public abstract T ReadConflictContent(ConflictProperties conflict);
+ public abstract Task> ReadCurrentAsync(ConflictProperties conflict, PartitionKey partitionKey, CancellationToken cancellationToken=default(CancellationToken));
+ }
+ public enum ConnectionMode
+ {
+ Direct = 1,
+ Gateway = 0,
+ }
+ public enum ConsistencyLevel
+ {
+ BoundedStaleness = 1,
+ ConsistentPrefix = 4,
+ Eventual = 3,
+ Session = 2,
+ Strong = 0,
+ }
+ public abstract class Container
+ {
+ protected Container();
+ public abstract Conflicts Conflicts { get; }
+ public abstract Database Database { get; }
+ public abstract string Id { get; }
+ public abstract Scripts Scripts { get; }
+ public abstract Task> CreateItemAsync(T item, Nullable partitionKey=default(Nullable), ItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task CreateItemStreamAsync(Stream streamPayload, PartitionKey partitionKey, ItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract TransactionalBatch CreateTransactionalBatch(PartitionKey partitionKey);
+ public abstract Task DeleteAllItemsByPartitionKeyStreamAsync(PartitionKey partitionKey, RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task DeleteContainerAsync(ContainerRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task DeleteContainerStreamAsync(ContainerRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task> DeleteItemAsync(string id, PartitionKey partitionKey, ItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task DeleteItemStreamAsync(string id, PartitionKey partitionKey, ItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract ChangeFeedEstimator GetChangeFeedEstimator(string processorName, Container leaseContainer);
+ public abstract ChangeFeedProcessorBuilder GetChangeFeedEstimatorBuilder(string processorName, Container.ChangesEstimationHandler estimationDelegate, Nullable estimationPeriod=default(Nullable));
+ public abstract FeedIterator GetChangeFeedIterator(ChangeFeedStartFrom changeFeedStartFrom, ChangeFeedMode changeFeedMode, ChangeFeedRequestOptions changeFeedRequestOptions=null);
+ public abstract ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder(string processorName, Container.ChangeFeedStreamHandler onChangesDelegate);
+ public abstract ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManualCheckpoint(string processorName, Container.ChangeFeedStreamHandlerWithManualCheckpoint onChangesDelegate);
+ public abstract ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManualCheckpoint(string processorName, Container.ChangeFeedHandlerWithManualCheckpoint onChangesDelegate);
+ public abstract ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder(string processorName, Container.ChangeFeedHandler onChangesDelegate);
+ public abstract ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder(string processorName, Container.ChangesHandler onChangesDelegate);
+ public abstract FeedIterator GetChangeFeedStreamIterator(ChangeFeedStartFrom changeFeedStartFrom, ChangeFeedMode changeFeedMode, ChangeFeedRequestOptions changeFeedRequestOptions=null);
+ public abstract Task> GetFeedRangesAsync(CancellationToken cancellationToken=default(CancellationToken));
+ public abstract IOrderedQueryable GetItemLinqQueryable(bool allowSynchronousQueryExecution=false, string continuationToken=null, QueryRequestOptions requestOptions=null, CosmosLinqSerializerOptions linqSerializerOptions=null);
+ public abstract FeedIterator GetItemQueryIterator(FeedRange feedRange, QueryDefinition queryDefinition, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public abstract FeedIterator GetItemQueryIterator(QueryDefinition queryDefinition, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public abstract FeedIterator GetItemQueryIterator(string queryText=null, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public abstract FeedIterator GetItemQueryStreamIterator(FeedRange feedRange, QueryDefinition queryDefinition, string continuationToken, QueryRequestOptions requestOptions=null);
+ public abstract FeedIterator GetItemQueryStreamIterator(QueryDefinition queryDefinition, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public abstract FeedIterator GetItemQueryStreamIterator(string queryText=null, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public abstract Task> GetPartitionKeyRangesAsync(FeedRange feedRange, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task> PatchItemAsync(string id, PartitionKey partitionKey, IReadOnlyList patchOperations, PatchItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task PatchItemStreamAsync(string id, PartitionKey partitionKey, IReadOnlyList patchOperations, PatchItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task ReadContainerAsync(ContainerRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task ReadContainerStreamAsync(ContainerRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task> ReadItemAsync(string id, PartitionKey partitionKey, ItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task ReadItemStreamAsync(string id, PartitionKey partitionKey, ItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task> ReadManyItemsAsync(IReadOnlyList> items, ReadManyRequestOptions readManyRequestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task ReadManyItemsStreamAsync(IReadOnlyList> items, ReadManyRequestOptions readManyRequestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task ReadThroughputAsync(RequestOptions requestOptions, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task> ReadThroughputAsync(CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task ReplaceContainerAsync(ContainerProperties containerProperties, ContainerRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task ReplaceContainerStreamAsync(ContainerProperties containerProperties, ContainerRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task> ReplaceItemAsync(T item, string id, Nullable partitionKey=default(Nullable), ItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task ReplaceItemStreamAsync(Stream streamPayload, string id, PartitionKey partitionKey, ItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task ReplaceThroughputAsync(ThroughputProperties throughputProperties, RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task ReplaceThroughputAsync(int throughput, RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task> UpsertItemAsync(T item, Nullable partitionKey=default(Nullable), ItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task UpsertItemStreamAsync(Stream streamPayload, PartitionKey partitionKey, ItemRequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public delegate Task ChangeFeedHandlerWithManualCheckpoint(ChangeFeedProcessorContext context, IReadOnlyCollection changes, Func checkpointAsync, CancellationToken cancellationToken);
+ public delegate Task ChangeFeedHandler(ChangeFeedProcessorContext context, IReadOnlyCollection changes, CancellationToken cancellationToken);
+ public delegate Task ChangeFeedMonitorErrorDelegate(string leaseToken, Exception exception);
+ public delegate Task ChangeFeedMonitorLeaseAcquireDelegate(string leaseToken);
+ public delegate Task ChangeFeedMonitorLeaseReleaseDelegate(string leaseToken);
+ public delegate Task ChangeFeedStreamHandler(ChangeFeedProcessorContext context, Stream changes, CancellationToken cancellationToken);
+ public delegate Task ChangeFeedStreamHandlerWithManualCheckpoint(ChangeFeedProcessorContext context, Stream changes, Func checkpointAsync, CancellationToken cancellationToken);
+ public delegate Task ChangesEstimationHandler(long estimatedPendingChanges, CancellationToken cancellationToken);
+ public delegate Task ChangesHandler(IReadOnlyCollection changes, CancellationToken cancellationToken);
+ }
+ public class ContainerProperties
+ {
+ public ContainerProperties();
+ public ContainerProperties(string id, IReadOnlyList partitionKeyPaths);
+ public ContainerProperties(string id, string partitionKeyPath);
+ public Nullable AnalyticalStoreTimeToLiveInSeconds { get; set; }
+ public ChangeFeedPolicy ChangeFeedPolicy { get; set; }
+ public ClientEncryptionPolicy ClientEncryptionPolicy { get; set; }
+ public Collection ComputedProperties { get; set; }
+ public ConflictResolutionPolicy ConflictResolutionPolicy { get; set; }
+ public Nullable DefaultTimeToLive { get; set; }
+ public string ETag { get; }
+ public GeospatialConfig GeospatialConfig { get; set; }
+ public string Id { get; set; }
+ public IndexingPolicy IndexingPolicy { get; set; }
+ public Nullable LastModified { get; }
+ public Nullable PartitionKeyDefinitionVersion { get; set; }
+ public string PartitionKeyPath { get; set; }
+ public IReadOnlyList PartitionKeyPaths { get; set; }
+ public string SelfLink { get; }
+ public string TimeToLivePropertyPath { get; set; }
+ public UniqueKeyPolicy UniqueKeyPolicy { get; set; }
+ }
+ public class ContainerRequestOptions : RequestOptions
+ {
+ public ContainerRequestOptions();
+ public bool PopulateQuotaInfo { get; set; }
+ }
+ public class ContainerResponse : Response
+ {
+ protected ContainerResponse();
+ public override string ActivityId { get; }
+ public virtual Container Container { get; }
+ public override CosmosDiagnostics Diagnostics { get; }
+ public override string ETag { get; }
+ public override Headers Headers { get; }
+ public override double RequestCharge { get; }
+ public override ContainerProperties Resource { get; }
+ public override HttpStatusCode StatusCode { get; }
+ public static implicit operator Container (ContainerResponse response);
+ }
+ public class CosmosClient : IDisposable
+ {
+ protected CosmosClient();
+ public CosmosClient(string accountEndpoint, AzureKeyCredential authKeyOrResourceTokenCredential, CosmosClientOptions clientOptions=null);
+ public CosmosClient(string accountEndpoint, TokenCredential tokenCredential, CosmosClientOptions clientOptions=null);
+ public CosmosClient(string connectionString, CosmosClientOptions clientOptions=null);
+ public CosmosClient(string accountEndpoint, string authKeyOrResourceToken, CosmosClientOptions clientOptions=null);
+ public virtual CosmosClientOptions ClientOptions { get; }
+ public virtual Uri Endpoint { get; }
+ public virtual CosmosResponseFactory ResponseFactory { get; }
+ public static Task CreateAndInitializeAsync(string accountEndpoint, AzureKeyCredential authKeyOrResourceTokenCredential, IReadOnlyList> containers, CosmosClientOptions cosmosClientOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public static Task CreateAndInitializeAsync(string accountEndpoint, TokenCredential tokenCredential, IReadOnlyList> containers, CosmosClientOptions cosmosClientOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public static Task CreateAndInitializeAsync(string connectionString, IReadOnlyList> containers, CosmosClientOptions cosmosClientOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public static Task CreateAndInitializeAsync(string accountEndpoint, string authKeyOrResourceToken, IReadOnlyList> containers, CosmosClientOptions cosmosClientOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public virtual Task CreateDatabaseAsync(string id, ThroughputProperties throughputProperties, RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public virtual Task CreateDatabaseAsync(string id, Nullable throughput=default(Nullable), RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public virtual Task CreateDatabaseIfNotExistsAsync(string id, ThroughputProperties throughputProperties, RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public virtual Task CreateDatabaseIfNotExistsAsync(string id, Nullable throughput=default(Nullable), RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public virtual Task CreateDatabaseStreamAsync(DatabaseProperties databaseProperties, Nullable throughput=default(Nullable), RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public void Dispose();
+ protected virtual void Dispose(bool disposing);
+ public virtual Container GetContainer(string databaseId, string containerId);
+ public virtual Database GetDatabase(string id);
+ public virtual FeedIterator GetDatabaseQueryIterator(QueryDefinition queryDefinition, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public virtual FeedIterator GetDatabaseQueryIterator(string queryText=null, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public virtual FeedIterator GetDatabaseQueryStreamIterator(QueryDefinition queryDefinition, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public virtual FeedIterator GetDatabaseQueryStreamIterator(string queryText=null, string continuationToken=null, QueryRequestOptions requestOptions=null);
+ public virtual Task ReadAccountAsync();
+ }
+ public class CosmosClientOptions
+ {
+ public CosmosClientOptions();
+ public bool AllowBulkExecution { get; set; }
+ public string ApplicationName { get; set; }
+ public IReadOnlyList ApplicationPreferredRegions { get; set; }
+ public string ApplicationRegion { get; set; }
+ public ConnectionMode ConnectionMode { get; set; }
+ public Nullable ConsistencyLevel { get; set; }
+ public Collection CustomHandlers { get; }
+ public Nullable EnableContentResponseOnWrite { get; set; }
+ public bool EnableTcpConnectionEndpointRediscovery { get; set; }
+ public int GatewayModeMaxConnectionLimit { get; set; }
+ public Func HttpClientFactory { get; set; }
+ public Nullable IdleTcpConnectionTimeout { get; set; }
+ public bool IsDistributedTracingEnabled { get; set; }
+ public bool LimitToEndpoint { get; set; }
+ public Nullable MaxRequestsPerTcpConnection { get; set; }
+ public Nullable MaxRetryAttemptsOnRateLimitedRequests { get; set; }
+ public Nullable MaxRetryWaitTimeOnRateLimitedRequests { get; set; }
+ public Nullable MaxTcpConnectionsPerEndpoint { get; set; }
+ public Nullable OpenTcpConnectionTimeout { get; set; }
+ public Nullable PortReuseMode { get; set; }
+ public TimeSpan RequestTimeout { get; set; }
+ public CosmosSerializer Serializer { get; set; }
+ public CosmosSerializationOptions SerializerOptions { get; set; }
+ public Func ServerCertificateCustomValidationCallback { get; set; }
+ public Nullable TokenCredentialBackgroundRefreshInterval { get; set; }
+ public IWebProxy WebProxy { get; set; }
+ }
+ public abstract class CosmosDiagnostics
+ {
+ protected CosmosDiagnostics();
+ public virtual TimeSpan GetClientElapsedTime();
+ public abstract IReadOnlyList> GetContactedRegions();
+ public virtual int GetFailedRequestCount();
+ public virtual Nullable GetStartTimeUtc();
+ public abstract override string ToString();
+ }
+ public class CosmosException : Exception
+ {
+ public CosmosException(string message, HttpStatusCode statusCode, int subStatusCode, string activityId, double requestCharge);
+ public virtual string ActivityId { get; }
+ public virtual CosmosDiagnostics Diagnostics { get; }
+ public virtual Headers Headers { get; }
+ public override string Message { get; }
+ public virtual double RequestCharge { get; }
+ public virtual string ResponseBody { get; }
+ public virtual Nullable RetryAfter { get; }
+ public override string StackTrace { get; }
+ public virtual HttpStatusCode StatusCode { get; }
+ public virtual int SubStatusCode { get; }
+ public override string ToString();
+ public virtual bool TryGetHeader(string headerName, out string value);
+ }
+ public sealed class CosmosLinqSerializerOptions
+ {
+ public CosmosLinqSerializerOptions();
+ public CosmosPropertyNamingPolicy PropertyNamingPolicy { get; set; }
+ }
+ public class CosmosOperationCanceledException : OperationCanceledException
+ {
+ public CosmosOperationCanceledException(OperationCanceledException originalException, CosmosDiagnostics diagnostics);
+ protected CosmosOperationCanceledException(SerializationInfo info, StreamingContext context);
+ public override IDictionary Data { get; }
+ public CosmosDiagnostics Diagnostics { get; }
+ public override string HelpLink { get; set; }
+ public override string Message { get; }
+ public override string Source { get; set; }
+ public override string StackTrace { get; }
+ public override Exception GetBaseException();
+ public override void GetObjectData(SerializationInfo info, StreamingContext context);
+ public override string ToString();
+ }
+ public enum CosmosPropertyNamingPolicy
+ {
+ CamelCase = 1,
+ Default = 0,
+ }
+ public abstract class CosmosResponseFactory
+ {
+ protected CosmosResponseFactory();
+ public abstract FeedResponse CreateItemFeedResponse(ResponseMessage responseMessage);
+ public abstract ItemResponse CreateItemResponse(ResponseMessage responseMessage);
+ public abstract StoredProcedureExecuteResponse CreateStoredProcedureExecuteResponse(ResponseMessage responseMessage);
+ }
+ public sealed class CosmosSerializationOptions
+ {
+ public CosmosSerializationOptions();
+ public bool IgnoreNullValues { get; set; }
+ public bool Indented { get; set; }
+ public CosmosPropertyNamingPolicy PropertyNamingPolicy { get; set; }
+ }
+ public abstract class CosmosSerializer
+ {
+ protected CosmosSerializer();
+ public abstract T FromStream(Stream stream);
+ public abstract Stream ToStream(T input);
+ }
+ public abstract class Database
+ {
+ protected Database();
+ public abstract CosmosClient Client { get; }
+ public abstract string Id { get; }
+ public abstract Task CreateClientEncryptionKeyAsync(ClientEncryptionKeyProperties clientEncryptionKeyProperties, RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task CreateContainerAsync(ContainerProperties containerProperties, ThroughputProperties throughputProperties, RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task CreateContainerAsync(ContainerProperties containerProperties, Nullable throughput=default(Nullable), RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task CreateContainerAsync(string id, string partitionKeyPath, Nullable throughput=default(Nullable), RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task CreateContainerIfNotExistsAsync(ContainerProperties containerProperties, ThroughputProperties throughputProperties, RequestOptions requestOptions=null, CancellationToken cancellationToken=default(CancellationToken));
+ public abstract Task CreateContainerIfNotExistsAsync(ContainerProperties containerProperties, Nullable