diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index bf65a41..c83f174 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -2,7 +2,13 @@ * Moved Sync-over-Async versions of `TableContext` operations into `namespace FSharp.AWS.DynamoDB.Scripting` * Added `?collector` parameter to each operation on `TableContext` to enable separated collection for concurrent requests * Ensured metrics are reported even for failed requests -* Added `TableContext.CreateUnverified` (`TableContext.CreateAsync` without the optional store round-trips) +* Clarified Creation/Verification APIs: + * Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize` and/or `TableContext.InitializeTableAsync`) + * Added `TableContext` constructor (replaces `TableContext.Create(verifyTable = false)`) + * Added `TableContext.Scripting.Initialize` (replaces `TableContext.Create()`) + * Added `TableContext.VerifyTableAsync` overload that only performs verification but never creates a Table + * Added `TableContext.InitializeTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) + * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `TableContext.InitializeTableAsync`) ### 0.9.3-beta * Added `RequestMetrics` record type diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 72cf555..6f4d516 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -8,7 +8,6 @@ open Microsoft.FSharp.Quotations open Amazon.DynamoDBv2 open Amazon.DynamoDBv2.Model -open FSharp.AWS.DynamoDB.KeySchema open FSharp.AWS.DynamoDB.ExprCommon /// Exception raised by DynamoDB in case where write preconditions are not satisfied @@ -47,6 +46,16 @@ type private LimitType = All | Default | Count of int static member AllOrCount (l : int option) = l |> Option.map Count |> Option.defaultValue All static member DefaultOrCount (l : int option) = l |> Option.map Count |> Option.defaultValue Default +/// Defines Verification and/or Provisioning steps to be applied when initializing and/or validating a TableContext +[] +type InitializationMode = + /// Validate the Table exists and has a compatible schema + | VerifyOnly + /// Perform a Validation step as per VerifyOnly, but Create the Table with the specified Throughput if it was not found + | CreateIfNotExists of provisionedThroughput : ProvisionedThroughput + /// Validate and/or Create as per CreateIfNotExists, but also re-apply the specified Throughput in case it has diverged + | CreateOrUpdateThroughput of provisionedThroughput : ProvisionedThroughput + /// DynamoDB client object for performing table operations in the context of given F# record representations [] type TableContext<'TRecord> internal @@ -274,6 +283,19 @@ type TableContext<'TRecord> internal /// Record-induced table template member __.Template = template + + /// + /// Creates a DynamoDB client instance for given F# record and table name.
+ /// For creating, provisioning or verification, see InitializeTableAsync and VerifyTableAsync. + ///
+ /// DynamoDB client instance. + /// Table name to target. + /// Function to receive request metrics. + new (client : IAmazonDynamoDB, tableName : string, ?metricsCollector : RequestMetrics -> unit) = + if not <| isValidTableName tableName then invalidArg "tableName" "unsupported DynamoDB table name." + TableContext<'TRecord>(client, tableName, RecordTemplate.Define<'TRecord>(), metricsCollector) + + /// Creates a new table context instance that uses /// a new F# record type. The new F# record type /// must define a compatible key schema. @@ -893,24 +915,14 @@ type TableContext<'TRecord> internal } - /// - /// Asynchronously verify that the table exists and is compatible with record key schema. - /// - /// Create the table instance now instance if it does not exist. Defaults to false. - /// Provisioned throughput for the table if newly created. Defaults to (10,10). - member __.VerifyTableAsync(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) : Async = async { - let createIfNotExists = defaultArg createIfNotExists false + member __.InternalInitializeAsync(mode : InitializationMode) : Async = async { let (|Conflict|_|) (e : exn) = match e with | :? AmazonDynamoDBException as e when e.StatusCode = HttpStatusCode.Conflict -> Some() | :? ResourceInUseException -> Some () | _ -> None - let rec verify lastExn retries = async { - match lastExn with - | Some e when retries = 0 -> do! Async.Raise e - | _ -> () - + let rec verify retries = async { let! ct = Async.CancellationToken let! response = client.DescribeTableAsync(tableName, ct) @@ -922,7 +934,7 @@ type TableContext<'TRecord> internal if td.Table.TableStatus <> TableStatus.ACTIVE then do! Async.Sleep 2000 // wait indefinitely if table is in transition state - return! verify None retries + return! verify retries else let existingSchema = TableKeySchemata.OfTableDescription td.Table @@ -930,71 +942,105 @@ type TableContext<'TRecord> internal sprintf "table '%s' exists with key schema %A, which is incompatible with record '%O'." tableName existingSchema typeof<'TRecord> |> invalidOp - - | Choice2Of2 (:? ResourceNotFoundException) when createIfNotExists -> - let provisionedThroughput = - match provisionedThroughput with - | None -> ProvisionedThroughput(10L,10L) - | Some pt -> pt - - let ctr = template.Info.Schemata.CreateCreateTableRequest (tableName, provisionedThroughput) - let! ct = Async.CancellationToken + match mode with + | InitializationMode.VerifyOnly | InitializationMode.CreateIfNotExists _ -> () + | InitializationMode.CreateOrUpdateThroughput t -> do! __.UpdateProvisionedThroughputAsync(t) + + | Choice2Of2 (:? ResourceNotFoundException) when mode <> InitializationMode.VerifyOnly -> + let throughput = + match mode with + | InitializationMode.VerifyOnly -> failwith "Unexpected" // the when guard should preclude this + | InitializationMode.CreateIfNotExists t | InitializationMode.CreateOrUpdateThroughput t -> t + let ctr = template.Info.Schemata.CreateCreateTableRequest(tableName, throughput) let! response = client.CreateTableAsync(ctr, ct) |> Async.AwaitTaskCorrect |> Async.Catch match response with - | Choice1Of2 _ -> return! verify None retries - | Choice2Of2 (Conflict as e) -> + | Choice1Of2 _ -> return! verify retries + | Choice2Of2 Conflict when retries > 0 -> do! Async.Sleep 2000 - return! verify (Some e) (retries - 1) + return! verify (retries - 1) | Choice2Of2 e -> do! Async.Raise e - | Choice2Of2 (Conflict as e) -> + | Choice2Of2 Conflict when retries > 0 -> do! Async.Sleep 2000 - return! verify (Some e) (retries - 1) + return! verify (retries - 1) | Choice2Of2 e -> do! Async.Raise e } - do! verify None 10 + do! verify 9 // up to 9 retries, i.e. 10 attempts before we let exception propagate } -/// Table context factory methods -type TableContext = + /// + /// Asynchronously verify that the table exists and is compatible with record key schema. + /// + /// Create the table instance now instance if it does not exist. Defaults to false. + /// Provisioned throughput for the table if newly created. Defaults to (10,10). + [] + member __.VerifyTableAsync(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) : Async = + let mode = + if createIfNotExists = Some true then + let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) + InitializationMode.CreateIfNotExists throughput + else InitializationMode.VerifyOnly + __.InternalInitializeAsync(mode) /// - /// Creates a DynamoDB client instance for given F# record and table name.
- /// See CreateAsync for the ability to create and/or verify the Table. + /// Asynchronously verify that the table exists and is compatible with record key schema, or throw.
+ /// See also InitializeTableAsync, which performs the same check, but can create or re-provision the Table if required. ///
- /// DynamoDB client instance. - /// Table name to target. - /// Function to receive request metrics. - static member CreateUnverified<'TRecord>(client : IAmazonDynamoDB, tableName : string, ?metricsCollector : RequestMetrics -> unit) : TableContext<'TRecord> = - if not <| isValidTableName tableName then invalidArg "tableName" "unsupported DynamoDB table name." - TableContext<'TRecord>(client, tableName, RecordTemplate.Define<'TRecord>(), metricsCollector) + member __.VerifyTableAsync() : Async = + __.InternalInitializeAsync(InitializationMode.VerifyOnly) /// - /// Creates a DynamoDB client instance for given F# record and table name.
- /// See CreateUnverified if your deployment phase handles the creation of the Table and verification of the schema. + /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
+ /// If the table is not present, it is provisioned, with the specified throughput.
+ /// Optionally can reapply the cited throughput level if the Table has already been created.
+ /// See also VerifyTableAsync, which only verifies the Table is present and correct. + ///
+ /// Provisioned throughput to use for the table. + /// Trigger re-provisioning of the Table's throughput if it was already present. Default: false + member __.InitializeTableAsync(throughput : ProvisionedThroughput, ?updateThroughputIfExists) : Async = + let mode = + if updateThroughputIfExists = Some true then InitializationMode.CreateOrUpdateThroughput + else InitializationMode.CreateIfNotExists + __.InternalInitializeAsync(mode throughput) + +// Deprecated factory method, to be removed. Replaced with +// 1. TableContext<'T> ctor (synchronous) +// 2. InitializeTableAsync OR VerifyTableAsync (explicitly async to signify that verification/creation is a costly and/or privileged operation) +type TableContext internal () = + + /// + /// Creates a DynamoDB client instance for given F# record and table name. /// /// DynamoDB client instance. /// Table name to target. /// Verify that the table exists and is compatible with supplied record schema. Defaults to true. - /// Create the table immediately if it does not exist. Defaults to false. - /// Provisioned throughput for the table if creation required. Defaults to (10,10). + /// Create the table now if it does not exist. Defaults to false. + /// Provisioned throughput for the table if newly created. /// Function to receive request metrics. - static member CreateAsync<'TRecord>(client : IAmazonDynamoDB, tableName : string, ?verifyTable : bool, ?createIfNotExists : bool, - ?provisionedThroughput : ProvisionedThroughput, ?metricsCollector : RequestMetrics -> unit) : Async> = async { - let context = TableContext.CreateUnverified(client, tableName, ?metricsCollector = metricsCollector) - let verifyTable = defaultArg verifyTable true - let createIfNotExists = defaultArg createIfNotExists false + [] + static member Create<'TRecord> + ( client : IAmazonDynamoDB, tableName : string, ?verifyTable : bool, + ?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput, + ?metricsCollector : RequestMetrics -> unit) = + let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) + let verifyTable, createIfNotExists = verifyTable <> Some false, createIfNotExists = Some true if verifyTable || createIfNotExists then - do! context.VerifyTableAsync(createIfNotExists = createIfNotExists, ?provisionedThroughput = provisionedThroughput) - return context - } + let mode = + if createIfNotExists then + let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) + InitializationMode.CreateIfNotExists throughput + else InitializationMode.VerifyOnly + context.InternalInitializeAsync(mode) |> Async.RunSynchronously + context /// /// Sync-over-Async helpers that can be opted-into when working in scripting scenarios. @@ -1003,6 +1049,23 @@ type TableContext = /// module Scripting = + /// Factory method that allows one to include auto-initialization easily for scripting scenarios + type TableContext internal () = + + /// Creates a DynamoDB client instance for the specified F# record type, client and table name. + /// DynamoDB client instance. + /// Table name to target. + /// Allows one to define auto-creation or re-provisioning rules via . + /// Function to receive request metrics. + static member Initialize<'TRecord> + ( client : IAmazonDynamoDB, tableName : string, + ?mode : InitializationMode, + ?metricsCollector : RequestMetrics -> unit) : TableContext<'TRecord> = + let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) + let mode = defaultArg mode InitializationMode.VerifyOnly + context.InternalInitializeAsync(mode) |> Async.RunSynchronously + context + type TableContext<'TRecord> with /// @@ -1420,30 +1483,3 @@ module Scripting = /// Provisioned throughput to use on table. member __.UpdateProvisionedThroughput(provisionedThroughput : ProvisionedThroughput) = __.UpdateProvisionedThroughputAsync(provisionedThroughput) |> Async.RunSynchronously - - - /// - /// Asynchronously verify that the table exists and is compatible with record key schema. - /// - /// Create the table instance now if it does not exist. Defaults to false. - /// Provisioned throughput for the table if newly created. - member __.VerifyTable(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) = - __.VerifyTableAsync(?createIfNotExists = createIfNotExists, ?provisionedThroughput = provisionedThroughput) - |> Async.RunSynchronously - - type TableContext with - - /// - /// Creates a DynamoDB client instance for given F# record and table name. - /// - /// DynamoDB client instance. - /// Table name to target. - /// Verify that the table exists and is compatible with supplied record schema. Defaults to true. - /// Create the table now instance if it does not exist. Defaults to false. - /// Provisioned throughput for the table if newly created. - /// Function to receive request metrics. - static member Create<'TRecord>(client : IAmazonDynamoDB, tableName : string, ?verifyTable : bool, ?createIfNotExists : bool, - ?provisionedThroughput : ProvisionedThroughput, ?metricsCollector : RequestMetrics -> unit) = - TableContext.CreateAsync<'TRecord>(client, tableName, ?verifyTable = verifyTable, ?createIfNotExists = createIfNotExists, - ?provisionedThroughput = provisionedThroughput, ?metricsCollector = metricsCollector) - |> Async.RunSynchronously diff --git a/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs index 1102151..afe724f 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs @@ -85,7 +85,7 @@ type ``Conditional Expression Tests`` (fixture : TableFixture) = Serialized = rand(), guid() } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() member this.``Item exists precondition`` () = let item = mkItem() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs index 230b858..d2097ef 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs @@ -42,7 +42,7 @@ type ``Pagination Tests`` (fixture : TableFixture) = LocalAttribute = int (rand () % 2L) } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() member __.``Paginated Query on Primary Key`` () = let hk = guid() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs index b5e5470..5da8133 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs @@ -96,7 +96,7 @@ type ``Projection Expression Tests`` (fixture : TableFixture) = Serialized = rand(), guid() ; Serialized2 = { NV = guid() ; NE = enum (int (rand()) % 3) } ; } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() member this.``Should fail on invalid projections`` () = let testProj (p : Expr 'T>) = diff --git a/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs index fba315e..df52079 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs @@ -55,7 +55,7 @@ type ``Simple Table Operation Tests`` (fixture : TableFixture) = Unions = [Choice1Of3 (guid()) ; Choice2Of3(rand()) ; Choice3Of3(Guid.NewGuid().ToByteArray())] } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() member this.``Convert to compatible table`` () = let table' = table.WithRecordType () diff --git a/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs b/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs index cc7a0fa..2077be5 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs @@ -30,7 +30,7 @@ type ``Sparse GSI Tests`` (fixture : TableFixture) = SecondaryHashKey = if rand() % 2L = 0L then Some (guid()) else None ; } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() member this.``GSI Put Operation`` () = let value = mkItem() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs index 3f2557f..ef1dbdf 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs @@ -93,7 +93,7 @@ type ``Update Expression Tests`` (fixture : TableFixture) = Serialized = rand(), guid() ; Serialized2 = { NV = guid() ; NE = enum (int (rand()) % 3) } ; } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() member this.``Attempt to update HashKey`` () = let item = mkItem() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs index 06bef2d..e1fdf0f 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs @@ -6,8 +6,6 @@ open System.IO open Expecto open FsCheck -open Amazon -open Amazon.Util open Amazon.DynamoDBv2 open Amazon.Runtime @@ -17,12 +15,12 @@ open FSharp.AWS.DynamoDB module Utils = let getRandomTableName() = - sprintf "fsdynamodb-%s" <| System.Guid.NewGuid().ToString("N") + sprintf "fsdynamodb-%s" <| Guid.NewGuid().ToString("N") let guid() = Guid.NewGuid().ToString("N") let shouldFailwith<'T, 'Exn when 'Exn :> exn>(f : unit -> 'T) = - ignore <| Expect.throws (f >> ignore) typeof<'Exn>.Name + Expect.throws (f >> ignore) typeof<'Exn>.Name let getDynamoDBAccount () = let credentials = new BasicAWSCredentials("Fake", "Fake") @@ -32,7 +30,7 @@ module Utils = type FsCheckGenerators = - static member MemoryStream = + static member MemoryStream = Arb.generate |> Gen.map (function None -> null | Some bs -> new MemoryStream(bs)) |> Arb.fromGen @@ -44,6 +42,10 @@ module Utils = member __.Client = client member __.TableName = tableName + member __.CreateContextAndTableIfNotExists<'TRecord>() = + let autoCreate = InitializationMode.CreateIfNotExists (ProvisionedThroughput(10, 10)) + Scripting.TableContext.Initialize<'TRecord>(__.Client, __.TableName, mode = autoCreate) + interface IDisposable with member __.Dispose() = - client.DeleteTableAsync(tableName) |> Async.AwaitTask |> Async.RunSynchronously |> ignore \ No newline at end of file + client.DeleteTableAsync(tableName) |> Async.AwaitTask |> Async.RunSynchronously |> ignore