From f669358b36a049652eeac833c0bf11a2e5bf4e00 Mon Sep 17 00:00:00 2001 From: Evanette Burrows Date: Wed, 20 Dec 2023 15:25:35 -0500 Subject: [PATCH 1/7] Address Issue 162 improve version support (#174) * Update functions for supported OMOP versions --- R/CreateCDMIndexAndConstraintScripts.r | 181 +++++++------------------ R/backupCDM.r | 10 +- R/createPrunedTables.r | 10 +- R/getEventConceptId.r | 10 +- R/pruneCDM.r | 4 +- R/restoreCDMTables.r | 10 +- 6 files changed, 68 insertions(+), 157 deletions(-) diff --git a/R/CreateCDMIndexAndConstraintScripts.r b/R/CreateCDMIndexAndConstraintScripts.r index 737d259..a83facf 100644 --- a/R/CreateCDMIndexAndConstraintScripts.r +++ b/R/CreateCDMIndexAndConstraintScripts.r @@ -6,11 +6,11 @@ #' function \code{createConnectionDetails} in the #' \code{DatabaseConnector} package. #' @param cdmSchema The name of the CDM database schema. Requires read and write permissions to this database. On SQL -#' Server, this should specifiy both the database and the schema, +#' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. -#' @param cdmVersion Your CDM version. Currently "5.3.1" and "6.0.0" are supported. +#' @param cdmVersion Your CDM version. Currently "5.3.1" and "5.4.0" are supported. #' @param githubTag An optional github tag from which to pull the DDL script. -#' Currently "v5.3.1", "v5.3.1_fixes", "v6.0.0", and "v6.0.0_fixes" are supported. The default is NULL. +#' Currently "v5.3.1" and "v5.4.0" are supported. The default is NULL. #' #' @details This function creates SQL scripts for the indices and constraints on tables in a CDM by referring to the #' correct SQL DDL script in the OHDSI CommonDataModel repo. The database platform is @@ -29,13 +29,13 @@ CreateCDMIndexAndConstraintScripts <- { supportedDbs <- c("oracle", "postgresql", "pdw", "netezza", "sql server") - supportedVersions <- c("5.3.1", "6.0.0") + supportedVersions <- c("5.3.1", "5.4.0") supportedTags <- - c("v5.3.1", "v6.0.0", "v5.3.1_fixes", "v6.0.0_fixes") + c("v5.3.1", "v5.4.0") rdbms <- tolower(connectionDetails$dbms) if (!(cdmVersion %in% supportedVersions)) { - stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"6.0.0\"") + stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"5.4.0\"") } if (!(rdbms %in% supportedDbs)) { @@ -46,17 +46,17 @@ CreateCDMIndexAndConstraintScripts <- if (!is.null(githubTag) && !(githubTag %in% supportedTags)) { stop( - "Unrecognized github tag. Supported values are \"v5.3.1\", \"v6.0.0\", \"v5.3.1_fixes\", and \"v6.0.0_fixes\"" + "Unrecognized github tag. Supported values are \"v5.3.1\" and \"v5.4.0\"" ) } if (cdmVersion == "5.3.1" && - !(githubTag %in% c("v5.3.1", "v5.3.1_fixes"))) { + !(githubTag %in% c("v5.3.1"))) { stop("cdmVersion and githubTag mismatch.") } - if (cdmVersion == "6.0.0" && - !(githubTag %in% c("v6.0.0", "v6.0.0_fixes"))) { + if (cdmVersion == "5.4.0" && + !(githubTag %in% c("v5.4.0"))) { stop("cdmVersion and githubTag mismatch.") } @@ -74,34 +74,16 @@ CreateCDMIndexAndConstraintScripts <- httr::GET( "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/Oracle/OMOP%20CDM%20oracle%20constraints.txt" ) - } else if (githubTag == "v5.3.1_fixes") { - webResponseInd <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Oracle/OMOP%20CDM%20oracle%20v5_3_1%20primary%20keys.sql" - ) - webResponseCon <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Oracle/OMOP%20CDM%20oracle%20v5_3_1%20constraints.sql" - ) } - } else if (cdmVersion == "6.0.0") { - if (is.null(githubTag) || githubTag == "v6.0.0") { - webResponseInd <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Oracle/OMOP%20CDM%20oracle%20pk%20indexes.txt" - ) - webResponseCon <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Oracle/OMOP%20CDM%20oracle%20constraints.txt" - ) - } else if (githubTag == "v6.0.0_fixes") { + } else if (cdmVersion == "5.4.0") { + if (is.null(githubTag) || githubTag == "v5.4.0") { webResponseInd <- httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Oracle/OMOP%20CDM%20oracle%20pk%20indexes.txt" + "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/oracle/OMOPCDM_oracle_5.4_indices.sql" ) webResponseCon <- httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Oracle/OMOP%20CDM%20oracle%20constraints.txt" + "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/oracle/OMOPCDM_oracle_5.4_constraints.sql" ) } } @@ -116,34 +98,16 @@ CreateCDMIndexAndConstraintScripts <- httr::GET( "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/PostgreSQL/OMOP%20CDM%20postgresql%20constraints.txt" ) - } else if (githubTag == "v5.3.1_fixes") { - webResponseInd <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20v5_3_1%20primary%20keys.sql" - ) - webResponseCon <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20v5_3_1%20constraints.sql" - ) } - } else if (cdmVersion == "6.0.0") { - if (is.null(githubTag) || githubTag == "v6.0.0") { - webResponseInd <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/PostgreSQL/OMOP%20CDM%20postgresql%20pk%20indexes.txt" - ) - webResponseCon <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/PostgreSQL/OMOP%20CDM%20postgresql%20constraints.txt" - ) - } else if (githubTag == "v6.0.0_fixes") { + } else if (cdmVersion == "5.4.0") { + if (is.null(githubTag) || githubTag == "v5.4.0") { webResponseInd <- httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20pk%20indexes.txt" + "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/postgresql/OMOPCDM_postgresql_5.4_indices.sql" ) webResponseCon <- httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20constraints.txt" + "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/postgresql/OMOPCDM_postgresql_5.4_constraints.sql" ) } } @@ -158,34 +122,16 @@ CreateCDMIndexAndConstraintScripts <- httr::GET( "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/ParallelDataWarehouse/OMOP%20CDM%20pdw%20constraints.txt" ) - } else if (githubTag == "v5.3.1_fixes") { - webResponseInd <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20v5_3_1%20indices.sql" - ) - webResponseCon <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20v5_3_1%20constraints.sql" - ) } - } else if (cdmVersion == "6.0.0") { - if (is.null(githubTag) || githubTag == "v6.0.0") { - webResponseInd <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/ParallelDataWarehouse/OMOP%20CDM%20pdw%20pk%20indexes.txt" - ) - webResponseCon <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/ParallelDataWarehouse/OMOP%20CDM%20pdw%20constraints.txt" - ) - } else if (githubTag == "v6.0.0_fixes") { + } else if (cdmVersion == "5.4.0") { + if (is.null(githubTag) || githubTag == "v5.4.0") { webResponseInd <- httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20pk%20indexes.txt" + "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/pdw/OMOPCDM_pdw_5.4_indices.sql" ) webResponseCon <- httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20constraints.txt" + "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/pdw/OMOPCDM_pdw_5.4_constraints.sql" ) } } @@ -196,22 +142,12 @@ CreateCDMIndexAndConstraintScripts <- httr::GET( "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/Netezza/OMOP%20CDM%20netezza%20primary%20keys.txt" ) - } else if (githubTag == "v5.3.1_fixes") { - webResponseInd <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Netezza/OMOP%20CDM%20netezza%20v5_3_1%20primary%20keys.sql" - ) } - } else if (cdmVersion == "6.0.0") { - if (is.null(githubTag) || githubTag == "v6.0.0") { - webResponseInd <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Netezza/OMOP%20CDM%20netezza%20primary%20keys.txt" - ) - } else if (githubTag == "v6.0.0_fixes") { + } else if (cdmVersion == "5.4.0") { + if (is.null(githubTag) || githubTag == "v5.4.0") { webResponseInd <- httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Netezza/OMOP%20CDM%20netezza%20primary%20keys.txt" + "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/netezza/OMOPCDM_netezza_5.4_primary_keys.sql" ) } } @@ -226,34 +162,16 @@ CreateCDMIndexAndConstraintScripts <- httr::GET( "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/Sql%20Server/OMOP%20CDM%20sql%20server%20constraints.txt" ) - } else if (githubTag == "v5.3.1_fixes") { - webResponseInd <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20v5_3_1%20primary%20keys.sql" - ) - webResponseCon <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20v5_3_1%20constraints.sql" - ) } - } else if (cdmVersion == "6.0.0") { - if (is.null(githubTag) || githubTag == "v6.0.0") { - webResponseInd <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Sql%20Server/OMOP%20CDM%20sql%20server%20pk%20indexes.txt" - ) - webResponseCon <- - httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Sql%20Server/OMOP%20CDM%20sql%20server%20constraints.txt" - ) - } else if (githubTag == "v6.0.0_fixes") { + } else if (cdmVersion == "5.4.0") { + if (is.null(githubTag) || githubTag == "v5.4.0") { webResponseInd <- httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20pk%20indexes.txt" + "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/sql_server/OMOPCDM_sql_server_5.4_indices.sql" ) webResponseCon <- httr::GET( - "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20constraints.txt" + "https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/sql_server/OMOPCDM_sql_server_5.4_primary_keys.sql" ) } } @@ -267,30 +185,23 @@ CreateCDMIndexAndConstraintScripts <- indexDDL <- toupper(indexDDL) constraintDDL <- toupper(constraintDDL) - if (githubTag == "v5.3.1_fixes") { - indexDDL <- - SqlRender::render(sql = indexDDL, CDMDATABASESCHEMA = cdmSchema) - constraintDDL <- - SqlRender::render(sql = constraintDDL, CDMDATABASESCHEMA = cdmSchema) - } else { - indexDDL <- gsub(" ON ", " ON @CDM_SCHEMA.", indexDDL) - indexDDL <- - gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", indexDDL) - indexDDL <- gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", indexDDL) - indexDDL <- - SqlRender::render(indexDDL, CDM_SCHEMA = cdmSchema) - indexDDL <- - SqlRender::translate(indexDDL, targetDialect = rdbms) - - constraintDDL <- - gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", constraintDDL) - constraintDDL <- - gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", constraintDDL) - constraintDDL <- - SqlRender::render(constraintDDL, CDM_SCHEMA = cdmSchema) - constraintDDL <- - SqlRender::translate(constraintDDL, targetDialect = rdbms) - } + indexDDL <- gsub(" ON ", " ON @CDM_SCHEMA.", indexDDL) + indexDDL <- + gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", indexDDL) + indexDDL <- gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", indexDDL) + indexDDL <- + SqlRender::render(indexDDL, CDM_SCHEMA = cdmSchema) + indexDDL <- + SqlRender::translate(indexDDL, targetDialect = rdbms) + + constraintDDL <- + gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", constraintDDL) + constraintDDL <- + gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", constraintDDL) + constraintDDL <- + SqlRender::render(constraintDDL, CDM_SCHEMA = cdmSchema) + constraintDDL <- + SqlRender::translate(constraintDDL, targetDialect = rdbms) # Save the translated sql ddl to be run at a later time if (!dir.exists("output")) { diff --git a/R/backupCDM.r b/R/backupCDM.r index 3c2a857..230455c 100644 --- a/R/backupCDM.r +++ b/R/backupCDM.r @@ -7,9 +7,9 @@ #' \code{DatabaseConnector} package. #' @param cdmSchema The name of the database schema that contains the CDM. #' Requires read and write permissions to this database. On SQL -#' Server, this should specifiy both the database and the schema, +#' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. -#' @param cdmVersion The version of your CDM. Currently "5.3.1" and "6.0.1". +#' @param cdmVersion The version of your CDM. Currently "5.3.1" and "5.4.0". #' #'@export @@ -18,10 +18,10 @@ backupCDM <- function(connectionDetails, cdmSchema, cdmVersion) { if (cdmVersion == "5.3.1") sqlFilePath <- "cdm_version/v531" - else if (cdmVersion == "6.0.0") - sqlFilePath <- "cdm_version/v600" + else if (cdmVersion == "5.4.0") + sqlFilePath <- "cdm_version/v540" else - stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"6.0.0\"") + stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"5.4.0\"") sql <- SqlRender::loadRenderTranslateSql( diff --git a/R/createPrunedTables.r b/R/createPrunedTables.r index 00ae8a1..bb48af4 100644 --- a/R/createPrunedTables.r +++ b/R/createPrunedTables.r @@ -9,10 +9,10 @@ #' #' @param cdmSchema The name of the database schema that contains the CDM. #' Requires read and write permissions to this database. On SQL -#' Server, this should specifiy both the database and the schema, +#' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. #' @param eventConceptId A vector of concept_ids returned from \code{getEventConceptId}. -#' @param cdmVersion The version of your CDM. Currently "5.3.1" and "6.0.0". +#' @param cdmVersion The version of your CDM. Currently "5.3.1" and "5.4.0". #' #'@export @@ -25,10 +25,10 @@ createPrunedTables <- { if (cdmVersion == "5.3.1") sqlFilePath <- "cdm_version/v531" - else if (cdmVersion == "6.0.0") - sqlFilePath <- "cdm_version/v600" + else if (cdmVersion == "5.4.0") + sqlFilePath <- "cdm_version/v540" else - stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"6.0.0\"") + stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"5.4.0\"") sql <- SqlRender::loadRenderTranslateSql( sqlFileName = paste0(sqlFilePath, "/create_pruned_tables.sql"), diff --git a/R/getEventConceptId.r b/R/getEventConceptId.r index 22a7859..279b587 100644 --- a/R/getEventConceptId.r +++ b/R/getEventConceptId.r @@ -8,9 +8,9 @@ #' #' @param cdmSchema The name of the database schema that contains the CDM. #' Requires read and write permissions to this database. On SQL -#' Server, this should specifiy both the database and the schema, +#' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. -#' @param cdmVersion The version of your CDM. Currently "5.3.1" and "6.0.0". +#' @param cdmVersion The version of your CDM. Currently "5.3.1" and "5.4.0". #' #'@export @@ -19,10 +19,10 @@ getEventConceptId <- { if (cdmVersion == "5.3.1") sqlFilePath <- "cdm_version/v531" - else if (cdmVersion == "6.0.0") - sqlFilePath <- "cdm_version/v600" + else if (cdmVersion == "5.4.0") + sqlFilePath <- "cdm_version/v540" else - stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"6.0.0\"") + stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"5.4.0\"") sql <- SqlRender::loadRenderTranslateSql( sqlFileName = paste0(sqlFilePath, "/get_event_concept_id.sql"), diff --git a/R/pruneCDM.r b/R/pruneCDM.r index 31e3937..649febd 100644 --- a/R/pruneCDM.r +++ b/R/pruneCDM.r @@ -7,9 +7,9 @@ #' \code{DatabaseConnector} package. #' @param cdmSchema The name of the database schema that contains the CDM. #' Requires read and write permissions to this database. On SQL -#' Server, this should specifiy both the database and the schema, +#' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. -#' @param cdmVersion The version of your CDM. Currently "5.3.1" and "6.0.0". +#' @param cdmVersion The version of your CDM. Currently "5.3.1" and "5.4.0". #' #'@export diff --git a/R/restoreCDMTables.r b/R/restoreCDMTables.r index 0c63535..6816d71 100644 --- a/R/restoreCDMTables.r +++ b/R/restoreCDMTables.r @@ -7,9 +7,9 @@ #' \code{DatabaseConnector} package. #' @param cdmSchema The name of the database schema that contains the CDM #' instance. Requires read and write permissions to this database. On SQL -#' Server, this should specifiy both the database and the schema, +#' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. -#' @param cdmVersion The version of your CDM. Currently "5.3.1" and "6.0.0" are supported. +#' @param cdmVersion The version of your CDM. Currently "5.3.1" and "5.4.0" are supported. #'@export @@ -18,10 +18,10 @@ restoreCDMTables <- { if (cdmVersion == "5.3.1") sqlFilePath <- "cdm_version/v531" - else if (cdmVersion == "6.0.0") - sqlFilePath <- "cdm_version/v600" + else if (cdmVersion == "5.4.0") + sqlFilePath <- "cdm_version/v540" else - stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"6.0.0\"") + stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"5.4.0\"") sql <- SqlRender::loadRenderTranslateSql( sqlFileName = paste0(sqlFilePath, "/restore_cdm_tables.sql"), From 768076c90248113e23cbee0b9f57bccec700f9ad Mon Sep 17 00:00:00 2001 From: Evanette Burrows Date: Fri, 22 Dec 2023 13:34:03 -0500 Subject: [PATCH 2/7] Update formatting for lintr issues (#175) --- R/CreateCDMIndexAndConstraintScripts.r | 4 ++-- R/CreateCDMIndices.r | 32 +++++++++++++------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/R/CreateCDMIndexAndConstraintScripts.r b/R/CreateCDMIndexAndConstraintScripts.r index a83facf..00496bc 100644 --- a/R/CreateCDMIndexAndConstraintScripts.r +++ b/R/CreateCDMIndexAndConstraintScripts.r @@ -51,12 +51,12 @@ CreateCDMIndexAndConstraintScripts <- } if (cdmVersion == "5.3.1" && - !(githubTag %in% c("v5.3.1"))) { + !(githubTag %in% c("v5.3.1"))) { stop("cdmVersion and githubTag mismatch.") } if (cdmVersion == "5.4.0" && - !(githubTag %in% c("v5.4.0"))) { + !(githubTag %in% c("v5.4.0"))) { stop("cdmVersion and githubTag mismatch.") } diff --git a/R/CreateCDMIndices.r b/R/CreateCDMIndices.r index 53eda5c..7b53f2a 100644 --- a/R/CreateCDMIndices.r +++ b/R/CreateCDMIndices.r @@ -6,7 +6,7 @@ #' function \code{createConnectionDetails} in the #' \code{DatabaseConnector} package. #' @param cdmSchema The name of the CDM database schema. Requires read and write permissions to this database. On SQL -#' Server, this should specifiy both the database and the schema, +#' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. #' @param cdmVersion Your CDM version. Currently "5.3" and "5.4" are supported. #' @param outputFolder Location of the SQL scripts if sqlOnly = TRUE. Default is NULL. @@ -25,23 +25,23 @@ CreateCDMIndices <- cdmVersion, outputFolder = NULL, sqlOnly = FALSE) -{ + { if (!sqlOnly) { - - print("Creating Indices on CDM Tables....") + print("Creating Indices on CDM Tables....") - indexSQLFile <- CommonDataModel::writeIndex( - targetDialect = connectionDetails$dbms, - cdmVersion = cdmVersion, - cdmDatabaseSchema = cdmSchema, - outputfolder = tempdir()) + indexSQLFile <- CommonDataModel::writeIndex( + targetDialect = connectionDetails$dbms, + cdmVersion = cdmVersion, + cdmDatabaseSchema = cdmSchema, + outputfolder = tempdir() + ) + + indexDDL <- SqlRender::readSql(paste0(tempdir(), "/", indexSQLFile)) + conn <- DatabaseConnector::connect(connectionDetails) + DatabaseConnector::executeSql(conn, indexDDL) + DatabaseConnector::disconnect(conn) + print("Index Creation Complete.") - indexDDL <- SqlRender::readSql(paste0(tempdir(),"/",indexSQLFile)) - conn <- DatabaseConnector::connect(connectionDetails) - DatabaseConnector::executeSql(conn,indexDDL) - DatabaseConnector::disconnect(conn) - print("Index Creation Complete.") - } else { if (is.null(outputFolder)) { stop("Must specify an outputFolder location when using sqlOnly = TRUE") @@ -56,4 +56,4 @@ CreateCDMIndices <- cdmDatabaseSchema = cdmSchema ) } -} + } From dfb403d56fdbfb75f00392738ac84222b9b8b8d1 Mon Sep 17 00:00:00 2001 From: Evanette Burrows Date: Fri, 29 Dec 2023 11:29:06 -0500 Subject: [PATCH 3/7] Update package to split intermediate steps and event table loading (#176) --- DESCRIPTION | 2 +- NAMESPACE | 3 +- R/CreateMapAndRollupTables.r | 71 +++++++ R/DropMapAndRollupTables.R | 2 +- R/LoadEventTables.r | 24 +-- R/createExtraIndices.R | 2 +- README.md | 18 +- docs/index.html | 5 + docs/pkgdown.yml | 2 +- .../CreateCDMIndexAndConstraintScripts.html | 6 +- docs/reference/CreateCDMIndices.html | 2 +- docs/reference/CreateMapAndRollupTables.html | 197 ++++++++++++++++++ docs/reference/DropMapAndRollupTables.html | 2 +- docs/reference/LoadEventTables.html | 8 +- docs/reference/backupCDM.html | 4 +- docs/reference/createExtraIndices.html | 6 +- docs/reference/createPrunedTables.html | 4 +- docs/reference/getEventConceptId.html | 4 +- docs/reference/index.html | 12 +- docs/reference/pruneCDM.html | 4 +- docs/reference/restoreCDMTables.html | 4 +- docs/sitemap.xml | 3 + man/CreateCDMIndexAndConstraintScripts.Rd | 6 +- man/CreateCDMIndices.Rd | 2 +- man/CreateMapAndRollupTables.Rd | 60 ++++++ man/DropMapAndRollupTables.Rd | 2 +- man/LoadEventTables.Rd | 8 +- man/backupCDM.Rd | 4 +- man/createExtraIndices.Rd | 8 +- man/createPrunedTables.Rd | 4 +- man/getEventConceptId.Rd | 4 +- man/pruneCDM.Rd | 4 +- man/restoreCDMTables.Rd | 4 +- 33 files changed, 413 insertions(+), 78 deletions(-) create mode 100644 R/CreateMapAndRollupTables.r create mode 100644 docs/reference/CreateMapAndRollupTables.html create mode 100644 man/CreateMapAndRollupTables.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 0d4db8e..376ed67 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -27,7 +27,7 @@ Remotes: OHDSI/CommonDataModel Encoding: UTF-8 LazyData: true -RoxygenNote: 7.2.2 +RoxygenNote: 7.2.3 Suggests: rmarkdown, knitr, diff --git a/NAMESPACE b/NAMESPACE index 60eaaf8..92bb720 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,6 +3,8 @@ export(CreateCDMIndexAndConstraintScripts) export(CreateCDMIndices) export(CreateCDMTables) +export(CreateExtraIndices) +export(CreateMapAndRollupTables) export(CreateSyntheaTables) export(CreateVisitRollupTables) export(CreateVocabMapTables) @@ -17,7 +19,6 @@ export(LoadVocabFromSchema) export(TruncateEventTables) export(TruncateVocabTables) export(backupCDM) -export(createExtraIndices) export(createPrunedTables) export(exportToSQLite) export(getEventConceptId) diff --git a/R/CreateMapAndRollupTables.r b/R/CreateMapAndRollupTables.r new file mode 100644 index 0000000..77378c7 --- /dev/null +++ b/R/CreateMapAndRollupTables.r @@ -0,0 +1,71 @@ +#' @title Create Vocab Mapping and Visit Rollup Tables. +#' +#' @description This function creates the vocabulary mapping and visit roll-up intermediate tables from created +#' by the \cr\code{CreateVocabMapTables()} and \cr\code{CreateVisitRollupTables()} scripts. +#' +#' @details This function assumes \cr\code{createCDMTables()}, \cr\code{createSyntheaTables()}, \cr\code{LoadSyntheaTables()}, +#' and \cr\code{LoadVocabTables()} have all been run. +#' +#' @param connectionDetails An R object of type\cr\code{connectionDetails} created using the +#' function \code{createConnectionDetails} in the +#' \code{DatabaseConnector} package. +#' @param cdmSchema The name of the database schema that will contain the CDM. +#' Requires read and write permissions to this database. On SQL +#' Server, this should specify both the database and the schema, +#' so for example 'cdm_instance.dbo'. +#' @param syntheaSchema The name of the database schema that contain the Synthea +#' instance. Requires read and write permissions to this database. On SQL +#' Server, this should specify both the database and the schema, +#' so for example 'cdm_instance.dbo'. +#' @param cdmVersion The version of your CDM. Currently "5.3" and "5.4". +#' @param syntheaVersion The version of Synthea used to generate the csv files. +#' Currently "2.7.0" and "3.0.0" are supported. +#' @param cdmSourceName The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database. +#' @param cdmSourceAbbreviation The source abbreviation to insert into the CDM_SOURCE table. Default is Synthea. +#' @param cdmHolder The holder to insert into the CDM_SOURCE table. Default is OHDSI +#' @param cdmSourceDescription The description of the source data. Default is generic Synthea description. +#' @param createIndices A boolean that determines whether or not to create indices on CDM tables before the ETL. +#' @param sqlOnly A boolean that determines whether or not to perform the load or generate SQL scripts. Default is FALSE. +#' +#'@export + + +CreateMapAndRollupTables <- function(connectionDetails, + cdmSchema, + syntheaSchema, + cdmVersion, + syntheaVersion = "2.7.0", + cdmSourceName = "Synthea synthetic health database", + cdmSourceAbbreviation = "Synthea", + cdmHolder = "OHDSI", + cdmSourceDescription = "SyntheaTM is a Synthetic Patient Population Simulator. The goal is to output synthetic, realistic (but not real), patient data and associated health records in a variety of formats.", + sqlOnly = FALSE) +{ + # Determine which sql scripts to run based on the given version. + # The path is relative to inst/sql/sql_server. + if (cdmVersion == "5.3") { + sqlFilePath <- "cdm_version/v531" + } else if (cdmVersion == "5.4") { + sqlFilePath <- "cdm_version/v540" + } else { + stop("Unsupported CDM specified. Supported CDM versions are \"5.3\" and \"5.4\".") + } + + supportedSyntheaVersions <- c("2.7.0", "3.0.0") + + if (!(syntheaVersion %in% supportedSyntheaVersions)) + stop("Invalid Synthea version specified. Currently \"2.7.0\" and \"3.0.0\" are supported.") + + # Create Vocabulary mapping tables + CreateVocabMapTables(connectionDetails, cdmSchema, cdmVersion, sqlOnly) + + # Perform visit rollup logic and create auxiliary tables + CreateVisitRollupTables(connectionDetails, + cdmSchema, + syntheaSchema, + cdmVersion, + sqlOnly) + +} + + diff --git a/R/DropMapAndRollupTables.R b/R/DropMapAndRollupTables.R index c49d65a..8e891a7 100644 --- a/R/DropMapAndRollupTables.R +++ b/R/DropMapAndRollupTables.R @@ -9,7 +9,7 @@ #' \code{DatabaseConnector} package. #' @param cdmSchema The name of the database schema that will contain the Vocab mapping #' tables. Requires read and write permissions to this database. On SQL -#' Server, this should specifiy both the database and the schema, +#' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. #' #'@export diff --git a/R/LoadEventTables.r b/R/LoadEventTables.r index 0dd71e5..a381fa9 100644 --- a/R/LoadEventTables.r +++ b/R/LoadEventTables.r @@ -3,23 +3,23 @@ #' @description This function loads the CDM Event tables with Synthea data. #' #' @details This function assumes \cr\code{createCDMTables()}, \cr\code{createSyntheaTables()}, \cr\code{LoadSyntheaTables()}, -#' and \cr\code{LoadVocabTables()} have all been run. +#' \cr\code{LoadVocabTables()}, and \cr\code{CreateMapAndRollupTables()} have all been run. #' #' @param connectionDetails An R object of type\cr\code{connectionDetails} created using the #' function \code{createConnectionDetails} in the #' \code{DatabaseConnector} package. #' @param cdmSchema The name of the database schema that will contain the CDM. #' Requires read and write permissions to this database. On SQL -#' Server, this should specifiy both the database and the schema, +#' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. #' @param syntheaSchema The name of the database schema that contain the Synthea #' instance. Requires read and write permissions to this database. On SQL -#' Server, this should specifiy both the database and the schema, +#' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. #' @param cdmVersion The version of your CDM. Currently "5.3" and "5.4". #' @param syntheaVersion The version of Synthea used to generate the csv files. #' Currently "2.7.0" and "3.0.0" are supported. -#' @param cdmSourceName The source name to insert into the CDM_SOURCE table. Default is Synthea synthentic health database. +#' @param cdmSourceName The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database. #' @param cdmSourceAbbreviation The source abbreviation to insert into the CDM_SOURCE table. Default is Synthea. #' @param cdmHolder The holder to insert into the CDM_SOURCE table. Default is OHDSI #' @param cdmSourceDescription The description of the source data. Default is generic Synthea description. @@ -71,16 +71,6 @@ LoadEventTables <- function(connectionDetails, DatabaseConnector::disconnect(conn) print("Index Creation Complete.") } - - # Create Vocabulary mapping tables - CreateVocabMapTables(connectionDetails, cdmSchema, cdmVersion, sqlOnly) - - # Perform visit rollup logic and create auxiliary tables - CreateVisitRollupTables(connectionDetails, - cdmSchema, - syntheaSchema, - cdmVersion, - sqlOnly) if (!sqlOnly) { conn <- DatabaseConnector::connect(connectionDetails) @@ -282,11 +272,11 @@ LoadEventTables <- function(connectionDetails, runStep(sql, fileQuery) # cost - if (syntheaVersion == "2.7.0") + if (syntheaVersion == "2.7.0") fileQuery <- "insert_cost_v270.sql" else if (syntheaVersion == "3.0.0") fileQuery <- "insert_cost_v300.sql" - + sql <- SqlRender::loadRenderTranslateSql( sqlFilename = file.path(sqlFilePath, fileQuery), packageName = "ETLSyntheaBuilder", @@ -295,7 +285,7 @@ LoadEventTables <- function(connectionDetails, synthea_schema = syntheaSchema ) runStep(sql, fileQuery) - + if (!sqlOnly) { DatabaseConnector::disconnect(conn) } diff --git a/R/createExtraIndices.R b/R/createExtraIndices.R index 813c5b8..b4899a9 100644 --- a/R/createExtraIndices.R +++ b/R/createExtraIndices.R @@ -21,7 +21,7 @@ #' @export -createExtraIndices <- function(connectionDetails, +CreateExtraIndices <- function(connectionDetails, cdmSchema, syntheaSchema, syntheaVersion, diff --git a/README.md b/README.md index eab7457..df16582 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ # Utility to Load Synthea CSV data to OMOP CDM -## Currently supports CDM v5.3 and v5.4 + +## Currently supports CDM v5.3 and v5.4 Follow the steps on the [synthea wiki](https://github.com/synthetichealth/synthea/wiki) to run the program and generate the files. This builder works off of the csv files, not the fhir files. To do this the `exporter.csv.export` option in the `./src/main/resources/synthea.properties` file needs to be set to TRUE. ### Step by Step Example -```r +``` r devtools::install_github("OHDSI/ETL-Synthea") library(ETLSyntheaBuilder) @@ -47,16 +48,19 @@ ETLSyntheaBuilder::CreateSyntheaTables(connectionDetails = cd, syntheaSchema = s ETLSyntheaBuilder::LoadSyntheaTables(connectionDetails = cd, syntheaSchema = syntheaSchema, syntheaFileLoc = syntheaFileLoc) ETLSyntheaBuilder::LoadVocabFromCsv(connectionDetails = cd, cdmSchema = cdmSchema, vocabFileLoc = vocabFileLoc) + +ETLSyntheaBuilder::CreateMapAndRollupTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion) + +## Optional Step to create extra indices +ETLSyntheaBuilder::CreateExtraIndices(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion) ETLSyntheaBuilder::LoadEventTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion) - ``` ### Simulating Data with Synthea + For commented code used to convert the Synthea data see extras/codeToRun.R -For more information on Synthea visit: -https://synthetichealth.github.io/synthea/ +For more information on Synthea visit: -Get Synthea from GitHub: -https://github.com/synthetichealth/synthea +Get Synthea from GitHub: diff --git a/docs/index.html b/docs/index.html index 2ec50c1..124030e 100644 --- a/docs/index.html +++ b/docs/index.html @@ -164,6 +164,11 @@

Step by Step ExampleETLSyntheaBuilder::LoadSyntheaTables(connectionDetails = cd, syntheaSchema = syntheaSchema, syntheaFileLoc = syntheaFileLoc) ETLSyntheaBuilder::LoadVocabFromCsv(connectionDetails = cd, cdmSchema = cdmSchema, vocabFileLoc = vocabFileLoc) + +ETLSyntheaBuilder::CreateMapAndRollupTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion) + +## Optional Step to create extra indices +ETLSyntheaBuilder::CreateExtraIndices(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion) ETLSyntheaBuilder::LoadEventTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion) diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml index 5ea8621..6c848e0 100644 --- a/docs/pkgdown.yml +++ b/docs/pkgdown.yml @@ -16,5 +16,5 @@ articles: provider: provider.html visit_detail: visit_detail.html visit_occurrence: visit_occurrence.html -last_built: 2023-12-15T20:12Z +last_built: 2023-12-28T20:36Z diff --git a/docs/reference/CreateCDMIndexAndConstraintScripts.html b/docs/reference/CreateCDMIndexAndConstraintScripts.html index 8604585..ead83ce 100644 --- a/docs/reference/CreateCDMIndexAndConstraintScripts.html +++ b/docs/reference/CreateCDMIndexAndConstraintScripts.html @@ -111,17 +111,17 @@

Arguments

cdmSchema

The name of the CDM database schema. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.

cdmVersion
-

Your CDM version. Currently "5.3.1" and "6.0.0" are supported.

+

Your CDM version. Currently "5.3.1" and "5.4.0" are supported.

githubTag

An optional github tag from which to pull the DDL script. -Currently "v5.3.1", "v5.3.1_fixes", "v6.0.0", and "v6.0.0_fixes" are supported. The default is NULL.

+Currently "v5.3.1" and "v5.4.0" are supported. The default is NULL.

diff --git a/docs/reference/CreateCDMIndices.html b/docs/reference/CreateCDMIndices.html index 4548c9e..e04d8a6 100644 --- a/docs/reference/CreateCDMIndices.html +++ b/docs/reference/CreateCDMIndices.html @@ -112,7 +112,7 @@

Arguments

cdmSchema

The name of the CDM database schema. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.

diff --git a/docs/reference/CreateMapAndRollupTables.html b/docs/reference/CreateMapAndRollupTables.html new file mode 100644 index 0000000..9e9612b --- /dev/null +++ b/docs/reference/CreateMapAndRollupTables.html @@ -0,0 +1,197 @@ + +Create Vocab Mapping and Visit Rollup Tables. — CreateMapAndRollupTables • ETLSyntheaBuilder + + +
+
+ + + +
+
+
+

Create Vocab Mapping and Visit Rollup Tables.

+ + +
+ +
+

This function creates the vocabulary mapping and visit roll-up intermediate tables from created + by the
CreateVocabMapTables() and
CreateVisitRollupTables() scripts.

+
+ +
+ 
CreateMapAndRollupTables(
+  connectionDetails,
+  cdmSchema,
+  syntheaSchema,
+  cdmVersion,
+  syntheaVersion = "2.7.0",
+  cdmSourceName = "Synthea synthetic health database",
+  cdmSourceAbbreviation = "Synthea",
+  cdmHolder = "OHDSI",
+  cdmSourceDescription =
+    "SyntheaTM is a Synthetic Patient Population Simulator. The goal is to output synthetic, realistic (but not real), patient data and associated health records in a variety of formats.",
+  sqlOnly = FALSE
+)
+
+ +
+

Arguments

+
connectionDetails
+

An R object of type
connectionDetails created using the +function createConnectionDetails in the +DatabaseConnector package.

+ + +
cdmSchema
+

The name of the database schema that will contain the CDM. +Requires read and write permissions to this database. On SQL +Server, this should specify both the database and the schema, +so for example 'cdm_instance.dbo'.

+ + +
syntheaSchema
+

The name of the database schema that contain the Synthea +instance. Requires read and write permissions to this database. On SQL +Server, this should specify both the database and the schema, +so for example 'cdm_instance.dbo'.

+ + +
cdmVersion
+

The version of your CDM. Currently "5.3" and "5.4".

+ + +
syntheaVersion
+

The version of Synthea used to generate the csv files. +Currently "2.7.0" and "3.0.0" are supported.

+ + +
cdmSourceName
+

The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database.

+ + +
cdmSourceAbbreviation
+

The source abbreviation to insert into the CDM_SOURCE table. Default is Synthea.

+ + +
cdmHolder
+

The holder to insert into the CDM_SOURCE table. Default is OHDSI

+ + +
cdmSourceDescription
+

The description of the source data. Default is generic Synthea description.

+ + +
sqlOnly
+

A boolean that determines whether or not to perform the load or generate SQL scripts. Default is FALSE.

+ + +
createIndices
+

A boolean that determines whether or not to create indices on CDM tables before the ETL.

+ +
+
+

Details

+

This function assumes
createCDMTables(),
createSyntheaTables(),
LoadSyntheaTables(), + and
LoadVocabTables() have all been run.

+
+ +
+ +
+ + +
+

Developed by Anthony Molinaro.

+
+ +
+

Site built with pkgdown 2.0.7.

+
+ +
+ + + + + + + + diff --git a/docs/reference/DropMapAndRollupTables.html b/docs/reference/DropMapAndRollupTables.html index badbeaf..f57a632 100644 --- a/docs/reference/DropMapAndRollupTables.html +++ b/docs/reference/DropMapAndRollupTables.html @@ -107,7 +107,7 @@

Arguments

cdmSchema

The name of the database schema that will contain the Vocab mapping tables. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.

diff --git a/docs/reference/LoadEventTables.html b/docs/reference/LoadEventTables.html index 313f287..947dde0 100644 --- a/docs/reference/LoadEventTables.html +++ b/docs/reference/LoadEventTables.html @@ -120,14 +120,14 @@

Arguments

cdmSchema

The name of the database schema that will contain the CDM. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.

syntheaSchema

The name of the database schema that contain the Synthea instance. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.

@@ -141,7 +141,7 @@

Arguments

cdmSourceName
-

The source name to insert into the CDM_SOURCE table. Default is Synthea synthentic health database.

+

The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database.

cdmSourceAbbreviation
@@ -167,7 +167,7 @@

Arguments

Details

This function assumes
createCDMTables(),
createSyntheaTables(),
LoadSyntheaTables(), - and
LoadVocabTables() have all been run.

+
LoadVocabTables(), and
CreateMapAndRollupTables() have all been run.

diff --git a/docs/reference/backupCDM.html b/docs/reference/backupCDM.html index 9e03c9b..da97565 100644 --- a/docs/reference/backupCDM.html +++ b/docs/reference/backupCDM.html @@ -107,12 +107,12 @@

Arguments

cdmSchema

The name of the database schema that contains the CDM. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.

cdmVersion
-

The version of your CDM. Currently "5.3.1" and "6.0.1".

+

The version of your CDM. Currently "5.3.1" and "5.4.0".

diff --git a/docs/reference/createExtraIndices.html b/docs/reference/createExtraIndices.html index e6f96ea..1f682cd 100644 --- a/docs/reference/createExtraIndices.html +++ b/docs/reference/createExtraIndices.html @@ -1,5 +1,5 @@ -Create Optional Extra Indices for ETL Performance — createExtraIndices • ETLSyntheaBuilderCreate Optional Extra Indices for ETL Performance — CreateExtraIndices • ETLSyntheaBuilder @@ -85,7 +85,7 @@

Create Optional Extra Indices for ETL Performance

- +
@@ -93,7 +93,7 @@

Create Optional Extra Indices for ETL Performance

- 
createExtraIndices(
+    
CreateExtraIndices(
   connectionDetails,
   cdmSchema,
   syntheaSchema,
diff --git a/docs/reference/createPrunedTables.html b/docs/reference/createPrunedTables.html
index 8c19088..3c0723d 100644
--- a/docs/reference/createPrunedTables.html
+++ b/docs/reference/createPrunedTables.html
@@ -109,7 +109,7 @@ 
Arguments

 
cdmSchema

 
The name of the database schema that contains the CDM.
 Requires read and write permissions to this database. On SQL
-Server, this should specifiy both the database and the schema,
+Server, this should specify both the database and the schema,
 so for example 'cdm_instance.dbo'.

 
 
@@ -118,7 +118,7 @@ 
Arguments

 
 
 
cdmVersion

-
The version of your CDM.  Currently "5.3.1" and "6.0.0".

+
The version of your CDM.  Currently "5.3.1" and "5.4.0".

 
 

 
diff --git a/docs/reference/getEventConceptId.html b/docs/reference/getEventConceptId.html
index 3d9bf54..f54688c 100644
--- a/docs/reference/getEventConceptId.html
+++ b/docs/reference/getEventConceptId.html
@@ -107,12 +107,12 @@ 
Arguments

 
cdmSchema

 
The name of the database schema that contains the CDM.
 Requires read and write permissions to this database. On SQL
-Server, this should specifiy both the database and the schema,
+Server, this should specify both the database and the schema,
 so for example 'cdm_instance.dbo'.

 
 
 
cdmVersion

-
The version of your CDM.  Currently "5.3.1" and "6.0.0".

+
The version of your CDM.  Currently "5.3.1" and "5.4.0".
diff --git a/docs/reference/index.html b/docs/reference/index.html index f71c907..ec838be 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -102,6 +102,14 @@

All functions CreateCDMTables()

Create Common Data Model Tables.

+ +

CreateExtraIndices()

+ +

Create Optional Extra Indices for ETL Performance

+ +

CreateMapAndRollupTables()

+ +

Create Vocab Mapping and Visit Rollup Tables.

CreateSyntheaTables()

@@ -158,10 +166,6 @@

All functions backupCDM()

Backup Vocabulary and Event tables before pruning.

- -

createExtraIndices()

- -

Create Optional Extra Indices for ETL Performance

createPrunedTables()

diff --git a/docs/reference/pruneCDM.html b/docs/reference/pruneCDM.html index 1669d77..79f7ba1 100644 --- a/docs/reference/pruneCDM.html +++ b/docs/reference/pruneCDM.html @@ -107,12 +107,12 @@

Arguments

cdmSchema

The name of the database schema that contains the CDM. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.

cdmVersion
-

The version of your CDM. Currently "5.3.1" and "6.0.0".

+

The version of your CDM. Currently "5.3.1" and "5.4.0".

diff --git a/docs/reference/restoreCDMTables.html b/docs/reference/restoreCDMTables.html index 70f7de7..f09bc33 100644 --- a/docs/reference/restoreCDMTables.html +++ b/docs/reference/restoreCDMTables.html @@ -107,12 +107,12 @@

Arguments

cdmSchema

The name of the database schema that contains the CDM instance. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.

cdmVersion
-

The version of your CDM. Currently "5.3.1" and "6.0.0" are supported.

+

The version of your CDM. Currently "5.3.1" and "5.4.0" are supported.

diff --git a/docs/sitemap.xml b/docs/sitemap.xml index a5ce9dc..3d59134 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -90,6 +90,9 @@ /reference/CreateCDMTables.html + + /reference/CreateMapAndRollupTables.html + /reference/CreateSyntheaTables.html diff --git a/man/CreateCDMIndexAndConstraintScripts.Rd b/man/CreateCDMIndexAndConstraintScripts.Rd index c681f42..24304c5 100644 --- a/man/CreateCDMIndexAndConstraintScripts.Rd +++ b/man/CreateCDMIndexAndConstraintScripts.Rd @@ -17,13 +17,13 @@ function \code{createConnectionDetails} in the \code{DatabaseConnector} package.} \item{cdmSchema}{The name of the CDM database schema. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} -\item{cdmVersion}{Your CDM version. Currently "5.3.1" and "6.0.0" are supported.} +\item{cdmVersion}{Your CDM version. Currently "5.3.1" and "5.4.0" are supported.} \item{githubTag}{An optional github tag from which to pull the DDL script. -Currently "v5.3.1", "v5.3.1_fixes", "v6.0.0", and "v6.0.0_fixes" are supported. The default is NULL.} +Currently "v5.3.1" and "v5.4.0" are supported. The default is NULL.} } \description{ This function creates one or more SQL scripts as defined in https://github.com/OHDSI/CommonDataModel. diff --git a/man/CreateCDMIndices.Rd b/man/CreateCDMIndices.Rd index 4f4b690..919d5cd 100644 --- a/man/CreateCDMIndices.Rd +++ b/man/CreateCDMIndices.Rd @@ -18,7 +18,7 @@ function \code{createConnectionDetails} in the \code{DatabaseConnector} package.} \item{cdmSchema}{The name of the CDM database schema. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} \item{cdmVersion}{Your CDM version. Currently "5.3" and "5.4" are supported.} diff --git a/man/CreateMapAndRollupTables.Rd b/man/CreateMapAndRollupTables.Rd new file mode 100644 index 0000000..aa0f651 --- /dev/null +++ b/man/CreateMapAndRollupTables.Rd @@ -0,0 +1,60 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/CreateMapAndRollupTables.r +\name{CreateMapAndRollupTables} +\alias{CreateMapAndRollupTables} +\title{Create Vocab Mapping and Visit Rollup Tables.} +\usage{ +CreateMapAndRollupTables( + connectionDetails, + cdmSchema, + syntheaSchema, + cdmVersion, + syntheaVersion = "2.7.0", + cdmSourceName = "Synthea synthetic health database", + cdmSourceAbbreviation = "Synthea", + cdmHolder = "OHDSI", + cdmSourceDescription = + "SyntheaTM is a Synthetic Patient Population Simulator. The goal is to output synthetic, realistic (but not real), patient data and associated health records in a variety of formats.", + sqlOnly = FALSE +) +} +\arguments{ +\item{connectionDetails}{An R object of type\cr\code{connectionDetails} created using the +function \code{createConnectionDetails} in the +\code{DatabaseConnector} package.} + +\item{cdmSchema}{The name of the database schema that will contain the CDM. +Requires read and write permissions to this database. On SQL +Server, this should specify both the database and the schema, +so for example 'cdm_instance.dbo'.} + +\item{syntheaSchema}{The name of the database schema that contain the Synthea +instance. Requires read and write permissions to this database. On SQL +Server, this should specify both the database and the schema, +so for example 'cdm_instance.dbo'.} + +\item{cdmVersion}{The version of your CDM. Currently "5.3" and "5.4".} + +\item{syntheaVersion}{The version of Synthea used to generate the csv files. +Currently "2.7.0" and "3.0.0" are supported.} + +\item{cdmSourceName}{The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database.} + +\item{cdmSourceAbbreviation}{The source abbreviation to insert into the CDM_SOURCE table. Default is Synthea.} + +\item{cdmHolder}{The holder to insert into the CDM_SOURCE table. Default is OHDSI} + +\item{cdmSourceDescription}{The description of the source data. Default is generic Synthea description.} + +\item{sqlOnly}{A boolean that determines whether or not to perform the load or generate SQL scripts. Default is FALSE.} + +\item{createIndices}{A boolean that determines whether or not to create indices on CDM tables before the ETL.} +} +\description{ +This function creates the vocabulary mapping and visit roll-up intermediate tables from created + by the \cr\code{CreateVocabMapTables()} and \cr\code{CreateVisitRollupTables()} scripts. +} +\details{ +This function assumes \cr\code{createCDMTables()}, \cr\code{createSyntheaTables()}, \cr\code{LoadSyntheaTables()}, + and \cr\code{LoadVocabTables()} have all been run. +} diff --git a/man/DropMapAndRollupTables.Rd b/man/DropMapAndRollupTables.Rd index a680dcd..ee3c60e 100644 --- a/man/DropMapAndRollupTables.Rd +++ b/man/DropMapAndRollupTables.Rd @@ -13,7 +13,7 @@ function \code{createConnectionDetails} in the \item{cdmSchema}{The name of the database schema that will contain the Vocab mapping tables. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} } \description{ diff --git a/man/LoadEventTables.Rd b/man/LoadEventTables.Rd index aed007e..cb1800f 100644 --- a/man/LoadEventTables.Rd +++ b/man/LoadEventTables.Rd @@ -26,12 +26,12 @@ function \code{createConnectionDetails} in the \item{cdmSchema}{The name of the database schema that will contain the CDM. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} \item{syntheaSchema}{The name of the database schema that contain the Synthea instance. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} \item{cdmVersion}{The version of your CDM. Currently "5.3" and "5.4".} @@ -39,7 +39,7 @@ so for example 'cdm_instance.dbo'.} \item{syntheaVersion}{The version of Synthea used to generate the csv files. Currently "2.7.0" and "3.0.0" are supported.} -\item{cdmSourceName}{The source name to insert into the CDM_SOURCE table. Default is Synthea synthentic health database.} +\item{cdmSourceName}{The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database.} \item{cdmSourceAbbreviation}{The source abbreviation to insert into the CDM_SOURCE table. Default is Synthea.} @@ -56,5 +56,5 @@ This function loads the CDM Event tables with Synthea data. } \details{ This function assumes \cr\code{createCDMTables()}, \cr\code{createSyntheaTables()}, \cr\code{LoadSyntheaTables()}, - and \cr\code{LoadVocabTables()} have all been run. + \cr\code{LoadVocabTables()}, and \cr\code{CreateMapAndRollupTables()} have all been run. } diff --git a/man/backupCDM.Rd b/man/backupCDM.Rd index 18c0980..1fee268 100644 --- a/man/backupCDM.Rd +++ b/man/backupCDM.Rd @@ -13,10 +13,10 @@ function \code{createConnectionDetails} in the \item{cdmSchema}{The name of the database schema that contains the CDM. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} -\item{cdmVersion}{The version of your CDM. Currently "5.3.1" and "6.0.1".} +\item{cdmVersion}{The version of your CDM. Currently "5.3.1" and "5.4.0".} } \description{ This function performs a "back up" of the original tables by renaming them. (Eunomia support) diff --git a/man/createExtraIndices.Rd b/man/createExtraIndices.Rd index c0ce4ef..158d861 100644 --- a/man/createExtraIndices.Rd +++ b/man/createExtraIndices.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/createExtraIndices.R -\name{createExtraIndices} -\alias{createExtraIndices} +% Please edit documentation in R/CreateExtraIndices.R +\name{CreateExtraIndices} +\alias{CreateExtraIndices} \title{Create Optional Extra Indices for ETL Performance} \usage{ -createExtraIndices( +CreateExtraIndices( connectionDetails, cdmSchema, syntheaSchema, diff --git a/man/createPrunedTables.Rd b/man/createPrunedTables.Rd index e3114dd..bb23cc6 100644 --- a/man/createPrunedTables.Rd +++ b/man/createPrunedTables.Rd @@ -13,12 +13,12 @@ function \code{createConnectionDetails} in the \item{cdmSchema}{The name of the database schema that contains the CDM. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} \item{eventConceptId}{A vector of concept_ids returned from \code{getEventConceptId}.} -\item{cdmVersion}{The version of your CDM. Currently "5.3.1" and "6.0.0".} +\item{cdmVersion}{The version of your CDM. Currently "5.3.1" and "5.4.0".} } \description{ This function creates the pruned cdm and vocab tables by keeping only those rows from diff --git a/man/getEventConceptId.Rd b/man/getEventConceptId.Rd index dcee560..fd1c869 100644 --- a/man/getEventConceptId.Rd +++ b/man/getEventConceptId.Rd @@ -13,10 +13,10 @@ function \code{createConnectionDetails} in the \item{cdmSchema}{The name of the database schema that contains the CDM. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} -\item{cdmVersion}{The version of your CDM. Currently "5.3.1" and "6.0.0".} +\item{cdmVersion}{The version of your CDM. Currently "5.3.1" and "5.4.0".} } \description{ This function gets the unique concept_ids in all event tables. diff --git a/man/pruneCDM.Rd b/man/pruneCDM.Rd index 173a0a8..c53b55c 100644 --- a/man/pruneCDM.Rd +++ b/man/pruneCDM.Rd @@ -13,10 +13,10 @@ function \code{createConnectionDetails} in the \item{cdmSchema}{The name of the database schema that contains the CDM. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} -\item{cdmVersion}{The version of your CDM. Currently "5.3.1" and "6.0.0".} +\item{cdmVersion}{The version of your CDM. Currently "5.3.1" and "5.4.0".} } \description{ This function is simply a convenience wrapper for the other calls. (Eunomia support) diff --git a/man/restoreCDMTables.Rd b/man/restoreCDMTables.Rd index 64c4260..ab9aece 100644 --- a/man/restoreCDMTables.Rd +++ b/man/restoreCDMTables.Rd @@ -13,10 +13,10 @@ function \code{createConnectionDetails} in the \item{cdmSchema}{The name of the database schema that contains the CDM instance. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} -\item{cdmVersion}{The version of your CDM. Currently "5.3.1" and "6.0.0" are supported.} +\item{cdmVersion}{The version of your CDM. Currently "5.3.1" and "5.4.0" are supported.} } \description{ This function undoes the pruning by renaming (the *_orig tables are given their names back). From 5c13c7f46d02d6bb8ef29cf9c38f930da4a600e1 Mon Sep 17 00:00:00 2001 From: Evanette Burrows Date: Tue, 2 Jan 2024 15:17:14 -0500 Subject: [PATCH 4/7] Update package to support synthea 3.1.0 and 3.2.0 (#178) * Add support for loading synthea v3.1 and v3.2 * Add synthea version to source description * Address lint errors --- DESCRIPTION | 7 +- R/CreateCDMTables.r | 58 ++-- R/CreateMapAndRollupTables.r | 39 +-- R/CreateSyntheaTables.r | 19 +- R/DropSyntheaTables.r | 41 +-- R/LoadEventTables.r | 59 ++-- R/LoadSyntheaTables.r | 64 ++-- R/LoadVocabFromCsv.r | 4 +- R/createExtraIndices.R | 4 +- R/createPrunedTables.r | 2 +- R/exportToSQLite.r | 8 +- R/getEventConceptId.r | 2 +- README.md | 2 +- docs/404.html | 4 +- docs/articles/Condition_occurrence.html | 4 +- docs/articles/Cost.html | 4 +- docs/articles/Device_exposure.html | 4 +- docs/articles/Drug_exposure.html | 4 +- docs/articles/Measurement.html | 4 +- docs/articles/Payer_plan_period.html | 10 +- docs/articles/death.html | 4 +- docs/articles/index.html | 4 +- docs/articles/observation.html | 4 +- docs/articles/observation_period.html | 4 +- docs/articles/person.html | 4 +- docs/articles/procedure_occurrence.html | 4 +- docs/articles/provider.html | 4 +- docs/articles/visit_detail.html | 4 +- docs/articles/visit_occurrence.html | 4 +- docs/authors.html | 16 +- docs/index.html | 8 +- docs/pkgdown.yml | 2 +- .../CreateCDMIndexAndConstraintScripts.html | 4 +- docs/reference/CreateCDMIndices.html | 4 +- docs/reference/CreateCDMTables.html | 4 +- docs/reference/CreateMapAndRollupTables.html | 10 +- docs/reference/CreateSyntheaTables.html | 6 +- docs/reference/CreateVisitRollupTables.html | 4 +- docs/reference/CreateVocabMapTables.html | 4 +- docs/reference/DropEventTables.html | 4 +- docs/reference/DropMapAndRollupTables.html | 4 +- docs/reference/DropSyntheaTables.html | 4 +- docs/reference/DropVocabTables.html | 4 +- docs/reference/ETLSyntheaBuilder-package.html | 4 +- docs/reference/LoadEventTables.html | 6 +- docs/reference/LoadSyntheaTables.html | 6 +- docs/reference/LoadVocabFromCsv.html | 4 +- docs/reference/LoadVocabFromSchema.html | 4 +- docs/reference/TruncateEventTables.html | 4 +- docs/reference/TruncateVocabTables.html | 4 +- docs/reference/backupCDM.html | 4 +- docs/reference/createExtraIndices.html | 8 +- docs/reference/createPrunedTables.html | 4 +- docs/reference/exportToSQLite.html | 4 +- docs/reference/getEventConceptId.html | 4 +- docs/reference/index.html | 4 +- docs/reference/pruneCDM.html | 4 +- docs/reference/restoreCDMTables.html | 4 +- extras/codeToRun.R | 4 + .../v531/insert_payer_plan_period.sql | 7 +- .../v540/insert_payer_plan_period.sql | 5 + inst/sql/sql_server/extra_indices.sql | 2 +- .../v310/create_synthea_tables.sql | 326 ++++++++++++++++++ .../v320/create_synthea_tables.sql | 326 ++++++++++++++++++ man/CreateMapAndRollupTables.Rd | 4 +- man/CreateSyntheaTables.Rd | 2 +- man/LoadEventTables.Rd | 2 +- man/LoadSyntheaTables.Rd | 2 +- man/createExtraIndices.Rd | 6 +- vignettes/Cost.Rmd | 2 +- vignettes/Payer_plan_period.Rmd | 6 +- vignettes/observation.Rmd | 2 +- vignettes/provider.Rmd | 2 +- 73 files changed, 955 insertions(+), 268 deletions(-) create mode 100644 inst/sql/sql_server/synthea_version/v310/create_synthea_tables.sql create mode 100644 inst/sql/sql_server/synthea_version/v320/create_synthea_tables.sql diff --git a/DESCRIPTION b/DESCRIPTION index 376ed67..620d0bb 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,12 +1,13 @@ Package: ETLSyntheaBuilder Type: Package Title: A Builder for Converting the Synthea Data to the OMOP CDM -Version: 1.0 +Version: 2.0 mDate: 2021-12-04 Author: Anthony Molinaro [aut, cre], Clair Blacketer [aut], - Frank DeFalco [aut] -Maintainer: Anthony Molinaro + Frank DeFalco [aut], + Evanette Burrows [aut] +Maintainer: Evanette Burrows Description: ETL and Builder to convert Synthea Data to the OMOP CDM. Staring with csv files for an OMOP Vocabulary and csv files for Synthea, this package creates database tables from these csv files and maps them diff --git a/R/CreateCDMTables.r b/R/CreateCDMTables.r index f12631f..36e8ce3 100644 --- a/R/CreateCDMTables.r +++ b/R/CreateCDMTables.r @@ -26,41 +26,41 @@ CreateCDMTables <- cdmSchema, cdmVersion, outputFolder = NULL, - createIndices = FALSE, + createIndices = FALSE, sqlOnly = FALSE) { if (!sqlOnly) { - - print("Creating CDM Tables....") - - CommonDataModel::executeDdl( - connectionDetails = connectionDetails, - cdmVersion = cdmVersion, - cdmDatabaseSchema = cdmSchema, - executeDdl = TRUE, - executePrimaryKey = TRUE, - executeForeignKey = FALSE - ) # False for now due to bug: https://github.com/OHDSI/CommonDataModel/issues/452 + print("Creating CDM Tables....") - print("CDM Tables Created.") - - if (createIndices) { - - print("Creating Indices on CDM Tables....") + CommonDataModel::executeDdl( + connectionDetails = connectionDetails, + cdmVersion = cdmVersion, + cdmDatabaseSchema = cdmSchema, + executeDdl = TRUE, + executePrimaryKey = TRUE, + executeForeignKey = FALSE + ) # False for now due to bug: https://github.com/OHDSI/CommonDataModel/issues/452 + + print("CDM Tables Created.") + + if (createIndices) { + print("Creating Indices on CDM Tables....") - indexSQLFile <- CommonDataModel::writeIndex( - targetDialect = connectionDetails$dbms, - cdmVersion = cdmVersion, - cdmDatabaseSchema = cdmSchema, - outputfolder = tempdir()) + indexSQLFile <- CommonDataModel::writeIndex( + targetDialect = connectionDetails$dbms, + cdmVersion = cdmVersion, + cdmDatabaseSchema = cdmSchema, + outputfolder = tempdir() + ) + + indexDDL <- + SqlRender::readSql(paste0(tempdir(), "/", indexSQLFile)) + conn <- DatabaseConnector::connect(connectionDetails) + DatabaseConnector::executeSql(conn, indexDDL) + DatabaseConnector::disconnect(conn) + print("Index Creation Complete.") + } - indexDDL <- SqlRender::readSql(paste0(tempdir(),"/",indexSQLFile)) - conn <- DatabaseConnector::connect(connectionDetails) - DatabaseConnector::executeSql(conn,indexDDL) - DatabaseConnector::disconnect(conn) - print("Index Creation Complete.") - } - } else { if (is.null(outputFolder)) { stop("Must specify an outputFolder location when using sqlOnly = TRUE") diff --git a/R/CreateMapAndRollupTables.r b/R/CreateMapAndRollupTables.r index 77378c7..5bf6261 100644 --- a/R/CreateMapAndRollupTables.r +++ b/R/CreateMapAndRollupTables.r @@ -19,42 +19,39 @@ #' so for example 'cdm_instance.dbo'. #' @param cdmVersion The version of your CDM. Currently "5.3" and "5.4". #' @param syntheaVersion The version of Synthea used to generate the csv files. -#' Currently "2.7.0" and "3.0.0" are supported. +#' Currently "2.7.0", "3.0.0", "3.1.0" and "3.2.0" are supported. #' @param cdmSourceName The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database. #' @param cdmSourceAbbreviation The source abbreviation to insert into the CDM_SOURCE table. Default is Synthea. #' @param cdmHolder The holder to insert into the CDM_SOURCE table. Default is OHDSI #' @param cdmSourceDescription The description of the source data. Default is generic Synthea description. -#' @param createIndices A boolean that determines whether or not to create indices on CDM tables before the ETL. #' @param sqlOnly A boolean that determines whether or not to perform the load or generate SQL scripts. Default is FALSE. #' #'@export CreateMapAndRollupTables <- function(connectionDetails, - cdmSchema, - syntheaSchema, - cdmVersion, - syntheaVersion = "2.7.0", - cdmSourceName = "Synthea synthetic health database", - cdmSourceAbbreviation = "Synthea", - cdmHolder = "OHDSI", - cdmSourceDescription = "SyntheaTM is a Synthetic Patient Population Simulator. The goal is to output synthetic, realistic (but not real), patient data and associated health records in a variety of formats.", - sqlOnly = FALSE) + cdmSchema, + syntheaSchema, + cdmVersion, + syntheaVersion = "2.7.0", + cdmSourceName = "Synthea synthetic health database", + cdmSourceAbbreviation = "Synthea", + cdmHolder = "OHDSI", + cdmSourceDescription = "SyntheaTM is a Synthetic Patient Population Simulator. The goal is to output synthetic, realistic (but not real), patient data and associated health records in a variety of formats.", + sqlOnly = FALSE) { - # Determine which sql scripts to run based on the given version. - # The path is relative to inst/sql/sql_server. - if (cdmVersion == "5.3") { - sqlFilePath <- "cdm_version/v531" - } else if (cdmVersion == "5.4") { - sqlFilePath <- "cdm_version/v540" - } else { + supportedCDMVersions <- c("5.3", "5.4") + + if (!(cdmVersion %in% supportedCDMVersions)) { stop("Unsupported CDM specified. Supported CDM versions are \"5.3\" and \"5.4\".") } - supportedSyntheaVersions <- c("2.7.0", "3.0.0") + supportedSyntheaVersions <- c("2.7.0", "3.0.0", "3.1.0", "3.2.0") if (!(syntheaVersion %in% supportedSyntheaVersions)) - stop("Invalid Synthea version specified. Currently \"2.7.0\" and \"3.0.0\" are supported.") + stop( + "Invalid Synthea version specified. Currently \"2.7.0\", \"3.0.0\",\"3.1.0\", and \"3.2.0\" are supported." + ) # Create Vocabulary mapping tables CreateVocabMapTables(connectionDetails, cdmSchema, cdmVersion, sqlOnly) @@ -67,5 +64,3 @@ CreateMapAndRollupTables <- function(connectionDetails, sqlOnly) } - - diff --git a/R/CreateSyntheaTables.r b/R/CreateSyntheaTables.r index 173c740..43a8954 100644 --- a/R/CreateSyntheaTables.r +++ b/R/CreateSyntheaTables.r @@ -10,25 +10,32 @@ #' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. #' @param syntheaVersion The version of Synthea used to generate the csv files. -#' Currently "2.7.0" and "3.0.0" are is supported. +#' Currently "2.7.0", "3.0.0", "3.1.0", and "3.2.0" are is supported. #' #'@export CreateSyntheaTables <- - function (connectionDetails, - syntheaSchema, - syntheaVersion = "2.7.0") + function(connectionDetails, + syntheaSchema, + syntheaVersion = "2.7.0") { if (syntheaVersion == "2.7.0") sqlFilePath <- "synthea_version/v270" else if (syntheaVersion == "3.0.0") sqlFilePath <- "synthea_version/v300" + else if (syntheaVersion == "3.1.0") + sqlFilePath <- "synthea_version/v310" + else if (syntheaVersion == "3.2.0") + sqlFilePath <- "synthea_version/v320" else - stop("Invalid synthea version specified. Currently \"2.7.0\" and \"3.0.0\" are supported.") + stop( + "Invalid synthea version specified. Currently \"2.7.0\", \"3.0.0\", \"3.1.0\" and \"3.2.0\" are supported." + ) - sqlFilename <- paste0(sqlFilePath, "/", "create_synthea_tables.sql") + sqlFilename <- + paste0(sqlFilePath, "/", "create_synthea_tables.sql") translatedSql <- SqlRender::loadRenderTranslateSql( sqlFilename = sqlFilename, diff --git a/R/DropSyntheaTables.r b/R/DropSyntheaTables.r index 4c81280..067b28b 100644 --- a/R/DropSyntheaTables.r +++ b/R/DropSyntheaTables.r @@ -19,8 +19,8 @@ DropSyntheaTables <- function(connectionDetails, syntheaSchema) syntheaTables <- c( "ALLERGIES", "CAREPLANS", - "CLAIMS", - "CLAIMS_TRANSACTIONS", + "CLAIMS", + "CLAIMS_TRANSACTIONS", "CONDITIONS", "DEVICES", "ENCOUNTERS", @@ -30,33 +30,34 @@ DropSyntheaTables <- function(connectionDetails, syntheaSchema) "OBSERVATIONS", "ORGANIZATIONS", "PATIENTS", - "PAYERS", - "PAYER_TRANSITIONS", + "PAYERS", + "PAYER_TRANSITIONS", "PROCEDURES", "PROVIDERS", - "SUPPLIES" + "SUPPLIES" ) conn <- DatabaseConnector::connect(connectionDetails) allTables <- DatabaseConnector::getTableNames(conn, syntheaSchema) tablesToDrop <- allTables[which(allTables %in% syntheaTables)] - + if (length(tablesToDrop) > 0) { - writeLines("Dropping Synthea tables...") - sql <- - paste( - "drop table @synthea_schema.", - tablesToDrop, - ";", - collapse = "\n", - sep = "" - ) - sql <- SqlRender::render(sql, synthea_schema = syntheaSchema) - sql <- SqlRender::translate(sql, targetDialect = connectionDetails$dbms) - DatabaseConnector::executeSql(conn, sql) + writeLines("Dropping Synthea tables...") + sql <- + paste( + "drop table @synthea_schema.", + tablesToDrop, + ";", + collapse = "\n", + sep = "" + ) + sql <- SqlRender::render(sql, synthea_schema = syntheaSchema) + sql <- + SqlRender::translate(sql, targetDialect = connectionDetails$dbms) + DatabaseConnector::executeSql(conn, sql) } else { - print(sprintf("No synthea tables to drop in schema %s",syntheaSchema)) + print(sprintf("No synthea tables to drop in schema %s", syntheaSchema)) } - + on.exit(DatabaseConnector::disconnect(conn)) } diff --git a/R/LoadEventTables.r b/R/LoadEventTables.r index a381fa9..c0dfea8 100644 --- a/R/LoadEventTables.r +++ b/R/LoadEventTables.r @@ -18,7 +18,7 @@ #' so for example 'cdm_instance.dbo'. #' @param cdmVersion The version of your CDM. Currently "5.3" and "5.4". #' @param syntheaVersion The version of Synthea used to generate the csv files. -#' Currently "2.7.0" and "3.0.0" are supported. +#' Currently "2.7.0","3.0.0","3.1.0" and "3.2.0" are supported. #' @param cdmSourceName The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database. #' @param cdmSourceAbbreviation The source abbreviation to insert into the CDM_SOURCE table. Default is Synthea. #' @param cdmHolder The holder to insert into the CDM_SOURCE table. Default is OHDSI @@ -39,7 +39,7 @@ LoadEventTables <- function(connectionDetails, cdmHolder = "OHDSI", cdmSourceDescription = "SyntheaTM is a Synthetic Patient Population Simulator. The goal is to output synthetic, realistic (but not real), patient data and associated health records in a variety of formats.", createIndices = FALSE, - sqlOnly = FALSE) + sqlOnly = FALSE) { # Determine which sql scripts to run based on the given version. # The path is relative to inst/sql/sql_server. @@ -51,25 +51,28 @@ LoadEventTables <- function(connectionDetails, stop("Unsupported CDM specified. Supported CDM versions are \"5.3\" and \"5.4\".") } - supportedSyntheaVersions <- c("2.7.0", "3.0.0") + supportedSyntheaVersions <- c("2.7.0", "3.0.0", "3.1.0", "3.2.0") if (!(syntheaVersion %in% supportedSyntheaVersions)) - stop("Invalid Synthea version specified. Currently \"2.7.0\" and \"3.0.0\" are supported.") + stop( + "Invalid Synthea version specified. Currently \"2.7.0\", \"3.0.0\",\"3.1.0\", and \"3.2.0\" are supported." + ) if (createIndices) { - print("Creating Indices on CDM Tables....") + print("Creating Indices on CDM Tables....") - indexSQLFile <- CommonDataModel::writeIndex( - targetDialect = connectionDetails$dbms, - cdmVersion = cdmVersion, - cdmDatabaseSchema = cdmSchema, - outputfolder = tempdir()) + indexSQLFile <- CommonDataModel::writeIndex( + targetDialect = connectionDetails$dbms, + cdmVersion = cdmVersion, + cdmDatabaseSchema = cdmSchema, + outputfolder = tempdir() + ) - indexDDL <- SqlRender::readSql(paste0(tempdir(),"/",indexSQLFile)) - conn <- DatabaseConnector::connect(connectionDetails) - DatabaseConnector::executeSql(conn,indexDDL) - DatabaseConnector::disconnect(conn) - print("Index Creation Complete.") + indexDDL <- SqlRender::readSql(paste0(tempdir(), "/", indexSQLFile)) + conn <- DatabaseConnector::connect(connectionDetails) + DatabaseConnector::executeSql(conn, indexDDL) + DatabaseConnector::disconnect(conn) + print("Index Creation Complete.") } if (!sqlOnly) { @@ -233,7 +236,7 @@ LoadEventTables <- function(connectionDetails, cdm_source_name = cdmSourceName, cdm_source_abbreviation = cdmSourceAbbreviation, cdm_holder = cdmHolder, - source_description = cdmSourceDescription + source_description = paste("Synthea version: ", syntheaVersion, " ", cdmSourceDescription) ) runStep(sql, fileQuery) @@ -267,24 +270,24 @@ LoadEventTables <- function(connectionDetails, dbms = connectionDetails$dbms, cdm_schema = cdmSchema, synthea_schema = syntheaSchema, - synthea_version = syntheaVersion + synthea_version = syntheaVersion ) runStep(sql, fileQuery) # cost if (syntheaVersion == "2.7.0") - fileQuery <- "insert_cost_v270.sql" - else if (syntheaVersion == "3.0.0") - fileQuery <- "insert_cost_v300.sql" + fileQuery <- "insert_cost_v270.sql" + else if (syntheaVersion %in% c("3.0.0", "3.1.0", "3.2.0")) + fileQuery <- "insert_cost_v300.sql" - sql <- SqlRender::loadRenderTranslateSql( - sqlFilename = file.path(sqlFilePath, fileQuery), - packageName = "ETLSyntheaBuilder", - dbms = connectionDetails$dbms, - cdm_schema = cdmSchema, - synthea_schema = syntheaSchema - ) - runStep(sql, fileQuery) + sql <- SqlRender::loadRenderTranslateSql( + sqlFilename = file.path(sqlFilePath, fileQuery), + packageName = "ETLSyntheaBuilder", + dbms = connectionDetails$dbms, + cdm_schema = cdmSchema, + synthea_schema = syntheaSchema + ) + runStep(sql, fileQuery) if (!sqlOnly) { DatabaseConnector::disconnect(conn) diff --git a/R/LoadSyntheaTables.r b/R/LoadSyntheaTables.r index ca356bb..2f1341b 100644 --- a/R/LoadSyntheaTables.r +++ b/R/LoadSyntheaTables.r @@ -11,7 +11,7 @@ #' \cr\code{./run_synthea -p 1000} #' #' You can enable csv records in src/main/resources/synthea.properties by setting exporter.csv.export = true. -#' As of the time of this writing the csv files can be found at synthe/output/csv. +#' As of the time of this writing the csv files can be found at synthea/output/csv. #' For more details: \href{https://github.com/synthetichealth/synthea/wiki/Basic-Setup-and-Running}{Synthea Basic Setup} #' #' @param connectionDetails An R object of type\cr\code{connectionDetails} created using the @@ -32,16 +32,15 @@ LoadSyntheaTables <- syntheaSchema, syntheaFileLoc, bulkLoad = FALSE) -{ - + { csvList <- list.files(syntheaFileLoc, pattern = "*.csv") - + conn <- DatabaseConnector::connect(connectionDetails) for (csv in csvList) { syntheaTable <- data.table::fread( - file = paste0(syntheaFileLoc,"/",csv), + file = paste0(syntheaFileLoc, "/", csv), stringsAsFactors = FALSE, header = TRUE, sep = ",", @@ -51,32 +50,49 @@ LoadSyntheaTables <- writeLines(paste0("Loading: ", csv)) # experiencing type conversion errors and need to explicitly case some columns - if ("START" %in% colnames(syntheaTable)) - syntheaTable$START <- - as.Date(syntheaTable$START, format = "%Y-%m-%d") - if ("STOP" %in% colnames(syntheaTable)) + if ("START" %in% colnames(syntheaTable)) { + syntheaTable$START <- + as.Date(syntheaTable$START, format = "%Y-%m-%d") + } + if ("STOP" %in% colnames(syntheaTable)) { syntheaTable$STOP <- - as.Date(syntheaTable$STOP, format = "%Y-%m-%d") - if ("DATE" %in% colnames(syntheaTable)) + as.Date(syntheaTable$STOP, format = "%Y-%m-%d") + } + if ("DATE" %in% colnames(syntheaTable)) { syntheaTable$DATE <- - as.Date(syntheaTable$DATE, format = "%Y-%m-%d") - if ("BIRTHDATE" %in% colnames(syntheaTable)) + as.Date(syntheaTable$DATE, format = "%Y-%m-%d") + } + if ("START_DATE" %in% colnames(syntheaTable)) { + syntheaTable$START_DATE <- + as.Date(syntheaTable$START_DATE, format = "%Y-%m-%d") + } + if ("END_DATE" %in% colnames(syntheaTable)) { + syntheaTable$END_DATE <- + as.Date(syntheaTable$END_DATE, format = "%Y-%m-%d") + } + if ("BIRTHDATE" %in% colnames(syntheaTable)) { syntheaTable$BIRTHDATE <- - as.Date(syntheaTable$BIRTHDATE, format = "%Y-%m-%d") - if ("DEATHDATE" %in% colnames(syntheaTable)) + as.Date(syntheaTable$BIRTHDATE, format = "%Y-%m-%d") + } + if ("DEATHDATE" %in% colnames(syntheaTable)) { syntheaTable$DEATHDATE <- - as.Date(syntheaTable$DEATHDATE, format = "%Y-%m-%d") - if ("CODE" %in% colnames(syntheaTable)) + as.Date(syntheaTable$DEATHDATE, format = "%Y-%m-%d") + } + if ("CODE" %in% colnames(syntheaTable)) { syntheaTable$CODE <- as.character(syntheaTable$CODE) - if ("REASONCODE" %in% colnames(syntheaTable)) + } + if ("REASONCODE" %in% colnames(syntheaTable)) { syntheaTable$REASONCODE <- - as.character(syntheaTable$REASONCODE) - if ("PHONE" %in% colnames(syntheaTable)) + as.character(syntheaTable$REASONCODE) + } + if ("PHONE" %in% colnames(syntheaTable)) { syntheaTable$PHONE <- - as.character(syntheaTable$PHONE) - if ("UTILIZATION" %in% colnames(syntheaTable)) + as.character(syntheaTable$PHONE) + } + if ("UTILIZATION" %in% colnames(syntheaTable)) { syntheaTable$UTILIZATION <- - as.numeric(syntheaTable$UTILIZATION) + as.numeric(syntheaTable$UTILIZATION) + } suppressWarnings({ DatabaseConnector::insertTable( @@ -85,7 +101,7 @@ LoadSyntheaTables <- data = as.data.frame(syntheaTable), dropTableIfExists = FALSE, createTable = FALSE, - bulkLoad = bulkLoad, + bulkLoad = bulkLoad, progressBar = TRUE ) }) diff --git a/R/LoadVocabFromCsv.r b/R/LoadVocabFromCsv.r index 3da247d..c2b6bf2 100644 --- a/R/LoadVocabFromCsv.r +++ b/R/LoadVocabFromCsv.r @@ -54,9 +54,7 @@ LoadVocabFromCsv <- na.strings = "" ) - if (tolower(csv) == "concept.csv" || - tolower(csv) == "concept_relationship.csv" || - tolower(csv) == "drug_strength.csv") { + if (tolower(csv) == "concept.csv" || tolower(csv) == "concept_relationship.csv" || tolower(csv) == "drug_strength.csv") { writeLines(" - handling dates") vocabTable$valid_start_date <- as.Date(as.character(vocabTable$valid_start_date), "%Y%m%d") diff --git a/R/createExtraIndices.R b/R/createExtraIndices.R index b4899a9..9d435d8 100644 --- a/R/createExtraIndices.R +++ b/R/createExtraIndices.R @@ -8,13 +8,13 @@ #' Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'. #' @param syntheaSchema The name of the Synthea database schema. Requires read and write permissions to this schema. On SQL #' Server, this should specify both the database and the schema, so for example 'synthea.dbo'. -#' @param syntheaVersion Your Synthea version. Currently "2.7.0" and "3.0.0" are supported. +#' @param syntheaVersion Your Synthea version. Currently "2.7.0", "3.0.0", "3.1.0" and "3.2.0" are supported. #' @param outputFolder Location of the SQL scripts if sqlOnly = TRUE. Default is NULL. #' @param sqlOnly A boolean that determines whether to create the indices or generate a SQL scripts. Default is FALSE. #' #' @details This function creates indices which have been found to speed up certain long-running INSERT queries in LoadEventTables, #' for some users. Indices are created on the intermediate vocabulary mapping tables; the person & provider CDM tables; -#' and the claims_transactions Synthea table (in Synthea 3.0.0). +#' and the claims_transactions Synthea table (in Synthea 3.0.0, 3.1.0 and 3.2.0) . #' #' @importFrom utils head #' diff --git a/R/createPrunedTables.r b/R/createPrunedTables.r index bb48af4..094818e 100644 --- a/R/createPrunedTables.r +++ b/R/createPrunedTables.r @@ -34,7 +34,7 @@ createPrunedTables <- sqlFileName = paste0(sqlFilePath, "/create_pruned_tables.sql"), packageName = "ETLSyntheaBuilder", dbms = connectionDetails$dbms, - cdm_schema = cdmDatabaseSchema, + cdm_schema = cdmSchema, event_concept_id = eventConceptId ) diff --git a/R/exportToSQLite.r b/R/exportToSQLite.r index 7407187..35445f3 100644 --- a/R/exportToSQLite.r +++ b/R/exportToSQLite.r @@ -72,10 +72,14 @@ exportToSQLite <- sqlQuery <- paste0("select * from ", paste0(cdmSchema, ".", tableName), ";") translatedSql <- - SqlRender::translate(renderedSql, targetDialect = connectionDetails$dbms) + SqlRender::translate(sqlQuery, targetDialect = connectionDetails$dbms) writeLines(paste0("Fetching: ", tableName)) tableData <- DatabaseConnector::querySql(conn, translatedSql) - DatabaseConnector::insertTable(sqliteCon, toupper(tableName), tableData) + DatabaseConnector::insertTable( + connection = sqliteCon, + tableName = toupper(tableName), + data = tableData + ) } DatabaseConnector::disconnect(conn) diff --git a/R/getEventConceptId.r b/R/getEventConceptId.r index 279b587..762b110 100644 --- a/R/getEventConceptId.r +++ b/R/getEventConceptId.r @@ -37,5 +37,5 @@ getEventConceptId <- on.exit(DatabaseConnector::disconnect(conn)) - return(eventConceptId) + eventConceptId } diff --git a/README.md b/README.md index df16582..58b603b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Follow the steps on the [synthea wiki](https://github.com/synthetichealth/synthe # The ETLSyntheaBuilder package leverages the OHDSI/CommonDataModel package for CDM creation. # Valid CDM versions are determined by executing CommonDataModel::listSupportedVersions(). # The strings representing supported CDM versions are currently "5.3" and "5.4". - # The Synthea version we use in this example is 2.7.0. However, at this time we also support 3.0.0. + # The Synthea version we use in this example is 2.7.0. However, at this time we also support 3.0.0, 3.1.0 and 3.2.0. # Please note that Synthea's MASTER branch is always active and this package will be updated to support # future versions as possible. # The schema to load the Synthea tables is called "native". diff --git a/docs/404.html b/docs/404.html index 22887cb..2536686 100644 --- a/docs/404.html +++ b/docs/404.html @@ -32,7 +32,7 @@ ETLSyntheaBuilder - 1.0 + 2.0 @@ -124,7 +124,7 @@

Page not found (404)