Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update ETL-Synthea for development updates #185

Merged
merged 8 commits into from
Feb 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions DESCRIPTION
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
Package: ETLSyntheaBuilder
Type: Package
Title: A Builder for Converting the Synthea Data to the OMOP CDM
Version: 1.0
Version: 2.0
mDate: 2021-12-04
Author: Anthony Molinaro [aut, cre],
Clair Blacketer [aut],
Frank DeFalco [aut]
Maintainer: Anthony Molinaro <[email protected]>
Frank DeFalco [aut],
Evanette Burrows [aut]
Maintainer: Evanette Burrows <[email protected]>
Description: ETL and Builder to convert Synthea Data to the OMOP CDM.
Staring with csv files for an OMOP Vocabulary and csv files for Synthea,
this package creates database tables from these csv files and maps them
Expand All @@ -27,7 +28,7 @@ Remotes:
OHDSI/CommonDataModel
Encoding: UTF-8
LazyData: true
RoxygenNote: 7.2.2
RoxygenNote: 7.2.3
Suggests:
rmarkdown,
knitr,
Expand Down
3 changes: 2 additions & 1 deletion NAMESPACE
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
export(CreateCDMIndexAndConstraintScripts)
export(CreateCDMIndices)
export(CreateCDMTables)
export(CreateExtraIndices)
export(CreateMapAndRollupTables)
export(CreateSyntheaTables)
export(CreateVisitRollupTables)
export(CreateVocabMapTables)
Expand All @@ -17,7 +19,6 @@ export(LoadVocabFromSchema)
export(TruncateEventTables)
export(TruncateVocabTables)
export(backupCDM)
export(createExtraIndices)
export(createPrunedTables)
export(exportToSQLite)
export(getEventConceptId)
Expand Down
181 changes: 46 additions & 135 deletions R/CreateCDMIndexAndConstraintScripts.r
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
#' function \code{createConnectionDetails} in the
#' \code{DatabaseConnector} package.
#' @param cdmSchema The name of the CDM database schema. Requires read and write permissions to this database. On SQL
#' Server, this should specifiy both the database and the schema,
#' Server, this should specify both the database and the schema,
#' so for example 'cdm_instance.dbo'.
#' @param cdmVersion Your CDM version. Currently "5.3.1" and "6.0.0" are supported.
#' @param cdmVersion Your CDM version. Currently "5.3.1" and "5.4.0" are supported.
#' @param githubTag An optional github tag from which to pull the DDL script.
#' Currently "v5.3.1", "v5.3.1_fixes", "v6.0.0", and "v6.0.0_fixes" are supported. The default is NULL.
#' Currently "v5.3.1" and "v5.4.0" are supported. The default is NULL.
#'
#' @details This function creates SQL scripts for the indices and constraints on tables in a CDM by referring to the
#' correct SQL DDL script in the OHDSI CommonDataModel repo. The database platform is
Expand All @@ -29,13 +29,13 @@ CreateCDMIndexAndConstraintScripts <-
{
supportedDbs <-
c("oracle", "postgresql", "pdw", "netezza", "sql server")
supportedVersions <- c("5.3.1", "6.0.0")
supportedVersions <- c("5.3.1", "5.4.0")
supportedTags <-
c("v5.3.1", "v6.0.0", "v5.3.1_fixes", "v6.0.0_fixes")
c("v5.3.1", "v5.4.0")
rdbms <- tolower(connectionDetails$dbms)

if (!(cdmVersion %in% supportedVersions)) {
stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"6.0.0\"")
stop("Unsupported CDM specified. Supported CDM versions are \"5.3.1\" and \"5.4.0\"")
}

if (!(rdbms %in% supportedDbs)) {
Expand All @@ -46,17 +46,17 @@ CreateCDMIndexAndConstraintScripts <-

if (!is.null(githubTag) && !(githubTag %in% supportedTags)) {
stop(
"Unrecognized github tag. Supported values are \"v5.3.1\", \"v6.0.0\", \"v5.3.1_fixes\", and \"v6.0.0_fixes\""
"Unrecognized github tag. Supported values are \"v5.3.1\" and \"v5.4.0\""
)
}

if (cdmVersion == "5.3.1" &&
!(githubTag %in% c("v5.3.1", "v5.3.1_fixes"))) {
!(githubTag %in% c("v5.3.1"))) {
stop("cdmVersion and githubTag mismatch.")
}

if (cdmVersion == "6.0.0" &&
!(githubTag %in% c("v6.0.0", "v6.0.0_fixes"))) {
if (cdmVersion == "5.4.0" &&
!(githubTag %in% c("v5.4.0"))) {
stop("cdmVersion and githubTag mismatch.")
}

Expand All @@ -74,34 +74,16 @@ CreateCDMIndexAndConstraintScripts <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/Oracle/OMOP%20CDM%20oracle%20constraints.txt"
)
} else if (githubTag == "v5.3.1_fixes") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Oracle/OMOP%20CDM%20oracle%20v5_3_1%20primary%20keys.sql"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Oracle/OMOP%20CDM%20oracle%20v5_3_1%20constraints.sql"
)
}
} else if (cdmVersion == "6.0.0") {
if (is.null(githubTag) || githubTag == "v6.0.0") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Oracle/OMOP%20CDM%20oracle%20pk%20indexes.txt"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Oracle/OMOP%20CDM%20oracle%20constraints.txt"
)
} else if (githubTag == "v6.0.0_fixes") {
} else if (cdmVersion == "5.4.0") {
if (is.null(githubTag) || githubTag == "v5.4.0") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Oracle/OMOP%20CDM%20oracle%20pk%20indexes.txt"
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/oracle/OMOPCDM_oracle_5.4_indices.sql"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Oracle/OMOP%20CDM%20oracle%20constraints.txt"
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/oracle/OMOPCDM_oracle_5.4_constraints.sql"
)
}
}
Expand All @@ -116,34 +98,16 @@ CreateCDMIndexAndConstraintScripts <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/PostgreSQL/OMOP%20CDM%20postgresql%20constraints.txt"
)
} else if (githubTag == "v5.3.1_fixes") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20v5_3_1%20primary%20keys.sql"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20v5_3_1%20constraints.sql"
)
}
} else if (cdmVersion == "6.0.0") {
if (is.null(githubTag) || githubTag == "v6.0.0") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/PostgreSQL/OMOP%20CDM%20postgresql%20pk%20indexes.txt"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/PostgreSQL/OMOP%20CDM%20postgresql%20constraints.txt"
)
} else if (githubTag == "v6.0.0_fixes") {
} else if (cdmVersion == "5.4.0") {
if (is.null(githubTag) || githubTag == "v5.4.0") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20pk%20indexes.txt"
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/postgresql/OMOPCDM_postgresql_5.4_indices.sql"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/PostgreSQL/OMOP%20CDM%20postgresql%20constraints.txt"
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/postgresql/OMOPCDM_postgresql_5.4_constraints.sql"
)
}
}
Expand All @@ -158,34 +122,16 @@ CreateCDMIndexAndConstraintScripts <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/ParallelDataWarehouse/OMOP%20CDM%20pdw%20constraints.txt"
)
} else if (githubTag == "v5.3.1_fixes") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20v5_3_1%20indices.sql"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20v5_3_1%20constraints.sql"
)
}
} else if (cdmVersion == "6.0.0") {
if (is.null(githubTag) || githubTag == "v6.0.0") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/ParallelDataWarehouse/OMOP%20CDM%20pdw%20pk%20indexes.txt"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/ParallelDataWarehouse/OMOP%20CDM%20pdw%20constraints.txt"
)
} else if (githubTag == "v6.0.0_fixes") {
} else if (cdmVersion == "5.4.0") {
if (is.null(githubTag) || githubTag == "v5.4.0") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20pk%20indexes.txt"
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/pdw/OMOPCDM_pdw_5.4_indices.sql"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/ParallelDataWarehouse/OMOP%20CDM%20pdw%20constraints.txt"
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/pdw/OMOPCDM_pdw_5.4_constraints.sql"
)
}
}
Expand All @@ -196,22 +142,12 @@ CreateCDMIndexAndConstraintScripts <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/Netezza/OMOP%20CDM%20netezza%20primary%20keys.txt"
)
} else if (githubTag == "v5.3.1_fixes") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Netezza/OMOP%20CDM%20netezza%20v5_3_1%20primary%20keys.sql"
)
}
} else if (cdmVersion == "6.0.0") {
if (is.null(githubTag) || githubTag == "v6.0.0") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Netezza/OMOP%20CDM%20netezza%20primary%20keys.txt"
)
} else if (githubTag == "v6.0.0_fixes") {
} else if (cdmVersion == "5.4.0") {
if (is.null(githubTag) || githubTag == "v5.4.0") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Netezza/OMOP%20CDM%20netezza%20primary%20keys.txt"
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/netezza/OMOPCDM_netezza_5.4_primary_keys.sql"
)
}
}
Expand All @@ -226,34 +162,16 @@ CreateCDMIndexAndConstraintScripts <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1/Sql%20Server/OMOP%20CDM%20sql%20server%20constraints.txt"
)
} else if (githubTag == "v5.3.1_fixes") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20v5_3_1%20primary%20keys.sql"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.3.1_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20v5_3_1%20constraints.sql"
)
}
} else if (cdmVersion == "6.0.0") {
if (is.null(githubTag) || githubTag == "v6.0.0") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Sql%20Server/OMOP%20CDM%20sql%20server%20pk%20indexes.txt"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0.0/Sql%20Server/OMOP%20CDM%20sql%20server%20constraints.txt"
)
} else if (githubTag == "v6.0.0_fixes") {
} else if (cdmVersion == "5.4.0") {
if (is.null(githubTag) || githubTag == "v5.4.0") {
webResponseInd <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20pk%20indexes.txt"
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/sql_server/OMOPCDM_sql_server_5.4_indices.sql"
)
webResponseCon <-
httr::GET(
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v6.0_fixes/Sql%20Server/OMOP%20CDM%20sql%20server%20constraints.txt"
"https://raw.githubusercontent.com/OHDSI/CommonDataModel/v5.4.0/inst/ddl/5.4/sql_server/OMOPCDM_sql_server_5.4_primary_keys.sql"
)
}
}
Expand All @@ -267,30 +185,23 @@ CreateCDMIndexAndConstraintScripts <-
indexDDL <- toupper(indexDDL)
constraintDDL <- toupper(constraintDDL)

if (githubTag == "v5.3.1_fixes") {
indexDDL <-
SqlRender::render(sql = indexDDL, CDMDATABASESCHEMA = cdmSchema)
constraintDDL <-
SqlRender::render(sql = constraintDDL, CDMDATABASESCHEMA = cdmSchema)
} else {
indexDDL <- gsub(" ON ", " ON @CDM_SCHEMA.", indexDDL)
indexDDL <-
gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", indexDDL)
indexDDL <- gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", indexDDL)
indexDDL <-
SqlRender::render(indexDDL, CDM_SCHEMA = cdmSchema)
indexDDL <-
SqlRender::translate(indexDDL, targetDialect = rdbms)

constraintDDL <-
gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", constraintDDL)
constraintDDL <-
gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", constraintDDL)
constraintDDL <-
SqlRender::render(constraintDDL, CDM_SCHEMA = cdmSchema)
constraintDDL <-
SqlRender::translate(constraintDDL, targetDialect = rdbms)
}
indexDDL <- gsub(" ON ", " ON @CDM_SCHEMA.", indexDDL)
indexDDL <-
gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", indexDDL)
indexDDL <- gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", indexDDL)
indexDDL <-
SqlRender::render(indexDDL, CDM_SCHEMA = cdmSchema)
indexDDL <-
SqlRender::translate(indexDDL, targetDialect = rdbms)

constraintDDL <-
gsub("ALTER TABLE", "ALTER TABLE @CDM_SCHEMA.", constraintDDL)
constraintDDL <-
gsub("@CDM_SCHEMA. ", "@CDM_SCHEMA.", constraintDDL)
constraintDDL <-
SqlRender::render(constraintDDL, CDM_SCHEMA = cdmSchema)
constraintDDL <-
SqlRender::translate(constraintDDL, targetDialect = rdbms)

# Save the translated sql ddl to be run at a later time
if (!dir.exists("output")) {
Expand Down
32 changes: 16 additions & 16 deletions R/CreateCDMIndices.r
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#' function \code{createConnectionDetails} in the
#' \code{DatabaseConnector} package.
#' @param cdmSchema The name of the CDM database schema. Requires read and write permissions to this database. On SQL
#' Server, this should specifiy both the database and the schema,
#' Server, this should specify both the database and the schema,
#' so for example 'cdm_instance.dbo'.
#' @param cdmVersion Your CDM version. Currently "5.3" and "5.4" are supported.
#' @param outputFolder Location of the SQL scripts if sqlOnly = TRUE. Default is NULL.
Expand All @@ -25,23 +25,23 @@ CreateCDMIndices <-
cdmVersion,
outputFolder = NULL,
sqlOnly = FALSE)
{
{
if (!sqlOnly) {

print("Creating Indices on CDM Tables....")
print("Creating Indices on CDM Tables....")

indexSQLFile <- CommonDataModel::writeIndex(
targetDialect = connectionDetails$dbms,
cdmVersion = cdmVersion,
cdmDatabaseSchema = cdmSchema,
outputfolder = tempdir())
indexSQLFile <- CommonDataModel::writeIndex(
targetDialect = connectionDetails$dbms,
cdmVersion = cdmVersion,
cdmDatabaseSchema = cdmSchema,
outputfolder = tempdir()
)

indexDDL <- SqlRender::readSql(paste0(tempdir(), "/", indexSQLFile))
conn <- DatabaseConnector::connect(connectionDetails)
DatabaseConnector::executeSql(conn, indexDDL)
DatabaseConnector::disconnect(conn)
print("Index Creation Complete.")

indexDDL <- SqlRender::readSql(paste0(tempdir(),"/",indexSQLFile))
conn <- DatabaseConnector::connect(connectionDetails)
DatabaseConnector::executeSql(conn,indexDDL)
DatabaseConnector::disconnect(conn)
print("Index Creation Complete.")

} else {
if (is.null(outputFolder)) {
stop("Must specify an outputFolder location when using sqlOnly = TRUE")
Expand All @@ -56,4 +56,4 @@ CreateCDMIndices <-
cdmDatabaseSchema = cdmSchema
)
}
}
}
Loading
Loading