From a997aa0852431288a93e319a27438ad0c3d3b79d Mon Sep 17 00:00:00 2001 From: Jenny Chan Date: Thu, 11 Jul 2024 15:49:46 -0400 Subject: [PATCH] LL 11505 OCI Data Transforms (#187) * Workshop 2120: Flow update and new repo validation * Update adb.md * Workshop 11092: Development for CloudWorld Initial development for CloudWorld LiveLab * Workshop 11092: Development for CloudWorld * Update README.md * Workshop 11092 - CloudWorld development * Workshop 11092: Self QA * Workshop 11092: Self QA * Workshop 11092: Self QA * Workshop 11092: Self QA * Update extract.md * Workshop 11092: Lint Checker changes * Update create.md * Workshop 11092: Grammarly checks * Workshop 11148: ATP to Kafka workshop New development * ATP-Streaming and OCW udpates * ATP-Stream development * OCI GGS workshop fixes * atp-streaming clean up unused images * atp-kafka clean up * atp-streaming clean up * bidirectional replication clean up * atp-objectstorage cleanup * oci-ggs cleanup * ogg-to-ggs cleanup * ggs-to-ogg cleanup * atp-kafka clean up and create ggs-common/adb/images to test * atp-objectstore consolidation * Workshop 11092: File clean up * Updated image references to ggs-common * Added learn more links to labs * Workshop 11148: File cleanup * Updated image references to ggs-common * Updated Learn more links * Updated README * Workshop 11093: File cleanup * Update adb.md * Workshop 11092: Replicat and monitor updates * Update extract.md * Workshop 2920/OCW #10 * Consolidation of duplicate image files to ggs-common * Updates for OCW * Updates to Workshop 4841/OCW Lab 10 Completed consolidation and update to workshop * Removed duplicate images * Self QA Lintchecker Addressed Linkchecker errors in Workshops 11092 and 4841 * Lintchecker updates on Workshop 11092 * Workshop 11092: Fixed typos * Update extract.md Fixed typos * Workshop 11091: MySQL to ADW Initial development - To be released Aug 30th * Workshop 11091: Added screenshots * Workshop 11091: Replicate MySQL to ADW * Updates as a result of testing * Added screenshots * Workshop 11092 Add screens for api key creation * Update adb.md Typo * Workshop 11092: Incorporated PM feedback * Fixed broken image link * Elaborated on a couple steps as recommended by PM * Expanded on a few notes as recommended by PM * Workshop 11093: QA fixes + Added prerequisite steps and screens to create.md + Edits to other files from testing * Workshop 11148: Self-QA - Fixed broken image links - Added NSG steps and screens * Workshop 4841: Fixed broken images * Update oggmp-atp.md * Workshop 2920 + Fixed broken image + renamed lab title * Workshop 4841/OCW 10 - Connections Changed Registered Databases to Connections * Workshop 4841/OCW 10: PM validation Updates to 4841 for OCW workshop #10 - Assign connection steps added - common screenshots updated * Workshop 11092/OCW 44 Updated My Profile link from User Settings * Workshop 11092/OCW 44 PM requested updates * Update create.md * Updates to Workshop: 11091 * Updates to ATP to OCI Streaming lab * Updates to 11093 * Updated README.md for bidirectional replication * Update introduction.md * Update create.md Fixed typo per Kaylien's feedback * Update create.md * Video link updates * Update manifest.json * Updates for MySQL to ADW workshop * Updates to MySQL workshop * Update mysql.md Fix broken image link * Update image links * Self-QA 11091 * 2920: Update video link * Fixed typos 11091 * Cleanup 2920 * 11091 QA verification tasks - Masked compartment names, IPs, ocids - Changed estimated lab time to estimated time * Masking images * Update create.md Typo * Workshop 11092: Feedback from PMs * Prep 11091 for green button * Update mysql.md Remove curl cmd per PM instructions * Redwood updates to 2920 * Workshop 2920 -Update screens for Redwood -Quartly self-QA * Workshop 2920 * Workshop 2920: Fix broken image links * Squashed commit of the following: commit c1550c55639659a96314adda5f711c79eb1a59b9 Author: Jenny Chan Date: Tue Feb 21 04:42:15 2023 -0500 Workshop 2920 (#49) * Workshop 2120: Flow update and new repo validation * Update adb.md * Workshop 11092: Development for CloudWorld Initial development for CloudWorld LiveLab * Workshop 11092: Development for CloudWorld * Update README.md * Workshop 11092 - CloudWorld development * Workshop 11092: Self QA * Workshop 11092: Self QA * Workshop 11092: Self QA * Workshop 11092: Self QA * Update extract.md * Workshop 11092: Lint Checker changes * Update create.md * Workshop 11092: Grammarly checks * Workshop 11148: ATP to Kafka workshop New development * ATP-Streaming and OCW udpates * ATP-Stream development * OCI GGS workshop fixes * atp-streaming clean up unused images * atp-kafka clean up * atp-streaming clean up * bidirectional replication clean up * atp-objectstorage cleanup * oci-ggs cleanup * ogg-to-ggs cleanup * ggs-to-ogg cleanup * atp-kafka clean up and create ggs-common/adb/images to test * atp-objectstore consolidation * Workshop 11092: File clean up * Updated image references to ggs-common * Added learn more links to labs * Workshop 11148: File cleanup * Updated image references to ggs-common * Updated Learn more links * Updated README * Workshop 11093: File cleanup * Update adb.md * Workshop 11092: Replicat and monitor updates * Update extract.md * Workshop 2920/OCW #10 * Consolidation of duplicate image files to ggs-common * Updates for OCW * Updates to Workshop 4841/OCW Lab 10 Completed consolidation and update to workshop * Removed duplicate images * Self QA Lintchecker Addressed Linkchecker errors in Workshops 11092 and 4841 * Lintchecker updates on Workshop 11092 * Workshop 11092: Fixed typos * Update extract.md Fixed typos * Workshop 11091: MySQL to ADW Initial development - To be released Aug 30th * Workshop 11091: Added screenshots * Workshop 11091: Replicate MySQL to ADW * Updates as a result of testing * Added screenshots * Workshop 11092 Add screens for api key creation * Update adb.md Typo * Workshop 11092: Incorporated PM feedback * Fixed broken image link * Elaborated on a couple steps as recommended by PM * Expanded on a few notes as recommended by PM * Workshop 11093: QA fixes + Added prerequisite steps and screens to create.md + Edits to other files from testing * Workshop 11148: Self-QA - Fixed broken image links - Added NSG steps and screens * Workshop 4841: Fixed broken images * Update oggmp-atp.md * Workshop 2920 + Fixed broken image + renamed lab title * Workshop 4841/OCW 10 - Connections Changed Registered Databases to Connections * Workshop 4841/OCW 10: PM validation Updates to 4841 for OCW workshop #10 - Assign connection steps added - common screenshots updated * Workshop 11092/OCW 44 Updated My Profile link from User Settings * Workshop 11092/OCW 44 PM requested updates * Update create.md * Updates to Workshop: 11091 * Updates to ATP to OCI Streaming lab * Updates to 11093 * Updated README.md for bidirectional replication * Update introduction.md * Update create.md Fixed typo per Kaylien's feedback * Update create.md * Video link updates * Update manifest.json * Updates for MySQL to ADW workshop * Updates to MySQL workshop * Update mysql.md Fix broken image link * Update image links * Self-QA 11091 * 2920: Update video link * Fixed typos 11091 * Cleanup 2920 * 11091 QA verification tasks - Masked compartment names, IPs, ocids - Changed estimated lab time to estimated time * Masking images * Update create.md Typo * Workshop 11092: Feedback from PMs * Prep 11091 for green button * Update mysql.md Remove curl cmd per PM instructions * Redwood updates to 2920 * Workshop 2920 -Update screens for Redwood -Quartly self-QA * Workshop 2920 --------- Co-authored-by: Michelle Malcher <48925485+malcherm@users.noreply.github.com> * Update create.md Fixed broken image link url Renamed connection type * 2920 Self QA Self QA * Update gb-intro.md * 2920 updates Fix typos in 2920. * Workshop 4841 - update screens for Redwood * Workshop 4841: self-QA * MySQL to ATP Updates to images in ggs-common and started quarterly QA * MySQL-GGS screen updates * Update oggmp-atp.md Revert changes from PR 63 * Edits to workshop 2920 for OCW23 Minor text updates * Workshop 2920, HOL 16 + Unlocked GGADMIN for TargetADW * Workshop 2920 Light edits * Workshop 2920 / HOL 16 Updated an image, added notes * Workshop 11091: Quarterly QA updates * Workshop 2920 / HOL 16 Added Support Lab to manifest * Workshop 4841: Quaterly QA Updates to Workshop 4841 (GGS to OGG) for Quaterly QA * Update extract.md * Updated screenshots * Update 03-11b-target-uri.png * Update create-gb.md * Workshop 4861 Quarterly QA * Quarterly QA 4861 - removed redundant images from ggs-commons - added missing images to ggs-commons * Workshop 2920/HOL 16 Updates to lab instructions due to recent UI updates. * Workshop 2920/HOL 16 Complete end to end testing with updates to lab instructions * Workshop 11396 / HOL 21 Edits to instructions based on testing result, updates to UI, and formatting instructions for clarity. * few more updates to ggsa-pipeline for HOL * WMS 11396 HOL 16 Feedback from Katherine * Quarterly QA of MySQL to GGS lab Quarterly QA of MySQL to GGS lab * MySQL-ADW Quarterly QA * Update setup images * GGS UI updates * PM feedback, WMS 2920 testing + Julien feedback + WMS 2920 testing and udpates * Updates to GGSA LL * Update to ggsa images ggsa images * Update create-ggsa-deployment.md * WMS: 11396 Prep for Tenancy (brown button) * Update introduction.md * Updates to GGSA lab * GGSA tenancy livelab + OCI Stream connection * LL 3410 ATP to ObjStore updates * ggsa updates * pubsubnet update * ggs-bidirectional-rep: Validation testing * Bidirectional updates after validation * Delete 02-checkpoint-trandata.md * Update 03-create-target-connection.md * OCI GGSA gb updates * OGG to GGS Workshop updates * Updates to 4861 and 2920 * WMS 11396 GB testing * Prereq updates WMS: 11505 and 11396 * WMS 2920 - Updates for OCW24 HOL17 Update lab to use 23ai deployment * WMS 2920 - OCW24 HOL 17 dev cont'd Updates to Replicats, Monitor, and Purge labs * WMS 2920 - FIx include paths Fixed include paths for building blocks * WMS 2920 HOL 17 - merge conflicts * Update manifest.json * WMS 2920 Update manifests to use 23i * Modify building blocks * WMS ID 11041 Bidirectional replication - 23ai updates * LL 11505 Sandbox updates --------- Co-authored-by: Michelle Malcher <48925485+malcherm@users.noreply.github.com> --- oci-dt/create/create.md | 7 +- oci-dt/extract/extract-23ai.md | 302 ++++++++++++++++++ oci-dt/extract/extract.md | 2 +- .../extracts-replicats-23ai.md | 230 +++++++++++++ oci-dt/monitor/monitor-23ai.md | 49 +++ oci-dt/workshops/sandbox/manifest.json | 21 +- 6 files changed, 594 insertions(+), 17 deletions(-) create mode 100644 oci-dt/extract/extract-23ai.md create mode 100644 oci-dt/extracts-replicats/extracts-replicats-23ai.md create mode 100644 oci-dt/monitor/monitor-23ai.md diff --git a/oci-dt/create/create.md b/oci-dt/create/create.md index 3183ab0a..5cce1346 100644 --- a/oci-dt/create/create.md +++ b/oci-dt/create/create.md @@ -122,10 +122,11 @@ You may now **proceed to the next lab.** ## Learn more -* [Managing deployments](https://docs.oracle.com/en/cloud/paas/goldengate-service/ebbpf/index.html) -* [Managing connections](https://docs.oracle.com/en/cloud/paas/goldengate-service/mcjzr/index.html) +* [Create a data replication deployment](https://docs.oracle.com/en/cloud/paas/goldengate-service/llyhq/#GUID-899C1348-58CA-43EE-B775-EAD3B365A7A9) +* [Connect to Oracle Autonomous Database](https://docs.oracle.com/en/cloud/paas/goldengate-service/tqrlh/) +* [Assign a connection to a deployment](https://docs.oracle.com/en/cloud/paas/goldengate-service/llyhq/#GUID-69F7C944-DD31-40FA-8589-4CB5113E74D0) ## Acknowledgements * **Author** - Katherine Wardhana, User Assistance Developer * **Contributors** - Jenny Chan, Consulting User Assistance Developer, Database User Assistance; Julien Testut, Database Product Management -* **Last Updated By/Date** - Katherine Wardhana, April 2024 +* **Last Updated By/Date** - Jenny Chan, July 2024 diff --git a/oci-dt/extract/extract-23ai.md b/oci-dt/extract/extract-23ai.md new file mode 100644 index 00000000..795aba94 --- /dev/null +++ b/oci-dt/extract/extract-23ai.md @@ -0,0 +1,302 @@ +# Create the Extract process and use Oracle Data Pump to export data + +## Introduction +This lab instructs you to use the OCI GoldenGate deployment console to add transaction information to the source database, add a checkpoint table to the target database, add and run an Extract, and instantiate the target database using Oracle Data Pump. + +Estimated time: 25 minutes + +Watch the video below for a quick walk through of the lab. +[Watch the video](videohub:1_j398evq1) + +### About the Extract process +An Extract is a process that extracts, or captures, data from a source database. + +### Objectives +In this lab, you learn to: +* Log in to the OCI GoldenGate deployment console +* Connect to the source database to add transaction information +* Connect to the target database to add a checkpoint table +* Add and run an Extract +* Export data from the source database using Oracle Data Pump + +### Prerequisites + +* This lab assumes that you completed all preceding labs +* Your deployment is in the Active state. + +## Task 1: Launch the OCI GoldenGate deployment console + +[](include:01-launch-console-23ai.md) + +## Task 2: Add Transaction Data and a Checkpoint Table + +> **NOTE:** Ensure that you enable supplemental logging before adding an Extract or you may encounter errors. If you encounter errors, delete and add the Extract before trying again. + +1. In the left navigation menu, click to expand **DB Connections**, then **SourceATP**, and then finally, **Trandata**. + + ![Administration Service navigation menu](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/03-01-nav-config-dbconnections.png " ") + +3. Next to **TRANDATA Information** click **Add TRANDATA** (plus icon). + + ![Add TRANDATA](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/03-03-trandata-23ai.png " ") + +4. In the Trandata panel, for **Schema Name**, enter `SRC_OCIGGLL`, and then click **Submit**. + + ![Schema Name field populated with SRC_OCIGGLL](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/03-04-schema-name-23ai.png " ") + +5. To verify, enter `SRC_OCIGGLL` into the **Search for Schema** field and click **Search TRANDATA** (magnifying glass icon). + + ![Search icon highlighted](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/03-05-search-23ai.png " ") + + ![SRC_OCIGGLL entered in search field and search results returned](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/03-05-trandata-23ai.png " ") + +6. In the left navigation menu, click to connect to **TargetADW**. + + ![Credentials list with Connect to database TargetADW highlighted](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/03-06-connect-targetadw-23ai.png " ") + +7. Next to Checkpoint, click **Add Checkpoint** (plus icon). + + ![Add Checkpoint highlighted](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/03-07-add-checkpoint-23ai.png " ") + +8. In the Checkpoint Table panel, for **Checkpoint Table**, enter `"SRCMIRROR_OCIGGLL"."CHECKTABLE"`, and then click **Submit**. + + ![Populated Checkpoint table field and submit button highlighted](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/03-08-checktable-23ai.png " ") + + The Checkpoint table updates with the newly added checkpoint table. + + ![Updated checkpoint table](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/03-09-checktable.png " ") + + +## Task 3: Add and run an Extract + +1. In the navigation menu, click **Extracts**. + + ![Click Extracts in left menu](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/04-01-extracts-23ai.png " ") + +2. On the Extracts page, click **Add Extract** (plus icon). + + ![Click Add Extract](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/04-02-add-extract-23ai.png " ") + +3. In the Add Extract panel, on the Extract Information page, for **Extract Type**, select **Integrated Extract**. + +4. For **Process Name**, enter `UAEXT`, and then click **Next**. + + ![Extract Information](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/04-03-int-extract-23ai.png " ") + +5. On the Extract Options page, for Source Credentials **Domain**, select **OracleGoldenGate**, and for **Alias**, select **SourceATP**. + +6. For Extract Trail **Name**, enter `E1`, and then click **Next**. + + ![Extract Options](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/04-06-extract-options-23ai.png " ") + +7. On the Managed Options page, enable **Critical to deployment health**, and then click **Next**. + + ![Critical to deployment health highlighted](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/04-07-dephealth-23ai.png " ") + +8. On the Parameter File page, in the text area, add a new line to the existing text and add the following: + + ``` + -- Capture DDL operations for listed schema tables + ddl include mapped + + -- Add step-by-step history of ddl operations captured + -- to the report file. Very useful when troubleshooting. + ddloptions report + + -- Write capture stats per table to the report file daily. + report at 00:01 + + -- Rollover the report file weekly. Useful when IE runs + -- without being stopped/started for long periods of time to + -- keep the report files from becoming too large. + reportrollover at 00:01 on Sunday + + -- Report total operations captured, and operations per second + -- every 10 minutes. + reportcount every 10 minutes, rate + + -- Table list for capture + table SRC_OCIGGLL.*; + ``` + + ![Copied and pasted script on the parameter file page](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/04-08-paramfile.png " ") + +10. Click **Create and Run**. You return to the Extracts page, where you can find your newly created **UAEXT** Extract after a few moments. + + The status updates to **Running**. + + ![Extract started](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/04-10-uaext-running.png " ") + +## Task 4: Check for long running transactions + +1. In the source database SQL window, enter the following script, and then click **Run Statement**: + + ``` + select start_scn, start_time from gv$transaction where start_scn < (select max(start_scn) from dba_capture); + ``` + + If the query returns any rows, then you must locate the transaction's SCN and then either commit or rollback the transaction. + + ![Run statement and no query results highlighted](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/05-01-sql.png " ") + +## Task 5: Export data using Oracle Data Pump (ExpDP) + +Before using Oracle Data Pump to export data from the source database, first create an Oracle Object Store bucket, then create yourself a Credential Token, and then use these resources to create a credential in ATP. + +1. From the Oracle Cloud Console navigation menu (hamburger icon), click **Storage**, and then **Buckets**. + + ![Object Storage bucket in Oracle Cloud navigation menu](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/06-01-storage-buckets.png " ") + +2. On the **Buckets in <compartment-name>** page, click **Create Bucket**. + + ![Buckets in compartment page](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/06-02-create-bucket.png " ") + +3. In the **Create Bucket** panel, enter a unique name, such as `Bucket-,` and then click **Create**. + + ![Create Bucket panel](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/06-03-bucket.png " ") + +4. From the list of buckets, click the bucket you created. You're brought to the bucket Details page. + +5. On the bucket details page, under Objects, click **Upload**. + +6. Enter a prefix, select a file (a simple text file will do), and then click **Upload**. + +7. Close the panel. The object you uploaded appears in the Objects list. + +8. In the Objects list, click the three dots icon for your object, and then select **View Object Details**. + + ![View Object Details](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/06-08-view-details.png) + +9. In the Object Details panel, copy the URL Path (URI) up the object name and paste it into a text editor. + + > **Note:** Ignore the URL deprecation notice shown on the Object Details page. DataPump does not yet support the new URL syntax. + + ![Object URL Path](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/06-09-url-path.png) + + The bucket URI should match the following syntax: + + ``` + https://objectstorage.<region>.oraclecloud.com/n/<namespace>/b/<bucket-name>/o/ + ``` + +10. In the Oracle Cloud Console global header, click **Profile** (user icon), and then select your username. + + ![Profile menu](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/06-10-profile.png " ") + +11. On the User Details page, under **Resources**, click **Auth Tokens**, and then click **Generate Token**. + + ![Resources on User Details page](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/06-11-auth-token.png " ") + +12. In the Generate Token dialog, enter a description, and then click **Generate Token**. + + ![Generate Token dialog](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/06-12-generate-token.png " ") + +13. Click **Copy**, and then click **Close**. + + > **Note:** Paste the token to a text editor for use in the next step. + + ![Copy highlighted](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/06-13-copy-token.png " ") + +14. In the source database SQL window, enter the following script, replace the placeholders with your user name (UPPERCASE) and token value, and then click **Run Statement**: + + ``` + BEGIN + DBMS_CLOUD.CREATE_CREDENTIAL( + credential_name => 'ADB_OBJECTSTORE', + username => '', + password => '' + ); +END; + ``` + + > **Note:** If you're using a federated user, then the username should be in the following format: `oracleidentitycloudservice/`. + + + ![SQL worksheet](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/06-14-create-credential.png " ") + +15. Run the following script to create the Export Data job using Oracle Data Pump ExpDP. Ensure that you replace the Object Store URI (`https://objectstorage..oraclecloud.com/n//b//o/`) with **your URI** from step 9. `SRC_OCIGGLL.dmp` is a file that will be created when this script runs. + + ``` + DECLARE + ind NUMBER; -- Loop index + h1 NUMBER; -- Data Pump job handle + percent_done NUMBER; -- Percentage of job complete + job_state VARCHAR2(30); -- To keep track of job state + le ku$_LogEntry; -- For WIP and error messages + js ku$_JobStatus; -- The job status from get_status + jd ku$_JobDesc; -- The job description from get_status + sts ku$_Status; -- The status object returned by get_status + + BEGIN + -- Create a (user-named) Data Pump job to do a schema export. + h1 := DBMS_DATAPUMP.OPEN('EXPORT','SCHEMA',NULL,'SRC_OCIGGLL_EXPORT','LATEST'); + + -- Specify a single dump file for the job (using the handle just returned) + -- and a directory object, which must already be defined and accessible + -- to the user running this procedure. + DBMS_DATAPUMP.ADD_FILE(h1,'https://objectstorage.<region>.oraclecloud.com/n/<namespace>/b/<bucket-name>/o/SRC_OCIGGLL.dmp','ADB_OBJECTSTORE','100MB',DBMS_DATAPUMP.KU$_FILE_TYPE_URIDUMP_FILE,1); + + -- A metadata filter is used to specify the schema that will be exported. + DBMS_DATAPUMP.METADATA_FILTER(h1,'SCHEMA_EXPR','IN (''SRC_OCIGGLL'')'); + + -- Start the job. An exception will be generated if something is not set up properly. + DBMS_DATAPUMP.START_JOB(h1); + + -- The export job should now be running. In the following loop, the job + -- is monitored until it completes. In the meantime, progress information is displayed. + percent_done := 0; + job_state := 'UNDEFINED'; + while (job_state != 'COMPLETED') and (job_state != 'STOPPED') loop + dbms_datapump.get_status(h1,dbms_datapump.ku$_status_job_error + dbms_datapump.ku$_status_job_status + dbms_datapump.ku$_status_wip,-1,job_state,sts); + js := sts.job_status; + + -- If the percentage done changed, display the new value. + if js.percent_done != percent_done + then + dbms_output.put_line('*** Job percent done = ' || to_char(js.percent_done)); + percent_done := js.percent_done; + end if; + + -- If any work-in-progress (WIP) or error messages were received for the job, display them. + if (bitand(sts.mask,dbms_datapump.ku$_status_wip) != 0) + then + le := sts.wip; + else + if (bitand(sts.mask,dbms_datapump.ku$_status_job_error) != 0) + then + le := sts.error; + else + le := null; + end if; + end if; + if le is not null + then + ind := le.FIRST; + while ind is not null loop + dbms_output.put_line(le(ind).LogText); + ind := le.NEXT(ind); + end loop; + end if; + end loop; + + -- Indicate that the job finished and detach from it. + dbms_output.put_line('Job has completed'); + dbms_output.put_line('Final job state = ' || job_state); + dbms_datapump.detach(h1); +END; + ``` + + ![SQL worksheet Run Script highlighted](https://oracle-livelabs.github.io/goldengate/ggs-common/extract/images/06-15-sql-return.png " ") + +> **Note:** The Script Output displays what looks like an error, but it reports "Job has completed successfully." You can also review the **DBMS Output** tab to see the same message displayed. + +You may now **proceed to the next lab.** + +## Learn more + +* [Add an Extract for Oracle Database](https://docs.oracle.com/en/cloud/paas/goldengate-service/eeske/index.html) + +## Acknowledgements +* **Author** - Jenny Chan, Consulting User Assistance Developer, Database User Assistance +* **Contributors** - Denis Gray, Database Product Management; Katherine Wardhana, User Assistance Developer +* **Last Updated By/Date** - Jenny Chan, June 2024 diff --git a/oci-dt/extract/extract.md b/oci-dt/extract/extract.md index 94b97c88..ee03abb2 100644 --- a/oci-dt/extract/extract.md +++ b/oci-dt/extract/extract.md @@ -26,7 +26,7 @@ In this lab, you learn to: ## Task 1: Launch the OCI GoldenGate deployment console -[](include:01-launch-console.md) +[](include:01-launch-console-23ai.md) ## Task 2: Add Transaction Data and a Checkpoint Table diff --git a/oci-dt/extracts-replicats/extracts-replicats-23ai.md b/oci-dt/extracts-replicats/extracts-replicats-23ai.md new file mode 100644 index 00000000..8b4837a3 --- /dev/null +++ b/oci-dt/extracts-replicats/extracts-replicats-23ai.md @@ -0,0 +1,230 @@ +# Use Oracle Data Pump to import data and create the Replicat + +## Introduction + +This lab walks you through the steps to instantiate the target database using Oracle Data Pump, and then add and run a Replicat in the Oracle Cloud Infrastructure (OCI) GoldenGate Deployment Console. + +Estimated time: 15 minutes + +Watch the video below for a quick walk through of the lab. +[Watch the video](videohub:1_253sko6i) + +### About Replicats + +A Replicat is a process that delivers data to a target database. + +### Objectives + +In this lab, you will: +* Instantiate the target database using Oracle Data Pump +* Add and run a Replicat + +### Prerequisites + +* This lab assumes that you completed all preceding labs +* Your deployment is in the Active state. + +## Task 1: Import data using Oracle Data Pump (ImpDP) + +Before importing data to the target database, create a credential in the target database to access the exported data file in Oracle Object Store. + +1. In the **Oracle Cloud Console**, open the navigation menu (hamburger icon), select **Oracle Database**, and then click **Autonomous Databases**. + +2. In the list of Autonomous Databases, click **TargetADW**. + + ![Autonomous Data Warehouse page](https://oracle-livelabs.github.io/goldengate/ggs-common/extracts-replicats/images/01-02-targetadw.png " ") + +3. On the **TargetADW Details** page, click **Database actions**, and then select **SQL** from the dropdown. + + ![TargetADW Details page](https://oracle-livelabs.github.io/goldengate/ggs-common/extracts-replicats/images/01-03-db-actions.png " ") + + > **Note:** (Optional) Close the SQL Help dialogs. + +4. Enter the following script and then click **Run Statement** to create a credential in ADW: + + ``` + BEGIN + DBMS_CLOUD.CREATE_CREDENTIAL( + credential_name => 'ADB_OBJECTSTORE', + username => '', + password => '' + ); + END; + + ``` + + > **Note:** If you're using a federated user, then the username should be in the following format: `oracle/` + + ![Script in SQL Worksheet](https://oracle-livelabs.github.io/goldengate/ggs-common/extracts-replicats/images/01-05-sql-script.png " ") + +5. Enter the following script and then click **Run Statement** to import data using ImpDP. Ensure that you replace the Object Store URI (`https://objectstorage..oraclecloud.com/n//b//o/`) with **your URI**: + + ``` + DECLARE + ind NUMBER; -- Loop index + h1 NUMBER; -- Data Pump job handle + percent_done NUMBER; -- Percentage of job complete + job_state VARCHAR2(30); -- To keep track of job state + le ku$_LogEntry; -- For WIP and error messages + js ku$_JobStatus; -- The job status from get_status + jd ku$_JobDesc; -- The job description from get_status + sts ku$_Status; -- The status object returned by get_status + BEGIN + + -- Create a (user-named) Data Pump job to do a "full" import (everything + -- in the dump file without filtering). + h1 := DBMS_DATAPUMP.OPEN('IMPORT','FULL',NULL,'SRCMIRROR_OCIGGLL_IMPORT'); + + -- Specify the single dump file for the job (using the handle just returned) + -- and directory object, which must already be defined and accessible + -- to the user running this procedure. This is the dump file created by + -- the export operation in the first example. + + DBMS_DATAPUMP.ADD_FILE(h1,'https://objectstorage..oraclecloud.com/n//b//o/SRC_OCIGGLL.dmp','ADB_OBJECTSTORE',null,DBMS_DATAPUMP.KU$_FILE_TYPE_URIDUMP_FILE); + + + -- A metadata remap will map all schema objects from SRC_OCIGGLL to SRCMIRROR_OCIGGLL. + DBMS_DATAPUMP.METADATA_REMAP(h1,'REMAP_SCHEMA','SRC_OCIGGLL','SRCMIRROR_OCIGGLL'); + + -- If a table already exists in the destination schema, skip it (leave + -- the preexisting table alone). This is the default, but it does not hurt + -- to specify it explicitly. + DBMS_DATAPUMP.SET_PARAMETER(h1,'TABLE_EXISTS_ACTION','SKIP'); + + -- Start the job. An exception is returned if something is not set up properly. + DBMS_DATAPUMP.START_JOB(h1); + + -- The import job should now be running. In the following loop, the job is + -- monitored until it completes. In the meantime, progress information is + -- displayed. Note: this is identical to the export example. + percent_done := 0; + job_state := 'UNDEFINED'; + while (job_state != 'COMPLETED') and (job_state != 'STOPPED') loop + dbms_datapump.get_status(h1, + dbms_datapump.ku$_status_job_error + + dbms_datapump.ku$_status_job_status + + dbms_datapump.ku$_status_wip,-1,job_state,sts); + js := sts.job_status; + + -- If the percentage done changed, display the new value. + if js.percent_done != percent_done + then + dbms_output.put_line('*** Job percent done = ' || + to_char(js.percent_done)); + percent_done := js.percent_done; + end if; + + -- If any work-in-progress (WIP) or Error messages were received for the job, display them. + if (bitand(sts.mask,dbms_datapump.ku$_status_wip) != 0) + then + le := sts.wip; + else + if (bitand(sts.mask,dbms_datapump.ku$_status_job_error) != 0) + then + le := sts.error; + else + le := null; + end if; + end if; + if le is not null + then + ind := le.FIRST; + while ind is not null loop + dbms_output.put_line(le(ind).LogText); + ind := le.NEXT(ind); + end loop; + end if; + end loop; + + -- Indicate that the job finished and gracefully detach from it. + dbms_output.put_line('Job has completed'); + dbms_output.put_line('Final job state = ' || job_state); + dbms_datapump.detach(h1); + END; + ``` + + ![Run script highlighted](https://oracle-livelabs.github.io/goldengate/ggs-common/extracts-replicats/images/01-06-sql-script.png " ") + + > **Note:** The Script Output displays what looks like an error, but it reports "Job has completed successfully." You can also review the **DBMS Output** tab to see the same message displayed. + +## Task 2: Add and run the Replicat + +1. Return to the OCI GoldenGate Deployment Console. In the left navigation menu, click **Replicats**. + + ![GoldenGate Deployment Console Home page](https://oracle-livelabs.github.io/goldengate/ggs-common/extracts-replicats/images/02-01-replicats.png) + +2. On Replicats page, click **Add Replicat** (plus icon). + + ![Add Replicat page](https://oracle-livelabs.github.io/goldengate/ggs-common/extracts-replicats/images/02-02-add-replicat.png " ") + +3. In the Add Replicat panel, on the Replicat Information page, for Replicate Type, select **Nonintegrated Replicat**. + +4. For **Process Name**, enter `REP`, and then click **Next**. + + ![Replicat information](https://oracle-livelabs.github.io/goldengate/ggs-common/extracts-replicats/images/02-04-replicat-info.png " ") + +4. On the Replicat Options page, for Replicat Trail **Name**, enter `E1`. + +5. For Target Credentials, from the **Domain** dropdown, select **OracleGoldenGate**, and then select **TargetADW** for **Alias**. + +6. For **Checkpoint Table**, select **"SRCMIRROR\_OCIGGLL","CHECKTABLE"**, and then click **Next**. + + ![Replicat options](https://oracle-livelabs.github.io/goldengate/ggs-common/extracts-replicats/images/04-06-replicat-opts.png " ") + +8. On the **Managed Options** page, enable **Critical to deployment health**, and then click **Next**. + + ![Managed options](https://oracle-livelabs.github.io/goldengate/ggs-common/extracts-replicats/images/04-07-managed-opts.png " ") + +9. On the **Parameter File** page, replace **MAP \*.\*, TARGET \*.\*;** with the following script: + + ``` + -- Capture DDL operations for listed schema tables + -- + ddl include mapped + -- + -- Add step-by-step history of ddl operations captured + -- to the report file. Very useful when troubleshooting. + -- + ddloptions report + -- + -- Write capture stats per table to the report file daily. + -- + report at 00:01 + -- + -- Rollover the report file weekly. Useful when PR runs + -- without being stopped/started for long periods of time to + -- keep the report files from becoming too large. + -- + reportrollover at 00:01 on Sunday + -- + -- Report total operations captured, and operations per second + -- every 10 minutes. + -- + reportcount every 10 minutes, rate + -- + -- Table map list for apply + -- + DBOPTIONS ENABLE_INSTANTIATION_FILTERING; + MAP SRC_OCIGGLL.*, TARGET SRCMIRROR_OCIGGLL.*; + ``` + + > **Note:** `DBOPTIONS ENABLE_INSTATIATION_FILTERING` enables CSN filtering on tables imported using Oracle Data Pump. For more information, see [DBOPTIONS Reference](https://docs.oracle.com/pls/topic/lookup?ctx=en/cloud/paas/goldengate-service/using&id=GWURF-GUID-BA8C0CED-D87F-4106-862E-4AD22D910160). + +10. Click **Create and Run**. + + ![Add Replicat - Parameter File](https://oracle-livelabs.github.io/goldengate/ggs-common/extracts-replicats/images/04-10-param-file.png " ") + + The Replicats page updates with your newly created Replicat after a few moments, with the status **Running**. + + ![Replicats page](https://oracle-livelabs.github.io/goldengate/ggs-common/extracts-replicats/images/04-11-replicats-page.png " ") + +You may now **proceed to the next lab.** + +## Learn more + +* [Add a Replicat for Oracle Database](https://docs.oracle.com/en/cloud/paas/goldengate-service/cress/index.html) + +## Acknowledgements +* **Author** - Jenny Chan, Consulting User Assistance Developer, Database User Assistance +* **Contributors** - Denis Gray, Database Product Management; Katherine Wardhana, User Assistance Developer +* **Last Updated By/Date** - Jenny Chan, June 2024 \ No newline at end of file diff --git a/oci-dt/monitor/monitor-23ai.md b/oci-dt/monitor/monitor-23ai.md new file mode 100644 index 00000000..09969011 --- /dev/null +++ b/oci-dt/monitor/monitor-23ai.md @@ -0,0 +1,49 @@ +# Monitor Extracts and Replicats + +## Introduction + +In this lab, you learn to monitor the Extract and Replicat processes that were created and run in the previous lab. + +Estimated time: 2 minutes + +Watch the video below for a quick walk through of the lab. +[Watch the video](videohub:1_fpr4rutd) + +### About Performance Monitoring + +Monitoring the performance of your GoldenGate instance ensures that your data replication processes are running smoothly and efficiently. You can monitor performance in both the Oracle Cloud Infrastructure (OCI) GoldenGate Deployment Console as well as in the Oracle Cloud Console on the Deployment Details page. + +### Objectives + +In this lab, you will: +* View charts and statistics using the Performance Metrics Server in the GoldenGate deployment console +* Use Metrics on the Deployment Details page in the Oracle Cloud Console to determine overall instance health and utilization. + +### Prerequisites + +In order to complete this lab: +* You should have completed the preceding lab +* Have both an Extract and Replicat running + +## Task 1: Perform inserts to the source database + +[](include:01-perform-inserts-source-db-23ai.md) + +## Task 2: Using the Performance Metrics Server + +[](include:02-performance-metrics-server-23ai.md) + +## Task 3: Viewing GoldenGate metrics in the Oracle Cloud console + +[](include:03-viewing-gg-metrics-console.md) + +(Optional) You may now **proceed to the next lab.** + +## Learn more + +* [Monitor performance using the Oracle Cloud console](https://docs.oracle.com/en/cloud/paas/goldengate-service/vddvk/index.html#articletitle) + +## Acknowledgements +* **Author** - Jenny Chan, Consulting User Assistance Developer, Database User Assistance +* **Contributors** - Denis Gray, Database Product Management & Katherine Wardhana, User Assistance Developer +* **Last Updated By/Date** - Jenny Chan, June 2024 diff --git a/oci-dt/workshops/sandbox/manifest.json b/oci-dt/workshops/sandbox/manifest.json index bde51f06..a4b61b49 100644 --- a/oci-dt/workshops/sandbox/manifest.json +++ b/oci-dt/workshops/sandbox/manifest.json @@ -2,17 +2,16 @@ "workshoptitle": "Discover OCI GoldenGate Data Transforms", "help": "livelabs-help-db_us@oracle.com", "include": { - "01-create-vcn-subnet.md":"../../../ggs-common/building-blocks/tasks/adb/01-create-vcn-subnet.md", - "02-create-atp-instance.md":"../../../ggs-common/building-blocks/tasks/adb/02-create-atp-instance.md", - "03-load-atp-schema.md":"../../../ggs-common/building-blocks/tasks/adb/03-load-atp-schema.md", "04-create-adw-instance.md":"../../../ggs-common/building-blocks/tasks/adb/04-create-adw-instance.md", "01-create-source-connection.md":"../../../ggs-common/building-blocks/tasks/create/01-create-source-connection.md", "02-unlock-source-ggadmin.md":"../../../ggs-common/building-blocks/tasks/create/02-unlock-source-ggadmin.md", "03-create-target-connection.md":"../../../ggs-common/building-blocks/tasks/create/03-create-target-connection.md", "04-deployment-details.md":"../../../ggs-common/building-blocks/tasks/create/04-deployment-details.md", "05-assign-connection.md":"../../../ggs-common/building-blocks/tasks/create/05-assign-connection.md", - "01-launch-console.md":"../../../ggs-common/building-blocks/tasks/extract/01-launch-console.md", - "01-create-purge-task.md":"../../../ggs-common/building-blocks/tasks/purge/01-create-purge-task.md" + "01-launch-console-23ai.md":"../../../ggs-common/building-blocks/tasks/extract/01-launch-console-23ai.md", + "01-perform-inserts-source-db-23ai.md":"../../../ggs-common/building-blocks/tasks/monitor/01-perform-inserts-source-db-23ai.md", + "02-performance-metrics-server-23ai.md":"../../../ggs-common/building-blocks/tasks/monitor/02-performance-metrics-server-23ai.md", + "03-viewing-gg-metrics-console.md":"../../../ggs-common/building-blocks/tasks/monitor/03-viewing-gg-metrics-console.md" }, "tutorials": [ { @@ -31,22 +30,18 @@ }, { "title": "Lab 2: Create the Extract process and use Oracle Data Pump to export data", - "filename": "../../extract/extract.md" + "filename": "../../extract/extract-23ai.md" }, { "title": "Lab 3: Use Oracle Data Pump to import data and create the Replicat", - "filename": "../../extracts-replicats/extracts-replicats.md" + "filename": "../../extracts-replicats/extracts-replicats-23ai.md" }, { "title": "Lab 4: Monitor Extracts and Replicats", - "filename": "../../monitor/monitor.md" + "filename": "../../monitor/monitor-23ai.md" }, { - "title": "Lab 5: Create the Data Transforms resources", - "filename": "../../create-dt/create-dt.md" - }, - { - "title": "Lab 6: Create and run a workflow", + "title": "Lab 5: Create and run a workflow", "filename": "../../workflow/workflow.md" }, {