APP-926: optimize sp_morbidity_report_datamart_postprocessing

reporting-pipeline-service/src/main/resources/db/changelog/migrations/v7.13/rdb/rdb.changelog-7.13.yaml

@@ -3321,3 +3321,11 @@ databaseChangeLog:
             path: db/changelog/migrations/v7.13/rdb/routines/371-sp_lab100_cleanup.sql
             splitStatements: true
             endDelimiter: GO
+  - changeSet:
+      id: app-926-add_ix_event_metric_event_uid
+      author: liquibase
+      runOnChange: false
+      changes:
+        - sqlFile:
+            path: db/changelog/migrations/v7.13/rdb/tables/265-add_ix_event_metric_event_uid-001.sql
+            splitStatements: false

reporting-pipeline-service/src/main/resources/db/changelog/migrations/v7.13/rdb/routines/048-sp_morbidity_report_datamart_postprocessing-001.sql

@@ -73,6 +73,95 @@ BEGIN
         SET @PROC_STEP_NO = @PROC_STEP_NO + 1;
         SET @PROC_STEP_NAME = ' GENERATING #MORB_EVENT_INIT';
 
+        /*
+            Parse each UID param ONCE into a typed bigint temp table with a clustered PK,
+            then probe with EXISTS. Replaces 7 per-row STRING_SPLIT re-parses / implicit converts.
+        */
+        -- uid_obs
+        IF OBJECT_ID('tempdb..#uid_obs') IS NOT NULL DROP TABLE #uid_obs;
+
+        SELECT DISTINCT
+            u = TRY_CONVERT(numeric(18,0), NULLIF(TRIM(ss.value), ''))
+        INTO #uid_obs
+        FROM STRING_SPLIT(@obs_uids, ',') ss
+        WHERE TRY_CONVERT(numeric(18,0), NULLIF(TRIM(ss.value), '')) IS NOT NULL;
+
+        -- we store this as numeric(18,0) because that's how mr.MORB_RPT_UID is stored
+        -- and we're joining on that later
+        ALTER TABLE #uid_obs ALTER COLUMN u numeric(18,0) NOT NULL;
+
+        ALTER TABLE #uid_obs ADD PRIMARY KEY CLUSTERED (u);
+        -- 
+
+        IF OBJECT_ID('tempdb..#uid_pat') IS NOT NULL DROP TABLE #uid_pat;
+        SELECT DISTINCT u = ISNULL(CAST(ss.value AS bigint), 0)
+        INTO #uid_pat
+        FROM STRING_SPLIT(@pat_uids, ',') ss
+        ;
+        ALTER TABLE #uid_pat ADD PRIMARY KEY CLUSTERED (u);
+
+        IF OBJECT_ID('tempdb..#uid_prov') IS NOT NULL DROP TABLE #uid_prov;
+        SELECT DISTINCT u = ISNULL(CAST(ss.value AS bigint), 0)
+        INTO #uid_prov
+        FROM STRING_SPLIT(@prov_uids, ',') ss
+        ;
+        ALTER TABLE #uid_prov ADD PRIMARY KEY CLUSTERED (u);
+
+        IF OBJECT_ID('tempdb..#uid_org') IS NOT NULL DROP TABLE #uid_org;
+        SELECT DISTINCT u = ISNULL(CAST(ss.value AS bigint), 0)
+        INTO #uid_org
+        FROM STRING_SPLIT(@org_uids, ',') ss
+        ;
+        ALTER TABLE #uid_org ADD PRIMARY KEY CLUSTERED (u);
+
+        IF OBJECT_ID('tempdb..#uid_inv') IS NOT NULL DROP TABLE #uid_inv;
+        SELECT DISTINCT u = ISNULL(CAST(ss.value AS bigint), 0)
+        INTO #uid_inv
+        FROM STRING_SPLIT(@inv_uids, ',') ss
+        ;
+        ALTER TABLE #uid_inv ADD PRIMARY KEY CLUSTERED (u);
+
+        /*
+            Candidate pre-filter (equivalence-safe batch scoping).
+            Collect a SUPERSET of MORB_RPT_KEYs that could pass the #MORB_EVENT_INIT
+            filter, via seek-driven UNION branches each driven by a small #uid_* temp.
+            This lets the main projection seek the ~250 candidate reports instead of
+            scanning every ACTIVE report. The original WHERE below is left UNCHANGED,
+            so it still decides per (MR,MRE) row -> output is byte-identical even when
+            a report has multiple MRE rows (some matching, some not). #cand may contain
+            keys the final WHERE then drops (e.g. inactive); that's fine, it's a superset.
+        */
+        IF OBJECT_ID('tempdb..#cand') IS NOT NULL DROP TABLE #cand;
+        SELECT MORB_RPT_KEY INTO #cand FROM (
+            SELECT MR.MORB_RPT_KEY FROM dbo.MORBIDITY_REPORT MR WITH (NOLOCK)
+              WHERE EXISTS (SELECT 1 FROM #uid_obs z WHERE z.u = MR.MORB_RPT_UID)
+            UNION
+            SELECT MRE.MORB_RPT_KEY FROM dbo.MORBIDITY_REPORT_EVENT MRE WITH (NOLOCK)
+              JOIN dbo.INVESTIGATION inv WITH (NOLOCK) ON inv.INVESTIGATION_KEY = MRE.INVESTIGATION_KEY
+              WHERE EXISTS (SELECT 1 FROM #uid_inv z WHERE z.u = inv.CASE_UID)
+            UNION
+            SELECT MRE.MORB_RPT_KEY FROM dbo.MORBIDITY_REPORT_EVENT MRE WITH (NOLOCK)
+              JOIN dbo.D_PATIENT pat WITH (NOLOCK) ON pat.PATIENT_KEY = MRE.PATIENT_KEY
+              WHERE EXISTS (SELECT 1 FROM #uid_pat z WHERE z.u = pat.PATIENT_UID)
+            UNION
+            SELECT MRE.MORB_RPT_KEY FROM dbo.MORBIDITY_REPORT_EVENT MRE WITH (NOLOCK)
+              JOIN dbo.D_PROVIDER prov WITH (NOLOCK) ON prov.PROVIDER_KEY = MRE.PHYSICIAN_KEY
+              WHERE EXISTS (SELECT 1 FROM #uid_prov z WHERE z.u = prov.PROVIDER_UID)
+            UNION
+            SELECT MRE.MORB_RPT_KEY FROM dbo.MORBIDITY_REPORT_EVENT MRE WITH (NOLOCK)
+              JOIN dbo.D_PROVIDER rep WITH (NOLOCK) ON rep.PROVIDER_KEY = MRE.REPORTER_KEY
+              WHERE EXISTS (SELECT 1 FROM #uid_prov z WHERE z.u = rep.PROVIDER_UID)
+            UNION
+            SELECT MRE.MORB_RPT_KEY FROM dbo.MORBIDITY_REPORT_EVENT MRE WITH (NOLOCK)
+              JOIN dbo.D_ORGANIZATION o WITH (NOLOCK) ON o.ORGANIZATION_KEY = MRE.MORB_RPT_SRC_ORG_KEY
+              WHERE EXISTS (SELECT 1 FROM #uid_org z WHERE z.u = o.ORGANIZATION_UID)
+            UNION
+            SELECT MRE.MORB_RPT_KEY FROM dbo.MORBIDITY_REPORT_EVENT MRE WITH (NOLOCK)
+              JOIN dbo.D_ORGANIZATION h WITH (NOLOCK) ON h.ORGANIZATION_KEY = MRE.HSPTL_KEY
+              WHERE EXISTS (SELECT 1 FROM #uid_org z WHERE z.u = h.ORGANIZATION_UID)
+        ) q;
+        ALTER TABLE #cand ADD PRIMARY KEY CLUSTERED (MORB_RPT_KEY);
+
         SELECT
             MR.MORB_RPT_KEY AS MORBIDITY_REPORT_KEY,
             MRE.PATIENT_KEY AS PERSON_KEY,
@@ -195,6 +284,8 @@ BEGIN
             IIF(MRD.MORBIDITY_REPORT_KEY IS NULL, 'I', 'U') AS DML_IND
         INTO #MORB_EVENT_INIT
         FROM dbo.MORBIDITY_REPORT MR WITH (NOLOCK)
+                 INNER JOIN #cand cnd
+                           ON cnd.MORB_RPT_KEY = MR.MORB_RPT_KEY
                  LEFT JOIN dbo.MORBIDITY_REPORT_EVENT MRE WITH (NOLOCK)
                            ON MR.MORB_RPT_KEY = MRE.MORB_RPT_KEY
                  LEFT JOIN dbo.MORBIDITY_REPORT_DATAMART MRD WITH (NOLOCK)
@@ -214,19 +305,19 @@ BEGIN
                  LEFT JOIN dbo.EVENT_METRIC EM WITH (NOLOCK)
                            ON MR.MORB_RPT_UID = EM.EVENT_UID
         WHERE
-            (inv.CASE_UID IN (SELECT value FROM STRING_SPLIT(@inv_uids, ','))
+            (   EXISTS (SELECT 1 FROM #uid_inv  z WHERE z.u = inv.CASE_UID)
                 OR
-             pat.PATIENT_UID IN (SELECT value FROM STRING_SPLIT(@pat_uids, ','))
+                EXISTS (SELECT 1 FROM #uid_pat  z WHERE z.u = pat.PATIENT_UID)
                 OR
-             prov.PROVIDER_UID IN (SELECT value FROM STRING_SPLIT(@prov_uids, ','))
+                EXISTS (SELECT 1 FROM #uid_prov z WHERE z.u = prov.PROVIDER_UID)
                 OR
-             rep.PROVIDER_UID IN (SELECT value FROM STRING_SPLIT(@prov_uids, ','))
+                EXISTS (SELECT 1 FROM #uid_prov z WHERE z.u = rep.PROVIDER_UID)
                 OR
-             rep_fac.ORGANIZATION_UID IN (SELECT value FROM STRING_SPLIT(@org_uids, ','))
+                EXISTS (SELECT 1 FROM #uid_org  z WHERE z.u = rep_fac.ORGANIZATION_UID)
                 OR
-             hsptl.ORGANIZATION_UID IN (SELECT value FROM STRING_SPLIT(@org_uids, ','))
+                EXISTS (SELECT 1 FROM #uid_org  z WHERE z.u = hsptl.ORGANIZATION_UID)
                 OR
-             CAST(mr.MORB_RPT_UID AS bigint) IN (SELECT value FROM STRING_SPLIT(@obs_uids, ','))
+                EXISTS (SELECT 1 FROM #uid_obs  z WHERE z.u = mr.MORB_RPT_UID)
                 )
           AND MR.MORB_RPT_KEY <> 1
           AND MR.RECORD_STATUS_CD = 'ACTIVE';
@@ -253,7 +344,7 @@ BEGIN
         INTO #INACTIVE_MORB
         FROM dbo.MORBIDITY_REPORT MR WITH (NOLOCK)
         WHERE
-            CAST(MR.MORB_RPT_UID AS bigint) IN (SELECT value FROM STRING_SPLIT(@obs_uids, ','))
+            EXISTS (SELECT 1 FROM #uid_obs z WHERE z.u = MR.MORB_RPT_UID)
           AND MR.RECORD_STATUS_CD = 'INACTIVE';
 
         if @debug = 'true'
@@ -280,6 +371,9 @@ BEGIN
              dbo.LAB_TEST LT WITH (NOLOCK) ON LTR.LAB_TEST_KEY = LT.LAB_TEST_KEY
         WHERE M.MORBIDITY_REPORT_KEY != 1 AND M.RECORD_STATUS_CD = 'ACTIVE';
 
+        CREATE NONCLUSTERED INDEX IX_MORB_TO_LAB_KEYS_LAB_RPT_LOCAL_ID
+            ON #MORB_TO_LAB_KEYS (LAB_RPT_LOCAL_ID);
+
         if @debug = 'true'
             SELECT @Proc_Step_Name, * from #MORB_TO_LAB_KEYS;
 
@@ -331,6 +425,9 @@ BEGIN
                            on m.LAB_RPT_LOCAL_ID = lt.LAB_RPT_LOCAL_ID
         WHERE LT.LAB_TEST_TYPE = 'Result';
 
+        CREATE NONCLUSTERED INDEX IX_MORB_LAB_RESULTS_KEY_ROWNUM
+            ON #MORB_LAB_RESULTS (MORBIDITY_REPORT_KEY, row_num);
+
         if @debug = 'true'
             SELECT @Proc_Step_Name, * from #MORB_LAB_RESULTS;
 
@@ -416,6 +513,9 @@ BEGIN
                  INNER JOIN dbo.TREATMENT T WITH (NOLOCK) ON TE.TREATMENT_KEY = T.TREATMENT_KEY
                  INNER JOIN dbo.RDB_DATE RD WITH (NOLOCK) ON TE.TREATMENT_DT_KEY = RD.DATE_KEY;
 
+        CREATE NONCLUSTERED INDEX IX_MORB_TREATMENTS_KEY_ROWNUM
+            ON #MORB_TREATMENTS (MORBIDITY_REPORT_KEY, row_num);
+
         if @debug = 'true'
             SELECT @Proc_Step_Name, * from #MORB_TREATMENTS;
 
@@ -645,7 +745,8 @@ BEGIN
                   LEFT JOIN dbo.RDB_DATE d2 WITH (NOLOCK)
                             ON src.ILLNESS_ONSET_DT_KEY = d2.DATE_KEY
                   LEFT JOIN dbo.RDB_DATE d3 WITH (NOLOCK)
-                            ON src.HSPTL_DISCHARGE_DT_KEY = d3.DATE_KEY;
+                            ON src.HSPTL_DISCHARGE_DT_KEY = d3.DATE_KEY
+        OPTION (MAXDOP 1);
 
         if @debug = 'true'
             SELECT @Proc_Step_Name, * from #MORB_EVENT_FINAL;
@@ -820,7 +921,7 @@ BEGIN
             tgt.MORB_REPORT_LAST_UPDATED_BY = src.MORB_REPORT_LAST_UPDATED_BY,
             tgt.EXTERNAL_IND = src.EXTERNAL_IND
         FROM #MORB_EVENT_FINAL src
-                 LEFT JOIN dbo.MORBIDITY_REPORT_DATAMART tgt
+                 INNER JOIN dbo.MORBIDITY_REPORT_DATAMART tgt
                            ON src.MORBIDITY_REPORT_KEY = tgt.MORBIDITY_REPORT_KEY
         WHERE src.DML_IND = 'U';
 

reporting-pipeline-service/src/main/resources/db/changelog/migrations/v7.13/rdb/tables/265-add_ix_event_metric_event_uid-001.sql

@@ -0,0 +1,15 @@
+-- APP-926: covering index on EVENT_METRIC(EVENT_UID).
+-- sp_morbidity_report_datamart_postprocessing builds #MORB_EVENT_INIT with
+--   LEFT JOIN dbo.EVENT_METRIC EM ON MR.MORB_RPT_UID = EM.EVENT_UID
+-- but the clustered PK is (EVENT_TYPE, EVENT_UID), so a join on EVENT_UID alone cannot seek and
+-- degrades to a full scan of EVENT_METRIC. This covering
+-- index turns that scan into a seek (the plan's own missing-index hint, 32.7% impact). Idempotent.
+IF NOT EXISTS (
+    SELECT 1 FROM sys.indexes
+    WHERE name = 'IX_EVENT_METRIC_EVENT_UID' AND object_id = OBJECT_ID('dbo.EVENT_METRIC')
+)
+BEGIN
+    CREATE NONCLUSTERED INDEX IX_EVENT_METRIC_EVENT_UID
+        ON dbo.EVENT_METRIC (EVENT_UID)
+        INCLUDE (PROG_AREA_DESC_TXT, ADD_TIME, ADD_USER_ID, LAST_CHG_TIME, LAST_CHG_USER_ID, ADD_USER_NAME, LAST_CHG_USER_NAME);
+END

reporting-pipeline-service/src/test/resources/testData/unit/app926_morbidity_datamart_empty/expected.json

@@ -0,0 +1,7 @@
+{
+  "0": [
+    {
+      "QUALIFYING_ROWS": 0
+    }
+  ]
+}

reporting-pipeline-service/src/test/resources/testData/unit/app926_morbidity_datamart_empty/query.sql

@@ -0,0 +1,8 @@
+-- Empty params must produce no datamart rows in this case's key range.
+-- A COUNT is used instead of a bare row SELECT because the framework requires the
+-- query to return at least one row, so the assertion is that the count equals 0.
+-- NOTE keep query.sql comments free of the statement separator character, since the
+-- QueryRunner splits the file on that character before it strips comment lines.
+SELECT COUNT(*) AS QUALIFYING_ROWS
+FROM RDB_MODERN.dbo.MORBIDITY_REPORT_DATAMART
+WHERE MORBIDITY_REPORT_KEY BETWEEN 9268000 AND 9268999;

reporting-pipeline-service/src/test/resources/testData/unit/app926_morbidity_datamart_empty/setup.sql

@@ -0,0 +1,25 @@
+-- APP-926: empty-parameter guard. Valid, ACTIVE report data is present but every UID
+-- list is empty, so no branch of the qualifying WHERE can match and the datamart gains
+-- no rows in this case's key range. STRING_SPLIT('', ',') yields a single '' token that
+-- casts to 0, and no seeded UID is 0, so nothing qualifies. Key range 9268xxx.
+-- (query.sql asserts a COUNT of 0; the framework requires a query to return a row, so a
+--  zero-row datamart SELECT can't be used here.)
+USE RDB_MODERN;
+
+INSERT INTO dbo.D_PATIENT (PATIENT_KEY, PATIENT_UID, PATIENT_LOCAL_ID)
+VALUES (9268010, 9268010, 'PAT9268010');
+
+INSERT INTO dbo.MORBIDITY_REPORT
+    (MORB_RPT_KEY, MORB_RPT_UID, MORB_RPT_LOCAL_ID, JURISDICTION_NM, ELECTRONIC_IND, RECORD_STATUS_CD)
+VALUES
+    (9268001, 9268001, 'MOR9268001', 'Empty County', NULL, 'ACTIVE');
+
+INSERT INTO dbo.MORBIDITY_REPORT_EVENT
+    (MORB_RPT_KEY, PATIENT_KEY, PHYSICIAN_KEY, REPORTER_KEY, MORB_RPT_SRC_ORG_KEY, HSPTL_KEY,
+     INVESTIGATION_KEY, CONDITION_KEY, NURSING_HOME_KEY, HEALTH_CARE_KEY, LDF_GROUP_KEY,
+     MORB_RPT_CREATE_DT_KEY, HSPTL_DISCHARGE_DT_KEY, ILLNESS_ONSET_DT_KEY, MORB_RPT_DT_KEY, RECORD_STATUS_CD)
+VALUES
+    (9268001, 9268010, 1,1,1,1, 1,1,1,1,1, 1,1,1,1, 'ACTIVE');
+
+EXEC dbo.sp_morbidity_report_datamart_postprocessing
+     @obs_uids=N'', @pat_uids=N'', @prov_uids=N'', @org_uids=N'', @inv_uids=N'', @debug=0;

reporting-pipeline-service/src/test/resources/testData/unit/app926_morbidity_datamart_inv/expected.json

@@ -0,0 +1,20 @@
+{
+  "0": [
+    {
+      "CASE_STATUS": "C",
+      "EXTERNAL_IND": "No",
+      "HOSPITAL_FAC_NAME": null,
+      "INVESTIGATION_CREATED_IND": "Yes",
+      "INVESTIGATION_KEY": 9265010,
+      "JURISDICTION_NAME": "Inv County",
+      "MORBIDITY_REPORT_KEY": 9265001,
+      "MORBIDITY_REPORT_LOCAL_ID": "MOR9265001",
+      "PATIENT_LOCAL_ID": null,
+      "PROGRAM_AREA_DESCRIPTION": "Enteric",
+      "PROVIDER_LAST_NAME": null,
+      "REPORTER_LAST_NAME": null,
+      "REPORTING_FACILITY_UID": null,
+      "REPORT_FAC_NAME": null
+    }
+  ]
+}

reporting-pipeline-service/src/test/resources/testData/unit/app926_morbidity_datamart_inv/query.sql

@@ -0,0 +1,19 @@
+-- Scoped to the 9265000..9265999 key range so only this case's rows are asserted.
+SELECT
+    MORBIDITY_REPORT_KEY,
+    MORBIDITY_REPORT_LOCAL_ID,
+    JURISDICTION_NAME,
+    PATIENT_LOCAL_ID,
+    PROVIDER_LAST_NAME,
+    REPORTER_LAST_NAME,
+    REPORT_FAC_NAME,
+    REPORTING_FACILITY_UID,
+    HOSPITAL_FAC_NAME,
+    INVESTIGATION_KEY,
+    INVESTIGATION_CREATED_IND,
+    CASE_STATUS,
+    PROGRAM_AREA_DESCRIPTION,
+    EXTERNAL_IND
+FROM RDB_MODERN.dbo.MORBIDITY_REPORT_DATAMART
+WHERE MORBIDITY_REPORT_KEY BETWEEN 9265000 AND 9265999
+ORDER BY MORBIDITY_REPORT_KEY;

reporting-pipeline-service/src/test/resources/testData/unit/app926_morbidity_datamart_inv/setup.sql

@@ -0,0 +1,32 @@
+-- APP-926: @inv_uids branch. A report qualifies when MRE.INVESTIGATION_KEY resolves
+-- to an INVESTIGATION whose CASE_UID is in @inv_uids. This also exercises the
+-- INVESTIGATION_CREATED_IND / INVESTIGATION_KEY / CASE_STATUS output columns
+-- (NULLIF(inv.INVESTIGATION_KEY, 1) and IIF(COALESCE(...,1)=1,'No','Yes')).
+-- Key range 9265xxx.
+--   report 9265001 -> INVESTIGATION_KEY 9265010 (CASE_UID 9265010, in @inv) -> qualifies
+--   report 9265002 -> INVESTIGATION_KEY 9265011 (CASE_UID 9265011, NOT in @inv) -> excluded
+USE RDB_MODERN;
+
+INSERT INTO dbo.INVESTIGATION (INVESTIGATION_KEY, CASE_UID, INV_CASE_STATUS, RECORD_STATUS_CD)
+VALUES (9265010, 9265010, 'C', 'ACTIVE'),
+       (9265011, 9265011, 'P', 'ACTIVE');
+
+INSERT INTO dbo.MORBIDITY_REPORT
+    (MORB_RPT_KEY, MORB_RPT_UID, MORB_RPT_LOCAL_ID, JURISDICTION_NM, ELECTRONIC_IND, RECORD_STATUS_CD)
+VALUES
+    (9265001, 9265001, 'MOR9265001', 'Inv County', NULL, 'ACTIVE'),
+    (9265002, 9265002, 'MOR9265002', 'Inv County', NULL, 'ACTIVE');
+
+INSERT INTO dbo.MORBIDITY_REPORT_EVENT
+    (MORB_RPT_KEY, PATIENT_KEY, PHYSICIAN_KEY, REPORTER_KEY, MORB_RPT_SRC_ORG_KEY, HSPTL_KEY,
+     INVESTIGATION_KEY, CONDITION_KEY, NURSING_HOME_KEY, HEALTH_CARE_KEY, LDF_GROUP_KEY,
+     MORB_RPT_CREATE_DT_KEY, HSPTL_DISCHARGE_DT_KEY, ILLNESS_ONSET_DT_KEY, MORB_RPT_DT_KEY, RECORD_STATUS_CD)
+VALUES
+    (9265001, 1,1,1,1,1, 9265010, 1,1,1,1, 1,1,1,1, 'ACTIVE'),
+    (9265002, 1,1,1,1,1, 9265011, 1,1,1,1, 1,1,1,1, 'ACTIVE');
+
+INSERT INTO dbo.EVENT_METRIC (EVENT_TYPE, EVENT_UID, PROG_AREA_DESC_TXT)
+VALUES ('Observation', 9265001, 'Enteric');
+
+EXEC dbo.sp_morbidity_report_datamart_postprocessing
+     @obs_uids=N'', @pat_uids=N'', @prov_uids=N'', @org_uids=N'', @inv_uids=N'9265010', @debug=0;