Repository: CDCgov/NEDSS-DataIngestion
Pull request #345: Update seeding to resume from the last processed record
Author: login: sizatenquizit
Created: 2025-04-14T11:56:37Z | Merged: 2025-04-16T15:56:11Z


## Notes

Update seeding to resume from the last processed record instead of restarting from the beginning

## JIRA

https://cdc-nbs.atlassian.net/browse/CND-228


Diff:
diff --git a/deduplication/src/main/java/gov/cdc/nbs/deduplication/matching/mapper/LinkRequestMapper.java b/deduplication/src/main/java/gov/cdc/nbs/deduplication/matching/mapper/LinkRequestMapper.java
index 2b45edf5b..bc0f2e6d1 100644
--- a/deduplication/src/main/java/gov/cdc/nbs/deduplication/matching/mapper/LinkRequestMapper.java
+++ b/deduplication/src/main/java/gov/cdc/nbs/deduplication/matching/mapper/LinkRequestMapper.java
@@ -1,10 +1,7 @@
 package gov.cdc.nbs.deduplication.matching.mapper;
 
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
+import java.util.*;
 import java.util.stream.Stream;
 
 import org.apache.logging.log4j.util.Strings;
@@ -36,9 +33,9 @@ public LinkRequest map(PersonMatchRequest request) {
       birthDate = formatter.format(request.personDto().birthTime());
     }
 
-    String race = null;
+    List<String> race = null;
     if (request.races() != null && !request.races().isEmpty()) {
-      race = request.races().get(0).raceCategoryCd();
+      race = request.races().stream().map(PersonMatchRequest.PersonRaceDto::raceCategoryCd).toList();
     }
 
     String sex = null;
@@ -64,8 +61,8 @@ List<Address> toAddresses(List<PostalLocatorDto> postalLocators) {
         .stream()
         .map(pl -> new Address(
             Stream.of(
-                pl.streetAddr1(),
-                pl.streetAddr2())
+                    pl.streetAddr1(),
+                    pl.streetAddr2())
                 .filter(Strings::isNotBlank)
                 .toList(),
             pl.cityDescTxt(),
@@ -82,13 +79,13 @@ List<Name> toNames(List<PersonNameDto> nameDtos) {
         .stream()
         .map(n -> new Name(
             Stream.of(
-                n.firstNm(),
-                n.middleNm())
+                    n.firstNm(),
+                    n.middleNm())
                 .filter(Strings::isNotBlank)
                 .toList(),
             n.lastNm(),
             Stream.of(
-                n.nmSuffix())
+                    n.nmSuffix())
                 .filter(Strings::isNotBlank)
                 .toList()))
         .toList();
diff --git a/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/JobConfig.java b/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/JobConfig.java
index d977c5688..ab3aab48a 100644
--- a/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/JobConfig.java
+++ b/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/JobConfig.java
@@ -45,11 +45,16 @@ public JobConfig(
   }
 
   @Bean("readNbsWriteToMpi")
-  public Step step1(JobRepository jobRepository, PlatformTransactionManager transactionManager) {
-    return new StepBuilder("Read and transform NBS data", jobRepository)
-        .<NbsPerson, NbsPerson>chunk(step1ChunkSize, transactionManager) // Process 10 items per chunk
-        .reader(personReader) // page ids to be processed from NBS
-        .writer(seedWriter) // fetch details and send cluster to MPI for seeding
+  public Step step1(
+      JobRepository jobRepository,
+      PlatformTransactionManager transactionManager
+  ) {
+    return new StepBuilder("readNbsWriteToMpi", jobRepository)
+        .<NbsPerson, NbsPerson>chunk(step1ChunkSize, transactionManager)
+        .reader(personReader)
+        .writer(seedWriter)
+        .faultTolerant()
+        .noSkip(RuntimeException.class)
         .build();
   }
 
diff --git a/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/mapper/MpiPersonMapper.java b/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/mapper/MpiPersonMapper.java
index 5834fac5c..2c315fd22 100644
--- a/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/mapper/MpiPersonMapper.java
+++ b/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/mapper/MpiPersonMapper.java
@@ -4,11 +4,7 @@
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
+import java.util.*;
 
 import org.springframework.jdbc.core.RowMapper;
 import org.springframework.lang.NonNull;
@@ -49,7 +45,7 @@ public MpiPerson mapRow(@NonNull ResultSet rs, int rowNum) throws SQLException {
 
     List<Identifier> identifiers = mapIdentifiers(rs.getString("identifiers"));
 
-    String race = mapRace(rs.getString("race"));
+    List<String> race = mapRace(rs.getString("race"));
 
     return new MpiPerson(
         rs.getString("external_id"),
@@ -160,11 +156,13 @@ List<Identifier> mapIdentifiers(String identifierString) {
   }
 
   /** Convert Race_cd to acceptable value for Record Linkage */
-  String mapRace(String raceString) {
+  List<String> mapRace(String raceString) {
     if (raceString == null || raceString.isBlank()) {
-      return null;
+      return Collections.emptyList();
     }
-    return RACE_MAP.get(raceString);
+    return RACE_MAP.get(raceString) != null ?
+        Collections.singletonList(RACE_MAP.get(raceString)) :
+        Collections.emptyList();
   }
 
 }
diff --git a/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/model/MpiPerson.java b/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/model/MpiPerson.java
index 4d8e5ed05..438485adb 100644
--- a/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/model/MpiPerson.java
+++ b/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/model/MpiPerson.java
@@ -11,7 +11,7 @@ public record MpiPerson(
     List<Address> address,
     List<Name> name,
     List<Telecom> telecom,
-    String race,
+    List<String> race,
     List<Identifier> identifiers) {
 
   public record Address(
diff --git a/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/step/PersonReader.java b/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/step/PersonReader.java
index 057e22399..7883a9653 100644
--- a/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/step/PersonReader.java
+++ b/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/step/PersonReader.java
@@ -2,30 +2,43 @@
 
 import javax.sql.DataSource;
 
+import org.springframework.batch.core.annotation.BeforeStep;
 import org.springframework.batch.item.database.JdbcPagingItemReader;
 import org.springframework.batch.item.database.PagingQueryProvider;
 import org.springframework.batch.item.database.support.SqlPagingQueryProviderFactoryBean;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.dao.EmptyResultDataAccessException;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
 import org.springframework.stereotype.Component;
 
 import gov.cdc.nbs.deduplication.seed.mapper.NbsPersonMapper;
 import gov.cdc.nbs.deduplication.seed.model.NbsPerson;
 
+import java.util.Collections;
+
 @Component
 public class PersonReader extends JdbcPagingItemReader<NbsPerson> {
 
   private final NbsPersonMapper mapper = new NbsPersonMapper();
+  private final NamedParameterJdbcTemplate mpiNamedJdbcTemplate;
+  private final DataSource dataSource;
+
+  private static final String LAST_SEEDED_PERSON_QUERY = "SELECT MAX(external_person_id) FROM mpi_patient";
 
   public PersonReader(
-      @Qualifier("nbs") DataSource dataSource) throws Exception {
+      @Qualifier("nbs") DataSource dataSource,
+      @Qualifier("mpiNamedTemplate") NamedParameterJdbcTemplate mpiNamedJdbcTemplate
+  ) throws Exception {
 
+    this.mpiNamedJdbcTemplate = mpiNamedJdbcTemplate;
+    String whereClause = buildWhereClause(null);
     SqlPagingQueryProviderFactoryBean provider = new SqlPagingQueryProviderFactoryBean();
+    this.dataSource = dataSource;
     provider.setDataSource(dataSource);
     provider.setSelectClause("SELECT person_uid, person_parent_uid");
     provider.setFromClause("FROM person");
-    provider.setWhereClause("WHERE person_uid = person_parent_uid AND record_status_cd = 'ACTIVE' AND cd = 'PAT'");
+    provider.setWhereClause(whereClause);
     provider.setSortKey("person_uid");
-
     this.setName("nbsPersonReader");
     this.setDataSource(dataSource);
     PagingQueryProvider queryProvider = provider.getObject();
@@ -33,7 +46,45 @@ public PersonReader(
       this.setQueryProvider(queryProvider);
     }
     this.setRowMapper(mapper);
-    this.setPageSize(10000);
+    this.setPageSize(1000);
   }
 
+  @BeforeStep
+  public void beforeStep() throws Exception {
+    // Dynamically update the query provider
+    Long lastProcessedId = getLastSeededPersonId();
+    String whereClause = buildWhereClause(lastProcessedId);
+    SqlPagingQueryProviderFactoryBean provider = new SqlPagingQueryProviderFactoryBean();
+    provider.setDataSource(dataSource);
+    provider.setSelectClause("SELECT person_uid, person_parent_uid");
+    provider.setFromClause("FROM person");
+    provider.setWhereClause(whereClause);
+    provider.setSortKey("person_uid");
+
+    PagingQueryProvider queryProvider = provider.getObject();
+    if (queryProvider != null) {
+      this.setQueryProvider(queryProvider);
+    }
+    // Reset the reader's state
+    this.afterPropertiesSet();
+    this.setRowMapper(mapper);
+    this.setPageSize(1000);
+  }
+
+  private String buildWhereClause(Long lastProcessedId) {
+    String baseClause = "WHERE person_uid = person_parent_uid AND record_status_cd = 'ACTIVE' AND cd = 'PAT'";
+    if (lastProcessedId != null && lastProcessedId > 0) {
+      return baseClause + " AND person_uid > " + lastProcessedId;
+    }
+    return baseClause;
+  }
+
+  private Long getLastSeededPersonId() {
+    try {
+      return mpiNamedJdbcTemplate.queryForObject(LAST_SEEDED_PERSON_QUERY,
+          Collections.emptyMap(), Long.class);
+    } catch (EmptyResultDataAccessException e) {
+      return null; // If table is empty, return null
+    }
+  }
 }
diff --git a/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/step/SeedWriter.java b/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/step/SeedWriter.java
index 7685edc89..b8c9fa25b 100644
--- a/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/step/SeedWriter.java
+++ b/deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/step/SeedWriter.java
@@ -8,6 +8,7 @@
 import gov.cdc.nbs.deduplication.seed.mapper.MpiPersonMapper;
 import gov.cdc.nbs.deduplication.seed.model.NbsPerson;
 import gov.cdc.nbs.deduplication.seed.model.SeedRequest;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.batch.item.Chunk;
 import org.springframework.batch.item.ItemWriter;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -30,6 +31,7 @@
  */
 
 @Component
+@Slf4j
 public class SeedWriter implements ItemWriter<NbsPerson> {
 
 
@@ -60,13 +62,17 @@ public void write(Chunk<? extends NbsPerson> chunk) throws Exception {
     SeedRequest request = new SeedRequest(clusters);
     String requestJson = objectMapper.writeValueAsString(request);
 
-    recordLinkageClient.post()
-        .uri("/seed")
-        .contentType(MediaType.APPLICATION_JSON)
-        .accept(MediaType.APPLICATION_JSON)
-        .body(requestJson)
-        .retrieve()
-        .body(MpiResponse.class);
+    try {
+      recordLinkageClient.post()
+          .uri("/seed")
+          .contentType(MediaType.APPLICATION_JSON)
+          .accept(MediaType.APPLICATION_JSON)
+          .body(requestJson)
+          .retrieve()
+          .body(MpiResponse.class);
+    }catch (Exception e){
+      throw new RuntimeException("Failed to seed cluster to MPI");//NOSONAR
+    }
   }
 
   private List<Cluster> fetchClusters(List<String> personParentUids) {
diff --git a/deduplication/src/test/java/gov/cdc/nbs/deduplication/matching/mapper/LinkRequestMapperTest.java b/deduplication/src/test/java/gov/cdc/nbs/deduplication/matching/mapper/LinkRequestMapperTest.java
index 0ed64c5f4..60bb3d5d1 100644
--- a/deduplication/src/test/java/gov/cdc/nbs/deduplication/matching/mapper/LinkRequestMapperTest.java
+++ b/deduplication/src/test/java/gov/cdc/nbs/deduplication/matching/mapper/LinkRequestMapperTest.java
@@ -59,7 +59,7 @@ void mapsRace() {
 
     LinkRequest linkRequest = mapper.map(request);
 
-    assertThat(linkRequest.record().race()).isEqualTo("raceCategoryCd");
+    assertThat(linkRequest.record().race().getFirst()).isEqualTo("raceCategoryCd");
   }
 
   @Test
diff --git a/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/SeedingTest.java b/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/SeedingTest.java
index 8918cd891..37fb3066c 100644
--- a/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/SeedingTest.java
+++ b/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/SeedingTest.java
@@ -5,6 +5,10 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.junit.jupiter.api.Test;
 import org.springframework.batch.core.Job;
 import org.springframework.batch.core.JobExecution;
@@ -105,17 +109,38 @@ void seedMpiTest(@Autowired @Qualifier("seedJob") Job seedJob) throws Exception
 
     // Verify a known patients content
     String rawData = mpiTemplate.queryForObject(MPI_DATA_SELECT, String.class);
+
+    // Preprocess the JSON payload to ensure the 'race' field is an array
+    rawData = transformRaceField(rawData);
+
     MpiPerson mpiData = mapper.readValue(rawData, MpiPerson.class);
 
     validatePatientData(mpiData);
 
   }
 
+  private String transformRaceField(String jsonData) throws JsonProcessingException {
+    ObjectMapper objectMapper = new ObjectMapper();
+    JsonNode jsonNode = objectMapper.readTree(jsonData);
+
+    JsonNode raceNode = jsonNode.get("race");
+    if (raceNode != null && raceNode.isTextual()) {
+      // Convert the single string into an array
+      ArrayNode raceArray = objectMapper.createArrayNode();
+      raceArray.add(raceNode.asText());
+
+      ((ObjectNode) jsonNode).set("race", raceArray);
+    }
+
+    return objectMapper.writeValueAsString(jsonNode);
+  }
+
+
   private void validatePatientData(MpiPerson mpiData) {
     // Personal
     assertThat(mpiData.birth_date()).isEqualTo("1990-01-01");
     assertThat(mpiData.sex()).isEqualTo("M");
-    assertThat(mpiData.race()).isEqualTo("ASIAN");
+    assertThat(mpiData.race().getFirst()).isEqualTo("ASIAN");
 
     // Address
     assertThat(mpiData.address()).hasSize(1);
@@ -170,6 +195,7 @@ private RowCount getMpiPersonCount() {
   private record RowCount(Integer unique, Integer total) {
   }
 
+
   private class RowCountMapper implements RowMapper<RowCount> {
     @Override
     public RowCount mapRow(@NonNull ResultSet rs, int rowNum) throws SQLException {
@@ -177,4 +203,4 @@ public RowCount mapRow(@NonNull ResultSet rs, int rowNum) throws SQLException {
     }
   }
 
-}
\ No newline at end of file
+}
diff --git a/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/mapper/MpiPersonMapperTest.java b/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/mapper/MpiPersonMapperTest.java
index 42baff5e0..39ce4c518 100644
--- a/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/mapper/MpiPersonMapperTest.java
+++ b/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/mapper/MpiPersonMapperTest.java
@@ -70,7 +70,7 @@ void testMap() throws SQLException {
     assertThat(person.address()).hasSize(2);
     assertThat(person.name()).hasSize(2);
     assertThat(person.telecom()).hasSize(2);
-    assertThat(person.race()).isEqualTo("WHITE");
+    assertThat(person.race().getFirst()).isEqualTo("WHITE");
     assertThat(person.sex()).isEqualTo(SEX);
     assertThat(person.birth_date()).isEqualTo(BIRTH_DATE);
 
@@ -258,9 +258,9 @@ void testPhone3() {
 
   @ParameterizedTest
   @CsvSource(value = {
-      "null,null",
-      " ,null",
-      "banana,null",
+      "null,''",
+      " ,''",
+      "banana,''",
       "1002-5,AMERICAN_INDIAN",
       "2028-9,ASIAN",
       "2054-5,BLACK",
@@ -269,7 +269,15 @@ void testPhone3() {
       "2131-1,OTHER",
       "U,UNKNOWN" }, nullValues = { "null" })
   void testRace(String input, String expected) {
-    String race = mapper.mapRace(input);
-    assertThat(race).isEqualTo(expected);
+    List<String> race = mapper.mapRace(input);
+
+    if (expected.isEmpty()) {
+      assertThat(race).isEmpty();
+    } else {
+      assertThat(race).containsExactly(expected);
+    }
   }
+
+
+
 }
diff --git a/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/step/PersonReaderTest.java b/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/step/PersonReaderTest.java
index 1bddee2c0..cf0365fe1 100644
--- a/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/step/PersonReaderTest.java
+++ b/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/step/PersonReaderTest.java
@@ -1,17 +1,25 @@
 package gov.cdc.nbs.deduplication.seed.step;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
 
+import java.lang.reflect.Field;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
+import java.util.Collections;
 
 import javax.sql.DataSource;
 
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
+
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.batch.item.database.JdbcPagingItemReader;
+import org.springframework.batch.item.database.PagingQueryProvider;
+import org.springframework.dao.EmptyResultDataAccessException;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
 
 @ExtendWith(MockitoExtension.class)
 class PersonReaderTest {
@@ -25,14 +33,66 @@ class PersonReaderTest {
   @Mock
   private DatabaseMetaData metadata;
 
-  @Test
-  void initializesReader() throws Exception {
+  @Mock
+  private NamedParameterJdbcTemplate mpiNamedJdbcTemplate;
+
+  private PersonReader reader;
+
+  @BeforeEach
+  void setUp() throws Exception {
     when(dataSource.getConnection()).thenReturn(connection);
     when(connection.getMetaData()).thenReturn(metadata);
     when(metadata.getDatabaseProductName()).thenReturn("sql server");
+    reader = new PersonReader(dataSource, mpiNamedJdbcTemplate);
+  }
 
-    final PersonReader reader = new PersonReader(dataSource);
+  @Test
+  void initializesReader() {
     assertThat(reader).isNotNull();
   }
 
+  @Test
+  void beforeStep_UpdatesQueryProvider_WhenNoLastProcessedId() throws Exception {
+    doReturn(null).when(mpiNamedJdbcTemplate).queryForObject(
+        "SELECT MAX(external_person_id) FROM mpi_patient",
+        Collections.emptyMap(),
+        Long.class
+    );
+
+    reader.beforeStep();
+    assertQueryProviderNotNull();
+  }
+
+  @Test
+  void beforeStep_UpdatesQueryProvider_WithLastProcessedId() throws Exception {
+    Long lastProcessedId = 10014306L;
+    doReturn(lastProcessedId).when(mpiNamedJdbcTemplate).queryForObject(
+        "SELECT MAX(external_person_id) FROM mpi_patient",
+        Collections.emptyMap(),
+        Long.class
+    );
+
+    reader.beforeStep();
+    assertQueryProviderNotNull();
+  }
+
+  @Test
+  void beforeStep_HandlesEmptyResultDataAccessException_WhenTableIsEmpty() throws Exception {
+    doThrow(new EmptyResultDataAccessException(1)).when(mpiNamedJdbcTemplate).queryForObject(
+        "SELECT MAX(external_person_id) FROM mpi_patient",
+        Collections.emptyMap(),
+        Long.class
+    );
+
+    reader.beforeStep();
+    assertQueryProviderNotNull();
+  }
+
+  private void assertQueryProviderNotNull() throws Exception {
+    Field queryProviderField = JdbcPagingItemReader.class.getDeclaredField("queryProvider");
+    queryProviderField.setAccessible(true);
+    PagingQueryProvider queryProvider = (PagingQueryProvider) queryProviderField.get(reader);
+    assertThat(queryProvider).isNotNull();
+  }
+
 }
diff --git a/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/step/SeedWriterTest.java b/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/step/SeedWriterTest.java
index a5fba2a53..d2bdd393b 100644
--- a/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/step/SeedWriterTest.java
+++ b/deduplication/src/test/java/gov/cdc/nbs/deduplication/seed/step/SeedWriterTest.java
@@ -1,6 +1,8 @@
 package gov.cdc.nbs.deduplication.seed.step;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.*;
 
 import java.util.ArrayList;
@@ -21,6 +23,7 @@
 import org.springframework.web.client.RestClient.ResponseSpec;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.web.client.RestClientException;
 
 
 @ExtendWith(MockitoExtension.class)
@@ -41,9 +44,10 @@ class SeedWriterTest {
   private final ObjectMapper mapper = new ObjectMapper();
   JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class);
 
+
   @Test
   void initializes() {
-    SeedWriter newWriter = new SeedWriter(jdbcTemplate,mapper, restClient);
+    SeedWriter newWriter = new SeedWriter(jdbcTemplate, mapper, restClient);
     assertThat(newWriter).isNotNull();
   }
 
@@ -68,4 +72,20 @@ void writesChunk() throws Exception {
     verify(restClient, times(1)).post();
   }
 
+  @Test
+  void writesChunk_shouldThrowExceptionWhenSeedingFails() {
+    SeedWriter writer = new SeedWriter(jdbcTemplate, mapper, restClient);
+    Chunk<NbsPerson> chunk = new Chunk<>(List.of(new NbsPerson("100", "1")));
+
+    when(restClient.post()).thenReturn(uriSpec);
+    when(uriSpec.uri("/seed")).thenReturn(bodySpec);
+    when(bodySpec.accept(MediaType.APPLICATION_JSON)).thenReturn(bodySpec);
+    when(bodySpec.contentType(MediaType.APPLICATION_JSON)).thenReturn(bodySpec);
+    when(bodySpec.body(anyString())).thenReturn(bodySpec);
+    when(bodySpec.retrieve()).thenThrow(new RestClientException("MPI Service Unavailable"));
+
+    Exception exception = assertThrows(RuntimeException.class, () -> writer.write(chunk));
+    assertEquals("Failed to seed cluster to MPI", exception.getMessage());
+  }
+
 }
diff --git a/deduplication/src/test/java/gov/cdc/nbs/deduplication/sync/service/PersonInsertSyncHandlerTest.java b/deduplication/src/test/java/gov/cdc/nbs/deduplication/sync/service/PersonInsertSyncHandlerTest.java
index fd8f27f68..e0eedcb29 100644
--- a/deduplication/src/test/java/gov/cdc/nbs/deduplication/sync/service/PersonInsertSyncHandlerTest.java
+++ b/deduplication/src/test/java/gov/cdc/nbs/deduplication/sync/service/PersonInsertSyncHandlerTest.java
@@ -280,7 +280,7 @@ private MpiPerson createMockMpiPerson(String personUid, String parentId) {
         Collections.emptyList(), // address
         Collections.emptyList(), // name
         Collections.emptyList(), // telecom
-        "race",
+        Collections.singletonList("race"),
         Collections.emptyList() // identifiers
     );
   }
diff --git a/deduplication/src/test/java/gov/cdc/nbs/deduplication/sync/service/PersonUpdateSyncHandlerTest.java b/deduplication/src/test/java/gov/cdc/nbs/deduplication/sync/service/PersonUpdateSyncHandlerTest.java
index f44abed73..fdee7a8ae 100644
--- a/deduplication/src/test/java/gov/cdc/nbs/deduplication/sync/service/PersonUpdateSyncHandlerTest.java
+++ b/deduplication/src/test/java/gov/cdc/nbs/deduplication/sync/service/PersonUpdateSyncHandlerTest.java
@@ -239,7 +239,7 @@ private MpiPerson createMockMpiPerson(String personUid, String parentId) {
         Collections.emptyList(), // address
         Collections.emptyList(), // name
         Collections.emptyList(), // telecom
-        "race",
+        Collections.singletonList("race"),
         Collections.emptyList() // identifiers
     );
   }


Reviews:
user: login: mpeels
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T14:19:04Z
user: login: mpeels
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T14:20:41Z
user: login: mpeels
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T14:22:10Z
user: login: mpeels
body: See comments
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T14:22:59Z
user: login: mpeels
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T13:58:30Z

Review comments:
diff_hunk: @@ -36,9 +33,9 @@ public LinkRequest map(PersonMatchRequest request) {
       birthDate = formatter.format(request.personDto().birthTime());
     }
 
-    String race = null;
+    List<String> race = null;
     if (request.races() != null && !request.races().isEmpty()) {
-      race = request.races().get(0).raceCategoryCd();
+      race = Collections.singletonList(request.races().get(0).raceCategoryCd());
path: deduplication/src/main/java/gov/cdc/nbs/deduplication/matching/mapper/LinkRequestMapper.java
user: login: mpeels
body: Suggestion: Instead of now only returning the first race `.get(0)`, should we send all available races since Record Linker now supports that?
created_at: 2025-04-14T14:19:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2,38 +2,89 @@
 
 import javax.sql.DataSource;
 
+import org.springframework.batch.core.annotation.BeforeStep;
 import org.springframework.batch.item.database.JdbcPagingItemReader;
 import org.springframework.batch.item.database.PagingQueryProvider;
 import org.springframework.batch.item.database.support.SqlPagingQueryProviderFactoryBean;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.dao.EmptyResultDataAccessException;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
 import org.springframework.stereotype.Component;
 
 import gov.cdc.nbs.deduplication.seed.mapper.NbsPersonMapper;
 import gov.cdc.nbs.deduplication.seed.model.NbsPerson;
 
+import java.util.Collections;
+
 @Component
 public class PersonReader extends JdbcPagingItemReader<NbsPerson> {
 
   private final NbsPersonMapper mapper = new NbsPersonMapper();
+  private final NamedParameterJdbcTemplate mpiNamedJdbcTemplate;
+  private final DataSource dataSource;
+
+  private static final String LAST_SEEDED_PERSON_QUERY = "SELECT MAX(external_person_id) FROM mpi_patient";
 
   public PersonReader(
-      @Qualifier("nbs") DataSource dataSource) throws Exception {
+      @Qualifier("nbs") DataSource dataSource,
+      @Qualifier("mpiNamedTemplate") NamedParameterJdbcTemplate mpiNamedJdbcTemplate
+  ) throws Exception {
 
+    this.mpiNamedJdbcTemplate = mpiNamedJdbcTemplate;
+    String whereClause = buildWhereClause(null);
     SqlPagingQueryProviderFactoryBean provider = new SqlPagingQueryProviderFactoryBean();
+    this.dataSource = dataSource;
     provider.setDataSource(dataSource);
     provider.setSelectClause("SELECT person_uid, person_parent_uid");
     provider.setFromClause("FROM person");
-    provider.setWhereClause("WHERE person_uid = person_parent_uid AND record_status_cd = 'ACTIVE' AND cd = 'PAT'");
+    provider.setWhereClause(whereClause);
     provider.setSortKey("person_uid");
-
     this.setName("nbsPersonReader");
     this.setDataSource(dataSource);
     PagingQueryProvider queryProvider = provider.getObject();
     if (queryProvider != null) {
       this.setQueryProvider(queryProvider);
     }
     this.setRowMapper(mapper);
-    this.setPageSize(10000);
+    this.setPageSize(1);
path: deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/step/PersonReader.java
user: login: mpeels
body: Issue: This was probably added for testing and accidentally left.
created_at: 2025-04-14T14:20:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -60,13 +62,18 @@ public void write(Chunk<? extends NbsPerson> chunk) throws Exception {
     SeedRequest request = new SeedRequest(clusters);
     String requestJson = objectMapper.writeValueAsString(request);
 
-    recordLinkageClient.post()
-        .uri("/seed")
-        .contentType(MediaType.APPLICATION_JSON)
-        .accept(MediaType.APPLICATION_JSON)
-        .body(requestJson)
-        .retrieve()
-        .body(MpiResponse.class);
+    try {
+      recordLinkageClient.post()
+          .uri("/seed")
+          .contentType(MediaType.APPLICATION_JSON)
+          .accept(MediaType.APPLICATION_JSON)
+          .body(requestJson)
+          .retrieve()
+          .body(MpiResponse.class);
+    }catch (Exception e){
+      log.error("Failed to seed clusters to MPI, Request Body: {}",request);
path: deduplication/src/main/java/gov/cdc/nbs/deduplication/seed/step/SeedWriter.java
user: login: mpeels
body: Issue: This will leak PII into our log messages
created_at: 2025-04-14T14:22:09Z
author_association: CONTRIBUTOR

PR comments:
user: login: sonarqubecloud[bot]
created_at: 2025-04-15T13:03:42Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=CDCgov_NEDSS-DataIngestion&pullRequest=345) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=CDCgov_NEDSS-DataIngestion&pullRequest=345&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=CDCgov_NEDSS-DataIngestion&pullRequest=345&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=CDCgov_NEDSS-DataIngestion&pullRequest=345&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [96.3% Coverage on New Code](https://sonarcloud.io/component_measures?id=CDCgov_NEDSS-DataIngestion&pullRequest=345&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=CDCgov_NEDSS-DataIngestion&pullRequest=345&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=CDCgov_NEDSS-DataIngestion&pullRequest=345)


author_association: NONE