Skip to content

MOSIP-40963 - fix/enhancement - batch + parallel enrichment for Zones/Centers Users Mapping #1262

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ymahtat-dev wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

MOSIP-40963 - fix/enhancement - batch + parallel enrichment for Zones/Centers Users Mapping #1262

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions ...rnel-masterdata-service/src/main/java/io/mosip/kernel/masterdata/config/CommonConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@

import jakarta.servlet.Filter;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.security.concurrent.DelegatingSecurityContextExecutorService;
import org.springframework.web.filter.CommonsRequestLoggingFilter;

import io.mosip.kernel.core.masterdata.util.model.Node;
Expand All @@ -17,6 +19,9 @@
import io.mosip.kernel.masterdata.utils.AuditUtil;
import io.mosip.kernel.masterdata.utils.DefaultSort;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
* Config class with beans for modelmapper and request logging
*
Expand All @@ -29,6 +34,12 @@
@EnableAspectJAutoProxy
public class CommonConfig {

@Value("${mosip.kernel.masterdata.enrichment.executor.min-fixed-thread-pool:2}")
private int minFixedThreadPool;

@Value("${mosip.kernel.masterdata.enrichment.executor.max-fixed-thread-pool:8}")
private int maxFixedThreadPool;

/**
* Produce Request Logging bean
*
Expand Down Expand Up @@ -77,5 +88,16 @@ public DefaultSort defaultSort() {
public AuditUtil auditUtil() {
return new AuditUtil();
}

@Bean(destroyMethod = "shutdown")
public ExecutorService enrichmentExecutor() {
final ExecutorService executorService = Executors.newFixedThreadPool(
Math.min(
Math.max(this.minFixedThreadPool, Runtime.getRuntime().availableProcessors()),
this.maxFixedThreadPool
)
);
return new DelegatingSecurityContextExecutorService(executorService);
}

}
3 changes: 3 additions & 0 deletions ...asterdata-service/src/main/java/io/mosip/kernel/masterdata/repository/ZoneRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ public interface ZoneRepository extends BaseRepository<Zone, CodeAndLanguageCode
@Query("FROM Zone zu WHERE zu.code=?1 and zu.langCode=?2 AND (zu.isDeleted IS NULL OR zu.isDeleted = false) AND zu.isActive=true")
public Zone findZoneByCodeAndLangCodeNonDeletedAndIsActive(String code, String langCode);

@Query("FROM Zone zu WHERE (zu.code in (:zonesCodes)) AND (zu.langCode = :langCode) AND (zu.isDeleted IS NULL OR zu.isDeleted = false) AND zu.isActive = true")
public List<Zone> findZonesByCodesAndLangCodeNonDeletedAndIsActive(List<String> zonesCodes, String langCode);

@Query("FROM Zone zu WHERE zu.code in (:zoneId) AND (zu.isDeleted IS NULL OR zu.isDeleted = false) ")
public List<Zone> findListZonesFromZone(List<String> zoneId);

Expand Down
2 changes: 2 additions & 0 deletions ...rnel-masterdata-service/src/main/java/io/mosip/kernel/masterdata/service/ZoneService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ public interface ZoneService {

public ZoneNameResponseDto getZone(String zoneCode, String langCode);

public List<ZoneNameResponseDto> getZonesByCodesAndLanguage(List<String> zonesCodes, String langCode);

public FilterResponseCodeDto zoneFilterValues(FilterValueDto filterValueDto);

public List<Zone> getZoneListBasedonZoneName(String zoneName);
Expand Down
90 changes: 88 additions & 2 deletions ...service/src/main/java/io/mosip/kernel/masterdata/service/impl/UserDetailsServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import io.mosip.kernel.masterdata.exception.RequestException;
Expand Down Expand Up @@ -149,6 +152,14 @@ public class UserDetailsServiceImpl implements UserDetailsService {
@Autowired
ZoneUserRepository zoneUserRepository;

@Autowired
private ExecutorService enrichmentExecutor;

@Value("${mosip.kernel.masterdata.enrichment.executor.timeout-seconds:30}")
private int enrichmentExecutorTimeoutSeconds;



@Override
public UserDetailsGetExtnDto getUser(String id) {
UserDetails ud = userDetailsRepository.findByIdAndIsDeletedFalseorIsDeletedIsNull(id);
Expand Down Expand Up @@ -685,6 +696,37 @@ public PageResponseDto<UserCenterMappingExtnDto> searchUserCenterMappingDetails(
new OptionalFilter[] { zoneOptionalFilter });
if (page.getContent() != null && !page.getContent().isEmpty()) {
zoneUserSearchDetails = MapperUtils.mapAll(page.getContent(), ZoneUserExtnDto.class);
// recover Users Ids for batch user's names search
final List<String> usersIds = zoneUserSearchDetails.stream().map(ZoneUserExtnDto::getUserId).toList();
// recover Users ZonesCodes for batch zones search
final List<String> usersZonesCodes = zoneUserSearchDetails.stream().map(
ZoneUserExtnDto::getZoneCode
).filter(org.springframework.util.StringUtils::hasText).toList();
// Launch both calls of batch user's names search and zone's names search in parallel
CompletableFuture<Map<String, String>> usersNamesByIdsFuture = CompletableFuture.supplyAsync(
() -> this.getUsersNames(usersIds),
this.enrichmentExecutor
);
CompletableFuture<List<Zone>> recoveredZonesFuture = CompletableFuture.supplyAsync(
() -> this.zoneUtils.getZonesByCodesAndLanguage(usersZonesCodes, searchDto.getLanguageCode()),
this.enrichmentExecutor
);
// Retrieve results (blocks here until available)
final Map<String, String> usersNamesByIds = usersNamesByIdsFuture.orTimeout(
this.enrichmentExecutorTimeoutSeconds,
TimeUnit.SECONDS
).exceptionally(ex -> {
logger.error("Failed to fetch user names in batch", ex);
return Collections.emptyMap();
}).join();
final List<Zone> recoveredZones = recoveredZonesFuture.orTimeout(
this.enrichmentExecutorTimeoutSeconds,
TimeUnit.SECONDS
).exceptionally(ex -> {
logger.error("Failed to fetch zones in batch", ex);
return Collections.emptyList();
}).join();
// Complete page result construction
pageDto = PageUtils.pageResponse(page);
zoneUserSearchDetails.forEach(z -> {
ZoneUserSearchDto dto = new ZoneUserSearchDto();
Expand All @@ -698,12 +740,14 @@ public PageResponseDto<UserCenterMappingExtnDto> searchUserCenterMappingDetails(
dto.setUserId(z.getUserId());
dto.setUpdatedDateTime(z.getUpdatedDateTime());
dto.setUpdatedBy(z.getUpdatedBy());
String username = getUserName(z.getUserId());
String username = usersNamesByIds.getOrDefault(z.getUserId(), null);
dto.setUserName(username == null || username.isBlank() ? z.getUserId() :
String.format(USERNAME_FORMAT, z.getUserId(), username));

if (null != z.getZoneCode()) {
Zone zn = zoneUtils.getZoneBasedOnZoneCodeLanguage(z.getZoneCode(), searchDto.getLanguageCode());
Zone zn = recoveredZones.stream().filter(
rzn -> z.getZoneCode().equals(rzn.getCode())
).findFirst().orElse(null);
dto.setZoneName(null != zn ? zn.getName() : null);
} else
dto.setZoneName(null);
Expand Down Expand Up @@ -748,6 +792,48 @@ private String getUserName(String userId) {
return null;

}

private Map<String, String> getUsersNames(List<String> usersIds) {
if(usersIds == null || usersIds.isEmpty()) {
return Collections.emptyMap();
}
final HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
final UriComponentsBuilder uribuilder = UriComponentsBuilder.fromUriString(userDetails + "/admin");
final List<String> userDetails = new ArrayList<>(usersIds);
final RequestWrapper<Map<String, List<String>>> requestWrapper = new RequestWrapper<>();
final Map<String, List<String>> requestEntry = new HashMap<>();
requestEntry.put("userDetails", userDetails);
requestWrapper.setRequest(requestEntry);
final HttpEntity<RequestWrapper<Map<String, List<String>>>> httpEntity = new HttpEntity<>(
requestWrapper,
headers
);
ResponseEntity<String> response = restTemplate.exchange(
uribuilder.toUriString(),
HttpMethod.POST,
httpEntity,
String.class
);
try {
final ObjectMapper mapper = new ObjectMapper();
final Map responseBody = mapper.readValue(response.getBody(), Map.class);
List<Map<String, String>> usersListResponse = (
(Map<String, List<Map<String, String>>>) responseBody.get("response")
).get("mosipUserDtoList");
if (usersListResponse.isEmpty()) {
return Collections.emptyMap();
} else {
return usersListResponse.stream().collect(Collectors.toMap(
entry -> entry.get("userId"),
entry -> entry.get("name")
));
}
} catch (Exception e) {
logger.error("failed to get users names from authmanager", e);
}
return Collections.emptyMap();
}

private List<UserCenterMappingExtnDto> dtoMapper(List<ZoneUserSearchDto> zoneUserSearchDtos, String languageCode) {
List<UserCenterMappingExtnDto> userCenterMappingExtnDtos=new ArrayList();
Expand Down
24 changes: 24 additions & 0 deletions ...erdata-service/src/main/java/io/mosip/kernel/masterdata/service/impl/ZoneServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,30 @@ public ZoneNameResponseDto getZone(String zoneCode, String langCode) {
return zoneNameResponseDto;
}

@Override
public List<ZoneNameResponseDto> getZonesByCodesAndLanguage(List<String> zonesCodes, String langCode) {
final List<ZoneNameResponseDto> zonesResponse = new ArrayList<>();
List<Zone> zones = null;
try {
final String effectiveLangCode = ((langCode == null) ? languageUtils.getDefaultLanguage() : langCode);
zones = this.zoneRepository.findZonesByCodesAndLangCodeNonDeletedAndIsActive(zonesCodes, effectiveLangCode);
} catch (DataAccessException | DataAccessLayerException exception) {
throw new MasterDataServiceException(ZoneErrorCode.INTERNAL_SERVER_ERROR.getErrorCode(),
ZoneErrorCode.INTERNAL_SERVER_ERROR.getErrorMessage());
}
if (zones == null || zones.isEmpty()) {
throw new DataNotFoundException(ZoneErrorCode.ZONE_ENTITY_NOT_FOUND.getErrorCode(),
ZoneErrorCode.ZONE_ENTITY_NOT_FOUND.getErrorMessage());
}
zones.forEach(zone -> {
ZoneNameResponseDto zoneNameResponseDto = new ZoneNameResponseDto();
zoneNameResponseDto.setZoneCode(zone.getCode());
zoneNameResponseDto.setZoneName(zone.getName());
zonesResponse.add(zoneNameResponseDto);
});
return zonesResponse;
}

@Override
public FilterResponseCodeDto zoneFilterValues(FilterValueDto filterValueDto) {
// TODO Auto-generated method stub
Expand Down
90 changes: 87 additions & 3 deletions ...ta-service/src/main/java/io/mosip/kernel/masterdata/service/impl/ZoneUserServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
import java.time.ZoneId;
import java.time.format.DateTimeParseException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import io.mosip.kernel.masterdata.dto.request.SearchFilter;
Expand All @@ -22,6 +25,7 @@
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;

Expand Down Expand Up @@ -95,6 +99,13 @@ public class ZoneUserServiceImpl implements ZoneUserService {
@Autowired
private AuditUtil auditUtil;

@Autowired
private ExecutorService enrichmentExecutor;

@Value("${mosip.kernel.masterdata.enrichment.executor.timeout-seconds:30}")
private int enrichmentExecutorTimeoutSeconds;


private ObjectMapper mapper = new ObjectMapper();

@Override
Expand Down Expand Up @@ -374,6 +385,37 @@ public PageResponseDto<ZoneUserSearchDto> searchZoneUserMapping(SearchDtoWithout
Page<ZoneUser> page = masterDataSearchHelper.searchMasterdataWithoutLangCode(ZoneUser.class, searchDto, new OptionalFilter[] { zoneOptionalFilter });
if (page.getContent() != null && !page.getContent().isEmpty()) {
zoneUserSearchDetails = MapperUtils.mapAll(page.getContent(), ZoneUserExtnDto.class);
// recover Users Ids for batch user's names search
final List<String> usersIds = zoneUserSearchDetails.stream().map(ZoneUserExtnDto::getUserId).toList();
// recover Users ZonesCodes for batch zones search
final List<String> usersZonesCodes = zoneUserSearchDetails.stream().map(
ZoneUserExtnDto::getZoneCode
).filter(StringUtils::hasText).toList();
// Launch both calls of batch user's names search and zone's names search in parallel
CompletableFuture<Map<String, String>> usersNamesByIdsFuture = CompletableFuture.supplyAsync(
() -> this.getUsersNames(usersIds),
this.enrichmentExecutor
);
CompletableFuture<List<ZoneNameResponseDto>> zoneNamesByCodesFuture = CompletableFuture.supplyAsync(
() -> this.zoneservice.getZonesByCodesAndLanguage(usersZonesCodes, searchDto.getLanguageCode()),
this.enrichmentExecutor
);
// Retrieve results (blocks here until available)
final Map<String, String> usersNamesByIds = usersNamesByIdsFuture.orTimeout(
this.enrichmentExecutorTimeoutSeconds,
TimeUnit.SECONDS
).exceptionally(ex -> {
logger.error("Failed to fetch user names in batch", ex);
return Collections.emptyMap();
}).join();
final List<ZoneNameResponseDto> zoneNamesByCodes = zoneNamesByCodesFuture.orTimeout(
this.enrichmentExecutorTimeoutSeconds,
TimeUnit.SECONDS
).exceptionally(ex -> {
logger.error("Failed to fetch zone names in batch", ex);
return Collections.emptyList();
}).join();
// Complete page result construction
pageDto = PageUtils.pageResponse(page);
zoneUserSearchDetails.forEach(z -> {
ZoneUserSearchDto dto = new ZoneUserSearchDto();
Expand All @@ -387,12 +429,13 @@ public PageResponseDto<ZoneUserSearchDto> searchZoneUserMapping(SearchDtoWithout
dto.setUserId(z.getUserId());
dto.setUpdatedDateTime(z.getUpdatedDateTime());
dto.setUpdatedBy(z.getUpdatedBy());
String username = getUserName(z.getUserId());
String username = usersNamesByIds.getOrDefault(z.getUserId(), null);
dto.setUserName(username == null || username.isBlank() ? z.getUserId() :
String.format(USERNAME_FORMAT, z.getUserId(), username));

if (null != z.getZoneCode()) {
ZoneNameResponseDto zn = zoneservice.getZone(z.getZoneCode(),searchDto.getLanguageCode());
ZoneNameResponseDto zn = zoneNamesByCodes.stream().filter(
znr -> znr.getZoneCode().equals(z.getZoneCode())
).findFirst().orElse(null);
dto.setZoneName(null != zn ? zn.getZoneName() : null);
} else
dto.setZoneName(null);
Expand Down Expand Up @@ -440,6 +483,47 @@ private String getUserName(String userId) {

}

private Map<String, String> getUsersNames(List<String> usersIds) {
if (usersIds == null || usersIds.isEmpty()) {
return Collections.emptyMap();
}
final HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
final UriComponentsBuilder uribuilder = UriComponentsBuilder.fromUriString(userDetails + "/admin");
final List<String> userDetails = new ArrayList<>(usersIds);
final RequestWrapper<Map<String, List<String>>> requestWrapper = new RequestWrapper<>();
final Map<String, List<String>> requestEntry = new HashMap<>();
requestEntry.put("userDetails", userDetails);
requestWrapper.setRequest(requestEntry);
final HttpEntity<RequestWrapper<Map<String, List<String>>>> httpEntity = new HttpEntity<>(
requestWrapper,
headers
);
ResponseEntity<String> response = restTemplate.exchange(
uribuilder.toUriString(),
HttpMethod.POST,
httpEntity,
String.class
);
try {
final Map responseBody = mapper.readValue(response.getBody(), Map.class);
List<Map<String, String>> usersListResponse = (
(Map<String, List<Map<String, String>>>) responseBody.get("response")
).get("mosipUserDtoList");
if (usersListResponse.isEmpty()) {
return Collections.emptyMap();
} else {
return usersListResponse.stream().collect(Collectors.toMap(
entry -> entry.get("userId"),
entry -> entry.get("name")
));
}
} catch (Exception e) {
logger.error("failed to get users names from authmanager", e);
}
return Collections.emptyMap();
}

private String getUserDetailsBasedonUserName(String userName) {
HttpHeaders h = new HttpHeaders();
h.setContentType(MediaType.APPLICATION_JSON);
Expand Down
3 changes: 3 additions & 0 deletions ...n/kernel-masterdata-service/src/main/java/io/mosip/kernel/masterdata/utils/ZoneUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,9 @@ public List<Zone> getChildZoneList(List<String> zoneIds, String zoneCode, String
public Zone getZoneBasedOnZoneCodeLanguage(String zoneCode, String langCode) {
return zoneRepository.findZoneByCodeAndLangCodeNonDeleted(zoneCode, langCode);
}
public List<Zone> getZonesByCodesAndLanguage(List<String> zonesCodes, String langCode) {
return zoneRepository.findZonesByCodesAndLangCodeNonDeletedAndIsActive(zonesCodes, langCode);
}
public List<Zone> getZoneListBasedonZoneName(String zoneName) {
return zoneRepository.findListZonesFromZoneName(zoneName.toLowerCase());
}
Expand Down
5 changes: 5 additions & 0 deletions admin/kernel-masterdata-service/src/main/resources/application-local1.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,3 +403,8 @@ mosip.kernel.masterdata.name.validate.regex=[^A-Za-z]

#Spring AntPathMatcher for flexible path pattern support
spring.mvc.pathmatch.matching-strategy=ANT_PATH_MATCHER


mosip.kernel.masterdata.enrichment.executor.timeout-seconds=30
mosip.kernel.masterdata.enrichment.executor.min-fixed-thread-pool=2
mosip.kernel.masterdata.enrichment.executor.max-fixed-thread-pool=8