RM/torgi2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added integration test

Browse Source 
use spring-retry instead of custom
add logging
This commit is contained in:
RM
2026-10-05 20:09:26 +03:00
parent 9813f2489e
commit b1e79e1e2b
6 changed files with 191 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
pom.xml
View File

@@ -29,6 +29,7 @@
<properties>
<java.version>17</java.version>
<spring-cloud.version>2025.0.0</spring-cloud.version>
<testcontainers.version>1.21.4</testcontainers.version>
</properties>
<dependencies>
<dependency>
@@ -57,11 +58,31 @@
<artifactId>postgresql</artifactId>
<scope>runtime</scope>
</dependency>
<!-- Testcontainers -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-testcontainers</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>postgresql</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.retry</groupId>
<artifactId>spring-retry</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>

2
src/main/java/com/pledge/torgi/MainApplication.java
View File

@@ -5,11 +5,13 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.retry.annotation.EnableRetry;
@SpringBootApplication
@EnableJpaRepositories
@EnableJpaAuditing
@EnableConfigurationProperties
@EnableRetry
public class MainApplication {
public static void main(String[] args) {
SpringApplication.run(MainApplication.class, args);

14
src/main/java/com/pledge/torgi/controller/DownloadHeadlessController.java
View File

@@ -33,10 +33,13 @@ public class DownloadHeadlessController {
private final Set<String> content = new HashSet<>();
private final TorgiEventRepository eventRepository;
private final MissedDataRepository missedDataRepository;
private final DownloadHeadlessService downloadHeadlessService;
public DownloadHeadlessController(TorgiEventRepository eventRepository, MissedDataRepository missedDataRepository) {
public DownloadHeadlessController(TorgiEventRepository eventRepository, MissedDataRepository missedDataRepository,
DownloadHeadlessService downloadHeadlessService) {
this.eventRepository = eventRepository;
this.missedDataRepository = missedDataRepository;
this.downloadHeadlessService = downloadHeadlessService;
eventRepository.findAllByOrderByCreatedAtDesc(PageRequest.of(0, 10000)).forEach(event -> content.add(event.getKey()));
}
@@ -46,8 +49,7 @@ public class DownloadHeadlessController {
private void addNewToDiff() {
businessLogger.info("startProcessing");
DownloadHeadlessService downloadHeadlessService = new DownloadHeadlessService();
Map<String, Lot> currentContent = downloadHeadlessService.downloadExcel(0, EXCEL_LINK);
Map<String, Lot> currentContent = downloadHeadlessService.downloadExcel(EXCEL_LINK);
List<TorgiEvent> newEvents = new ArrayList<>();
for (String key : currentContent.keySet()) {
@@ -58,11 +60,10 @@ public class DownloadHeadlessController {
}
}
eventRepository.saveAll(newEvents);
businessLogger.info("completeProcessing");
businessLogger.info("completeProcessing readed from torgi = {}, added to DB = {}", currentContent.size(), newEvents.size());
}
public void sendNewContent(StartParams params, List<String> cadastral, String email) {
DownloadHeadlessService downloadHeadlessService = new DownloadHeadlessService();
LocalDate now = LocalDate.now();
downloadHeadlessService.createNewDocument(eventRepository, cadastral, now);
createArchive(now);
@@ -142,9 +143,8 @@ public class DownloadHeadlessController {
public void richData() {
try {
List<MissedData> all = missedDataRepository.findAll(PageRequest.of(0, 10)).toList();
DownloadHeadlessService downloadHeadlessService = new DownloadHeadlessService();
for (MissedData missedCadastral : all) {
Map<String, Lot> currentContent = downloadHeadlessService.downloadExcel(0, String.format(CUSTOM_EXCEL_LINK, missedCadastral.getCadastral()));
Map<String, Lot> currentContent = downloadHeadlessService.downloadExcel(String.format(CUSTOM_EXCEL_LINK, missedCadastral.getCadastral()));
businessLogger.info("get content by cadastral {} {}", missedCadastral.getCadastral(), currentContent.size());
if (!currentContent.isEmpty()) {
List<TorgiEvent> newEvents = new ArrayList<>();

55
src/main/java/com/pledge/torgi/service/DownloadHeadlessService.java
View File

@@ -10,9 +10,15 @@ import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.retry.annotation.Backoff;
import org.springframework.retry.annotation.Recover;
import org.springframework.retry.annotation.Retryable;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.FileSystems;
@@ -23,6 +29,7 @@ import java.util.*;
import static com.pledge.torgi.utils.AppUtils.EMPTY;
import static java.util.Objects.nonNull;
@Service
public class DownloadHeadlessService {
private final Logger logger = LoggerFactory.getLogger(DownloadHeadlessService.class);
@@ -40,19 +47,30 @@ public class DownloadHeadlessService {
private static final int COL_FINAL_PRICE = 27;
private static final int COL_FULL_INFO = 29;
public Map<String, Lot> downloadExcel(int retryCount, String link) {
@Retryable(retryFor = UncheckedIOException.class, maxAttempts = 11,
backoff = @Backoff(delay = 1000))
public Map<String, Lot> downloadExcel(String link) {
HttpURLConnection connection;
int responseCode;
try {
HttpURLConnection connection = (HttpURLConnection) new URL(link).openConnection();
connection = (HttpURLConnection) new URL(link).openConnection();
connection.setRequestMethod("GET");
connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
connection.setRequestProperty("Accept", "application/json, text/plain, */*");
connection.setRequestProperty("Accept-Language", "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7");
connection.setConnectTimeout(5000);
connection.setReadTimeout(10000);
int responseCode = connection.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK) {
responseCode = connection.getResponseCode();
} catch (IOException e) {
logger.error("Error during processing link {} ", link, e);
throw new UncheckedIOException(e);
}
if (responseCode != HttpURLConnection.HTTP_OK) {
logger.error("Wrong response code {} during call link {} ", responseCode, link);
throw new UncheckedIOException(new IOException("Wrong response code " + responseCode + " during call link " + link));
}
try (Workbook wb = new XSSFWorkbook(connection.getInputStream())) {
Map<String, Lot> data = new HashMap<>();
Workbook wb = new XSSFWorkbook(connection.getInputStream());
Sheet sheetWithData = wb.getSheetAt(0);
int currentRow = 2;
while (sheetWithData.getRow(currentRow) != null) {
@@ -70,25 +88,20 @@ public class DownloadHeadlessService {
currentRow++;
}
return data;
} else {
logger.error("Wrong response code {} during call link {} ", responseCode, link);
Thread.sleep(1000);
return downloadExcel(retryCount + 1, link);
}
} catch (Exception e) {
} catch (IOException e) {
logger.error("Error during processing link {} ", link, e);
if (retryCount > 10) {
throw new UncheckedIOException(e);
} catch (Exception e) {
logger.error("Error during parsing content by link {} ", link, e);
throw new UncheckedIOException(new IOException("Failed to parse content by link " + link, e));
}
}
@Recover
public Map<String, Lot> downloadExcelFallback(Exception e, String link) {
logger.error("Failed to download excel by link {} after retries", link, e);
return new HashMap<>();
}
}
try {
Thread.sleep(1000);
return downloadExcel(retryCount + 1, link);
} catch (Exception ex) {
logger.error("Exception during retry downloadExcel by link {}", link, ex);
return new HashMap<>();
}
}
private Lot extractDataFromRow(Row row, String cadastral) {
Lot lot = new Lot();

34
src/test/java/com/pledge/torgi/DatabaseIntegrationTest.java Normal file
View File

@@ -0,0 +1,34 @@
package com.pledge.torgi;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.DynamicPropertyRegistry;
import org.springframework.test.context.DynamicPropertySource;
import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import static org.assertj.core.api.Assertions.assertThat;
@SpringBootTest
@Testcontainers
class DatabaseIntegrationTest {
@Container
static PostgreSQLContainer<?> postgres = new PostgreSQLContainer<>("postgres:15-alpine")
.withDatabaseName("torgi_test")
.withUsername("postgres")
.withPassword("postgres");
@DynamicPropertySource
static void configureProperties(DynamicPropertyRegistry registry) {
registry.add("spring.datasource.url", postgres::getJdbcUrl);
registry.add("spring.datasource.username", postgres::getUsername);
registry.add("spring.datasource.password", postgres::getPassword);
}
@Test
void connectionEstablished() {
assertThat(postgres.isRunning()).isTrue();
}
}

77
src/test/java/com/pledge/torgi/controller/DownloadHeadlessControllerTest.java
View File

@@ -56,6 +56,9 @@ class DownloadHeadlessControllerTest {
@Mock
private MissedDataRepository missedDataRepository;
@Mock
private DownloadHeadlessService downloadHeadlessService;
@BeforeEach
void setUp() {
lenient().when(eventRepository.findAllByOrderByCreatedAtDesc(any(PageRequest.class)))
@@ -73,7 +76,7 @@ class DownloadHeadlessControllerTest {
.thenReturn(page);
// when
new DownloadHeadlessController(eventRepository, missedDataRepository);
new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
// then
verify(eventRepository).findAllByOrderByCreatedAtDesc(any(PageRequest.class));
@@ -90,14 +93,14 @@ class DownloadHeadlessControllerTest {
.thenReturn(new PageImpl<>(List.of(event1, event2)));
// when
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
// then
assertThat(getContent(controller)).containsExactlyInAnyOrder("key-1", "key-2");
}
@Test
void process_shouldSaveNewEvents() {
void process_shouldSaveNewEvents() throws Exception {
// given
Lot lot = new Lot();
lot.setStatus("PUBLISHED");
@@ -106,10 +109,8 @@ class DownloadHeadlessControllerTest {
lot.setCadastral(CADASTRAL);
lot.setFullInfo("Full info");
Map<String, Lot> downloaded = Map.of(EVENT_KEY, lot);
try (MockedConstruction<DownloadHeadlessService> ignored = mockConstruction(DownloadHeadlessService.class,
(mock, context) -> when(mock.downloadExcel(anyInt(), anyString())).thenReturn(downloaded))) {
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
when(downloadHeadlessService.downloadExcel(anyString())).thenReturn(downloaded);
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
// when
controller.process();
@@ -127,20 +128,17 @@ class DownloadHeadlessControllerTest {
assertThat(saved.get(0).getFullInfo()).isEqualTo("Full info");
assertThat(getContent(controller)).contains(EVENT_KEY);
}
}
@Test
void process_shouldSkipAlreadyKnownKeys() {
void process_shouldSkipAlreadyKnownKeys() throws Exception {
// given
TorgiEvent existing = new TorgiEvent();
existing.setKey(EVENT_KEY);
when(eventRepository.findAllByOrderByCreatedAtDesc(any(PageRequest.class)))
.thenReturn(new PageImpl<>(List.of(existing)));
Map<String, Lot> downloaded = Map.of(EVENT_KEY, new Lot());
try (MockedConstruction<DownloadHeadlessService> ignored = mockConstruction(DownloadHeadlessService.class,
(mock, context) -> when(mock.downloadExcel(anyInt(), anyString())).thenReturn(downloaded))) {
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
when(downloadHeadlessService.downloadExcel(anyString())).thenReturn(downloaded);
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
// when
controller.process();
@@ -151,7 +149,6 @@ class DownloadHeadlessControllerTest {
assertThat(captor.getValue()).isEmpty();
assertThat(getContent(controller)).containsExactly(EVENT_KEY);
}
}
@Test
void sendNewContent_shouldCreateDocumentSendArchiveAndCleanUp() throws Exception {
@@ -161,17 +158,15 @@ class DownloadHeadlessControllerTest {
params.setToken("token");
List<String> cadastral = List.of(CADASTRAL);
try (MockedConstruction<DownloadHeadlessService> serviceConstruction = mockConstruction(DownloadHeadlessService.class);
MockedConstruction<SendEmailService> emailConstruction = mockConstruction(SendEmailService.class)) {
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
try (MockedConstruction<SendEmailService> emailConstruction = mockConstruction(SendEmailService.class)) {
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
// when
controller.sendNewContent(params, cadastral, "to@example.com");
// then
DownloadHeadlessService service = serviceConstruction.constructed().get(0);
ArgumentCaptor<LocalDate> dateCaptor = ArgumentCaptor.forClass(LocalDate.class);
verify(service).createNewDocument(eq(eventRepository), eq(cadastral), dateCaptor.capture());
verify(downloadHeadlessService).createNewDocument(eq(eventRepository), eq(cadastral), dateCaptor.capture());
LocalDate now = dateCaptor.getValue();
SendEmailService emailService = emailConstruction.constructed().get(0);
@@ -191,7 +186,7 @@ class DownloadHeadlessControllerTest {
StartParams params = new StartParams();
params.setFrom("from@example.com");
params.setToken("token");
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
try (MockedConstruction<SendEmailService> ignored = mockConstruction(SendEmailService.class,
(mock, context) -> doThrow(new RuntimeException("smtp failure"))
@@ -206,7 +201,7 @@ class DownloadHeadlessControllerTest {
}
@Test
void richData_shouldSaveNewEventsForMissedCadastral() {
void richData_shouldSaveNewEventsForMissedCadastral() throws Exception {
// given
MissedData missed = new MissedData();
missed.setCadastral(CADASTRAL);
@@ -219,10 +214,8 @@ class DownloadHeadlessControllerTest {
lot.setLotNumber(1);
lot.setCadastral(CADASTRAL);
Map<String, Lot> downloaded = Map.of(EVENT_KEY, lot);
try (MockedConstruction<DownloadHeadlessService> ignored = mockConstruction(DownloadHeadlessService.class,
(mock, context) -> when(mock.downloadExcel(anyInt(), anyString())).thenReturn(downloaded))) {
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
when(downloadHeadlessService.downloadExcel(anyString())).thenReturn(downloaded);
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
// when
controller.richData();
@@ -234,10 +227,9 @@ class DownloadHeadlessControllerTest {
assertThat(captor.getValue().get(0).getKey()).isEqualTo(EVENT_KEY);
verify(missedDataRepository).delete(missed);
}
}
@Test
void richData_shouldSkipKnownEventsForCadastral() {
void richData_shouldSkipKnownEventsForCadastral() throws Exception {
// given
MissedData missed = new MissedData();
missed.setCadastral(CADASTRAL);
@@ -248,10 +240,8 @@ class DownloadHeadlessControllerTest {
when(eventRepository.findAllByFullInfoLikeOrderByCreatedAtDesc("%" + CADASTRAL + "%")).thenReturn(List.of(existing));
Map<String, Lot> downloaded = Map.of(EVENT_KEY, new Lot());
try (MockedConstruction<DownloadHeadlessService> ignored = mockConstruction(DownloadHeadlessService.class,
(mock, context) -> when(mock.downloadExcel(anyInt(), anyString())).thenReturn(downloaded))) {
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
when(downloadHeadlessService.downloadExcel(anyString())).thenReturn(downloaded);
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
// when
controller.richData();
@@ -262,18 +252,15 @@ class DownloadHeadlessControllerTest {
assertThat(captor.getValue()).isEmpty();
verify(missedDataRepository).delete(missed);
}
}
@Test
void richData_shouldDeleteMissedDataWhenDownloadedContentEmpty() {
void richData_shouldDeleteMissedDataWhenDownloadedContentEmpty() throws Exception {
// given
MissedData missed = new MissedData();
missed.setCadastral(CADASTRAL);
when(missedDataRepository.findAll(any(PageRequest.class))).thenReturn(new PageImpl<>(List.of(missed)));
try (MockedConstruction<DownloadHeadlessService> ignored = mockConstruction(DownloadHeadlessService.class,
(mock, context) -> when(mock.downloadExcel(anyInt(), anyString())).thenReturn(Collections.emptyMap()))) {
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
when(downloadHeadlessService.downloadExcel(anyString())).thenReturn(Collections.emptyMap());
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
// when
controller.richData();
@@ -282,10 +269,9 @@ class DownloadHeadlessControllerTest {
verify(eventRepository, never()).saveAll(anyList());
verify(missedDataRepository).delete(missed);
}
}
@Test
void richData_shouldAbortProcessingOnRepositoryError() {
void richData_shouldAbortProcessingOnRepositoryError() throws Exception {
// given
MissedData missed = new MissedData();
missed.setCadastral(CADASTRAL);
@@ -294,10 +280,8 @@ class DownloadHeadlessControllerTest {
.thenThrow(new RuntimeException("Database error"));
Map<String, Lot> downloaded = Map.of(EVENT_KEY, new Lot());
try (MockedConstruction<DownloadHeadlessService> ignored = mockConstruction(DownloadHeadlessService.class,
(mock, context) -> when(mock.downloadExcel(anyInt(), anyString())).thenReturn(downloaded))) {
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
when(downloadHeadlessService.downloadExcel(anyString())).thenReturn(downloaded);
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
// when
controller.richData();
@@ -306,13 +290,12 @@ class DownloadHeadlessControllerTest {
verify(eventRepository, never()).saveAll(anyList());
verify(missedDataRepository, never()).delete(any());
}
}
@Test
void richData_shouldHandleExceptionGracefully() {
// given
when(missedDataRepository.findAll(any(PageRequest.class))).thenThrow(new RuntimeException("Database error"));
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
// when / then
assertThatCode(controller::richData).doesNotThrowAnyException();
@@ -325,7 +308,7 @@ class DownloadHeadlessControllerTest {
File deeper = new File(nested, "deeper");
assertThat(deeper.mkdirs()).isTrue();
writeFile(new File(deeper, "file.txt"), "content");
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
// when
boolean result = invokeDeleteDirectory(controller, nested);
@@ -388,7 +371,7 @@ class DownloadHeadlessControllerTest {
File nested = new File(sourceDir, "nested");
assertThat(nested.mkdirs()).isTrue();
writeFile(new File(nested, "data.txt"), "archive-content");
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository);
DownloadHeadlessController controller = new DownloadHeadlessController(eventRepository, missedDataRepository, downloadHeadlessService);
try {
// when