From 9ce7dfee9911f638213a03d38fb637b420536171 Mon Sep 17 00:00:00 2001 From: RM Date: Sat, 15 Nov 2025 15:39:03 +0300 Subject: [PATCH] fix torgi.gov infinite failing --- .../DownloadHeadlessController.java | 2 +- .../service/DownloadHeadlessService.java | 13 ++++++--- .../pledge/torgi/service/EmailService.java | 7 +++++ .../torgi/service/ReceiveEmailService.java | 27 ++++++++++--------- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/pledge/torgi/controller/DownloadHeadlessController.java b/src/main/java/com/pledge/torgi/controller/DownloadHeadlessController.java index 8aed7d1..22d046f 100644 --- a/src/main/java/com/pledge/torgi/controller/DownloadHeadlessController.java +++ b/src/main/java/com/pledge/torgi/controller/DownloadHeadlessController.java @@ -47,7 +47,7 @@ public class DownloadHeadlessController { private void addNewToDiff() { logger.error("startProcessing"); DownloadHeadlessService downloadHeadlessService = new DownloadHeadlessService(); - Map currentContent = downloadHeadlessService.downloadExcel(); + Map currentContent = downloadHeadlessService.downloadExcel(0); List newEvents = new ArrayList<>(); for (String key : currentContent.keySet()) { diff --git a/src/main/java/com/pledge/torgi/service/DownloadHeadlessService.java b/src/main/java/com/pledge/torgi/service/DownloadHeadlessService.java index 2bce514..86074c9 100644 --- a/src/main/java/com/pledge/torgi/service/DownloadHeadlessService.java +++ b/src/main/java/com/pledge/torgi/service/DownloadHeadlessService.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.pledge.torgi.model.TorgiEvent; import com.pledge.torgi.repositories.TorgiEventRepository; +import lombok.SneakyThrows; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Workbook; @@ -38,7 +39,8 @@ public class DownloadHeadlessService { Pattern cadastralPattern = Pattern.compile("\\d{1,2}:\\d{1,2}:\\d{1,7}:\\d{1,9}"); Pattern vinPattern = Pattern.compile("\\b[A-HJ-NPR-Z0-9-]{12,17}\\b"); - public Map downloadExcel() { + @SneakyThrows + public Map downloadExcel(int retryCount) { try { HttpURLConnection connection = (HttpURLConnection) new URL(EXCEL_LINK).openConnection(); connection.setRequestMethod("GET"); @@ -64,12 +66,17 @@ public class DownloadHeadlessService { return data; } else { logger.error("Error in line 99 "); - return downloadExcel(); + Thread.sleep(1000); + return downloadExcel(retryCount + 1); } } catch (Exception e) { logger.error("Error in line 103 " + e); + if (retryCount > 10) { + return new HashMap<>(); + } } - return downloadExcel(); + Thread.sleep(1000); + return downloadExcel(retryCount + 1); } private Lot extractDataFromRow(Row row, String cadastral) { diff --git a/src/main/java/com/pledge/torgi/service/EmailService.java b/src/main/java/com/pledge/torgi/service/EmailService.java index d8eaee0..5333461 100644 --- a/src/main/java/com/pledge/torgi/service/EmailService.java +++ b/src/main/java/com/pledge/torgi/service/EmailService.java @@ -5,11 +5,13 @@ import com.pledge.torgi.model.Emails; import com.pledge.torgi.repositories.EmailsRepository; import com.pledge.torgi.utils.StartParams; import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import java.time.LocalDateTime; import java.util.List; +@Slf4j @Service @AllArgsConstructor public class EmailService { @@ -18,16 +20,21 @@ public class EmailService { private final EmailsRepository emailsRepository; public void processEmails(StartParams params) { + log.error("Start email processing"); List alreadyProcessed = emailsRepository.findTop10ByOrderByIdDesc(); List emailData = new ReceiveEmailService("imap.gmail.com", 993, params.getFrom(), params.getToken(), params.getSpecialEmail()) .downloadEmails(alreadyProcessed); + int sendLetters = 0; + log.error("Extracted letters = {}", emailData.size()); for (EmailData data : emailData) { if (alreadyProcessed(data.getEmailId())) { continue; } fileService.processSync(data.getFile(), data.getEmails()); saveProcessedEmail(data); + sendLetters++; } + log.error("End email processing, send letters = {}", sendLetters); } private boolean alreadyProcessed(String emailId) { diff --git a/src/main/java/com/pledge/torgi/service/ReceiveEmailService.java b/src/main/java/com/pledge/torgi/service/ReceiveEmailService.java index 2877461..8f4f161 100644 --- a/src/main/java/com/pledge/torgi/service/ReceiveEmailService.java +++ b/src/main/java/com/pledge/torgi/service/ReceiveEmailService.java @@ -49,19 +49,22 @@ public class ReceiveEmailService { Message[] messages = folderInbox.getMessages(messageCount - 50, messageCount); for (int i = 0; i < messages.length; i++) { - Message msg = messages[i]; - Address[] toList = msg.getRecipients(Message.RecipientType.TO); - boolean specialAddress = Arrays.stream(toList).anyMatch(addr -> ((InternetAddress) addr).getAddress().equals(specialEmail)); - if (!specialAddress) { - continue; - } - - String messageID = ((IMAPMessage) msg).getMessageID(); - if (alreadyProcessed.stream().anyMatch(processed -> processed.getMailId().equals(messageID))) { - continue; - } - try { + Message msg = messages[i]; + Address[] toList = msg.getRecipients(Message.RecipientType.TO); + if (toList == null) { + continue; + } + boolean specialAddress = Arrays.stream(toList).anyMatch(addr -> ((InternetAddress) addr).getAddress().equals(specialEmail)); + if (!specialAddress) { + continue; + } + + String messageID = ((IMAPMessage) msg).getMessageID(); + if (alreadyProcessed.stream().anyMatch(processed -> processed.getMailId().equals(messageID))) { + continue; + } + MimeMultipart content = (MimeMultipart) msg.getDataHandler().getContent(); if (content.getCount() == 1 || !Part.ATTACHMENT.equalsIgnoreCase(content.getBodyPart(1).getDisposition())) { continue;