add processing for missed data

src/main/java/com/pledge/torgi/controller/DownloadHeadlessController.java

@@ -1,13 +1,16 @@
 package com.pledge.torgi.controller;
 
 import com.pledge.torgi.model.Configs;
+import com.pledge.torgi.model.MissedData;
 import com.pledge.torgi.model.TorgiEvent;
 import com.pledge.torgi.repositories.ConfigsRepository;
+import com.pledge.torgi.repositories.MissedDataRepository;
 import com.pledge.torgi.repositories.TorgiEventRepository;
 import com.pledge.torgi.service.DownloadHeadlessService;
-import com.pledge.torgi.service.SendEmailService;
 import com.pledge.torgi.service.Lot;
+import com.pledge.torgi.service.SendEmailService;
 import com.pledge.torgi.utils.StartParams;
+import lombok.SneakyThrows;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.data.domain.PageRequest;
@@ -25,6 +28,8 @@ import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
 import static com.pledge.torgi.model.Configs.LAST_RUN_TIME;
+import static com.pledge.torgi.utils.AppUtils.CUSTOM_EXCEL_LINK;
+import static com.pledge.torgi.utils.AppUtils.EXCEL_LINK;
 
 @Service
 public class DownloadHeadlessController {
@@ -33,10 +38,12 @@ public class DownloadHeadlessController {
     private final Set<String> content = new HashSet<>();
     private final TorgiEventRepository eventRepository;
     private final ConfigsRepository configsRepository;
+    private final MissedDataRepository missedDataRepository;
 
-    public DownloadHeadlessController(TorgiEventRepository eventRepository, ConfigsRepository configsRepository) {
+    public DownloadHeadlessController(TorgiEventRepository eventRepository, ConfigsRepository configsRepository, MissedDataRepository missedDataRepository) {
         this.eventRepository = eventRepository;
         this.configsRepository = configsRepository;
+        this.missedDataRepository = missedDataRepository;
         eventRepository.findAllByOrderByCreatedAtDesc(PageRequest.of(0, 10000)).forEach(event -> content.add(event.getKey()));
     }
 
@@ -47,7 +54,7 @@ public class DownloadHeadlessController {
     private void addNewToDiff() {
         logger.error("startProcessing");
         DownloadHeadlessService downloadHeadlessService = new DownloadHeadlessService();
-        Map<String, Lot> currentContent = downloadHeadlessService.downloadExcel(0);
+        Map<String, Lot> currentContent = downloadHeadlessService.downloadExcel(0, EXCEL_LINK);
         List<TorgiEvent> newEvents = new ArrayList<>();
 
         for (String key : currentContent.keySet()) {
@@ -102,8 +109,7 @@ public class DownloadHeadlessController {
             newValue.setKey(LAST_RUN_TIME);
             newValue.setValue(LocalDateTime.now().toString());
             configsRepository.save(newValue);
-        }
+        } else {
-        else {
             Configs oldValue = configs.get(0);
             oldValue.setValue(LocalDateTime.now().toString());
             configsRepository.save(oldValue);
@@ -164,8 +170,8 @@ public class DownloadHeadlessController {
         String sourceFile = now.toString();
         try (FileOutputStream fos = new FileOutputStream(now + ".zip");
              ZipOutputStream zipOut = new ZipOutputStream(fos)) {
-             File fileToZip = new File(sourceFile);
+            File fileToZip = new File(sourceFile);
-             zipFile(fileToZip, fileToZip.getName(), zipOut);
+            zipFile(fileToZip, fileToZip.getName(), zipOut);
         } catch (Exception ex) {
             logger.error("in createArchive" + ex.getMessage());
         }
@@ -199,4 +205,30 @@ public class DownloadHeadlessController {
         }
         fis.close();
     }
+
+    @SneakyThrows
+    public void richData() {
+        List<MissedData> all = missedDataRepository.findAll(PageRequest.of(0, 10)).toList();
+        DownloadHeadlessService downloadHeadlessService = new DownloadHeadlessService();
+        for (MissedData missedCadastral : all) {
+            Map<String, Lot> currentContent = downloadHeadlessService.downloadExcel(0, String.format(CUSTOM_EXCEL_LINK, missedCadastral.getCadastral()));
+            logger.error("get content by cadastral {} {}", missedCadastral.getCadastral(), currentContent.size());
+            if (!currentContent.isEmpty()) {
+                List<TorgiEvent> newEvents = new ArrayList<>();
+                Set<String> contentByCadastral = new HashSet<>();
+                eventRepository.findAllByFullInfoLikeOrderByCreatedAtDesc("%" + missedCadastral.getCadastral() + "%").forEach(event -> contentByCadastral.add(event.getKey()));
+
+                for (String key : currentContent.keySet()) {
+                    if (!contentByCadastral.contains(key)) {
+                        newEvents.add(new TorgiEvent(key, currentContent.get(key)));
+                        content.add(key);
+                        logger.error("added new missed content");
+                    }
+                }
+                eventRepository.saveAll(newEvents);
+            }
+            missedDataRepository.delete(missedCadastral);
+            Thread.sleep(5000);
+        }
+    }
 }

src/main/java/com/pledge/torgi/controller/storage/FileServiceImpl.java

@@ -1,6 +1,8 @@
 package com.pledge.torgi.controller.storage;
 
 import com.pledge.torgi.controller.DownloadHeadlessController;
+import com.pledge.torgi.model.MissedData;
+import com.pledge.torgi.repositories.MissedDataRepository;
 import com.pledge.torgi.utils.StartParams;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -14,7 +16,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 
 import java.io.IOException;
-import java.io.InputStream;
+import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -25,6 +27,7 @@ public class FileServiceImpl implements FileService {
 
 	private final DownloadHeadlessController headlessController;
 	private final StartParams params;
+	private final MissedDataRepository missedDataRepository;
 
 	@Override
 	public String processAsync(MultipartFile file, String email) {
@@ -53,11 +56,23 @@ public class FileServiceImpl implements FileService {
 		log.error("Extract cadastrals: " + cadastrals.size());
 
 		headlessController.sendNewContent(params, cadastrals, email);
-
+		saveCadastralToGetMissedData(cadastrals);
 		log.error("Complete request");
 		return "Обработано " + cadastrals.size() + " кадастровых номеров";
 	}
 
+	private void saveCadastralToGetMissedData(List<String> cadastrals) {
+		List<MissedData> all = missedDataRepository.findAll();
+		List<String> allExistedCadastrals = all.stream().map(MissedData::getCadastral).toList();
+		LocalDateTime now = LocalDateTime.now();
+		missedDataRepository.saveAll(cadastrals.stream().filter(cad -> !allExistedCadastrals.contains(cad)).map(cad -> {
+			MissedData md = new MissedData();
+			md.setCadastral(cad);
+			md.setCreatedTime(now);
+			return md;
+		}).toList());
+	}
+
 	private List<String> extractCadastral(Workbook wb) {
 		List<String> result = new ArrayList<>();
 		Sheet firstSheet = wb.getSheetAt(0);

src/main/java/com/pledge/torgi/model/MissedData.java

@@ -0,0 +1,27 @@
+package com.pledge.torgi.model;
+
+import jakarta.persistence.*;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import org.springframework.data.jpa.domain.support.AuditingEntityListener;
+
+import java.time.LocalDateTime;
+
+@Entity
+@Getter
+@Setter
+@EntityListeners(AuditingEntityListener.class)
+@AllArgsConstructor
+@NoArgsConstructor
+public class MissedData {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "MissedDataId")
+    @SequenceGenerator(name = "MissedDataId", sequenceName = "MISSED_DATA_SEQ", allocationSize = 1)
+    private Long id;
+
+    private String cadastral;
+    private LocalDateTime createdTime;
+}

src/main/java/com/pledge/torgi/repositories/MissedDataRepository.java

@@ -0,0 +1,7 @@
+package com.pledge.torgi.repositories;
+
+import com.pledge.torgi.model.MissedData;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+public interface MissedDataRepository extends JpaRepository<MissedData, Long> {
+}

src/main/java/com/pledge/torgi/service/DownloadHeadlessService.java

@@ -29,7 +29,6 @@ import java.util.*;
 import java.util.regex.Pattern;
 
 import static com.pledge.torgi.utils.AppUtils.EMPTY;
-import static com.pledge.torgi.utils.AppUtils.EXCEL_LINK;
 import static java.util.Objects.nonNull;
 
 public class DownloadHeadlessService {
@@ -40,9 +39,9 @@ public class DownloadHeadlessService {
     Pattern vinPattern = Pattern.compile("\\b[A-HJ-NPR-Z0-9-]{12,17}\\b");
 
     @SneakyThrows
-    public Map<String, Lot> downloadExcel(int retryCount) {
+    public Map<String, Lot> downloadExcel(int retryCount, String link) {
         try {
-            HttpURLConnection connection = (HttpURLConnection) new URL(EXCEL_LINK).openConnection();
+            HttpURLConnection connection = (HttpURLConnection) new URL(link).openConnection();
             connection.setRequestMethod("GET");
             if (connection.getResponseCode() == HttpURLConnection.HTTP_OK) {
                 Map<String, Lot> data = new HashMap<>();
@@ -53,11 +52,11 @@ public class DownloadHeadlessService {
                     List<String> cadastrals = extractCadastral(sheetWithData.getRow(currentRow).getCell(29).getStringCellValue());
                     if (cadastrals.isEmpty()) {
                         Lot extractedContent = extractDataFromRow(sheetWithData.getRow(currentRow), "");
-                        data.put(extractedContent.getNotice()+ "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus(), extractedContent);
+                        data.put(extractedContent.getNotice() + "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus(), extractedContent);
                     } else {
                         for (String cadastral : cadastrals) {
                             Lot extractedContent = extractDataFromRow(sheetWithData.getRow(currentRow), cadastral);
-                            data.put(extractedContent.getNotice()+ "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus() + "_" + cadastral, extractedContent);
+                            data.put(extractedContent.getNotice() + "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus() + "_" + cadastral, extractedContent);
                         }
                     }
 
@@ -67,7 +66,7 @@ public class DownloadHeadlessService {
             } else {
                 logger.error("Error in line 99 ");
                 Thread.sleep(1000);
-                return downloadExcel(retryCount + 1);
+                return downloadExcel(retryCount + 1, link);
             }
         } catch (Exception e) {
             logger.error("Error in line 103 " + e);
@@ -76,7 +75,7 @@ public class DownloadHeadlessService {
             }
         }
         Thread.sleep(1000);
-        return downloadExcel(retryCount + 1);
+        return downloadExcel(retryCount + 1, link);
     }
 
     private Lot extractDataFromRow(Row row, String cadastral) {
@@ -84,7 +83,7 @@ public class DownloadHeadlessService {
         lot.setBidStart(row.getCell(6).getStringCellValue());
         lot.setBidEnd(row.getCell(7).getStringCellValue());
         lot.setCadastral(cadastral);
-        lot.setLotNumber((int)row.getCell(16).getNumericCellValue());
+        lot.setLotNumber((int) row.getCell(16).getNumericCellValue());
         lot.setFinalPrice(row.getCell(27).getStringCellValue());
         lot.setNotice(row.getCell(3).getStringCellValue());
         lot.setInitMoney(row.getCell(24).getStringCellValue());
@@ -189,7 +188,7 @@ public class DownloadHeadlessService {
     }
 
     private void setRowHead(XSSFSheet sheet) {
-        XSSFRow rowHead = sheet.createRow( 0);
+        XSSFRow rowHead = sheet.createRow(0);
         rowHead.createCell(0).setCellValue("Статус торгов");
         rowHead.createCell(1).setCellValue("Дата начала подачи заявок");
         rowHead.createCell(2).setCellValue("Дата окончания подачи заявок");
@@ -236,7 +235,7 @@ public class DownloadHeadlessService {
 
     private File getFileToSave(LocalDate now) {
         File dir = new File(now.toString());
-        if(!dir.exists()) {
+        if (!dir.exists()) {
             dir.mkdir();
         }
         return new File(dir.getAbsolutePath() + FileSystems.getDefault().getSeparator() + now + ".xlsx");

src/main/java/com/pledge/torgi/service/MainService.java

@@ -1,16 +1,12 @@
 package com.pledge.torgi.service;
 
 import com.pledge.torgi.controller.DownloadHeadlessController;
-import com.pledge.torgi.controller.storage.FileService;
 import com.pledge.torgi.utils.StartParams;
 import lombok.AllArgsConstructor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 
-import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 @Service
@@ -24,6 +20,7 @@ public class MainService {
     @Scheduled(timeUnit = TimeUnit.SECONDS, fixedRate = 120)
     public void getNewData() {
         headlessController.process();
+        headlessController.richData();
     }
 
     @Scheduled(timeUnit = TimeUnit.SECONDS, fixedRate = 600)

src/main/java/com/pledge/torgi/utils/AppUtils.java

@@ -30,4 +30,6 @@ public final class AppUtils {
     public static final String SEARCH_URL = "https://torgi.gov.ru/new/api/public/lotcards/search?"; //example https://torgi.gov.ru/new/api/public/lotcards/search?pubFrom=2025-02-04&pubTo=2025-02-04&byFirstVersion=true&withFacets=true&page=0&size=10&sort=firstVersionPublicationDate,desc
     public static final Format DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     public static final String EXCEL_LINK = "https://torgi.gov.ru/new/api/public/lotcards/export/excel?byFirstVersion=true&sort=updateDate,desc";
+    public static final String CUSTOM_EXCEL_LINK = "https://torgi.gov.ru/new/api/public/lotcards/export/excel?text=%s&matchPhrase=false&byFirstVersion=true&sort=firstVersionPublicationDate,desc";
+
 }

src/main/resources/db/migration/V4__add_missed_data.sql

@@ -0,0 +1,9 @@
+CREATE TABLE missed_data
+(
+    id                      BIGINT NOT NULL,
+    cadastral               VARCHAR(255),
+    created_time            TIMESTAMP,
+    CONSTRAINT pk_missed_data PRIMARY KEY (id)
+);
+
+CREATE SEQUENCE missed_data_seq;