diff --git a/src/main/java/com/pledge/torgi/controller/DownloadHeadlessController.java b/src/main/java/com/pledge/torgi/controller/DownloadHeadlessController.java index 22d046f..11ea631 100644 --- a/src/main/java/com/pledge/torgi/controller/DownloadHeadlessController.java +++ b/src/main/java/com/pledge/torgi/controller/DownloadHeadlessController.java @@ -1,13 +1,16 @@ package com.pledge.torgi.controller; import com.pledge.torgi.model.Configs; +import com.pledge.torgi.model.MissedData; import com.pledge.torgi.model.TorgiEvent; import com.pledge.torgi.repositories.ConfigsRepository; +import com.pledge.torgi.repositories.MissedDataRepository; import com.pledge.torgi.repositories.TorgiEventRepository; import com.pledge.torgi.service.DownloadHeadlessService; -import com.pledge.torgi.service.SendEmailService; import com.pledge.torgi.service.Lot; +import com.pledge.torgi.service.SendEmailService; import com.pledge.torgi.utils.StartParams; +import lombok.SneakyThrows; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.data.domain.PageRequest; @@ -25,6 +28,8 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import static com.pledge.torgi.model.Configs.LAST_RUN_TIME; +import static com.pledge.torgi.utils.AppUtils.CUSTOM_EXCEL_LINK; +import static com.pledge.torgi.utils.AppUtils.EXCEL_LINK; @Service public class DownloadHeadlessController { @@ -33,10 +38,12 @@ public class DownloadHeadlessController { private final Set content = new HashSet<>(); private final TorgiEventRepository eventRepository; private final ConfigsRepository configsRepository; + private final MissedDataRepository missedDataRepository; - public DownloadHeadlessController(TorgiEventRepository eventRepository, ConfigsRepository configsRepository) { + public DownloadHeadlessController(TorgiEventRepository eventRepository, ConfigsRepository configsRepository, MissedDataRepository missedDataRepository) { this.eventRepository = eventRepository; this.configsRepository = configsRepository; + this.missedDataRepository = missedDataRepository; eventRepository.findAllByOrderByCreatedAtDesc(PageRequest.of(0, 10000)).forEach(event -> content.add(event.getKey())); } @@ -47,7 +54,7 @@ public class DownloadHeadlessController { private void addNewToDiff() { logger.error("startProcessing"); DownloadHeadlessService downloadHeadlessService = new DownloadHeadlessService(); - Map currentContent = downloadHeadlessService.downloadExcel(0); + Map currentContent = downloadHeadlessService.downloadExcel(0, EXCEL_LINK); List newEvents = new ArrayList<>(); for (String key : currentContent.keySet()) { @@ -102,8 +109,7 @@ public class DownloadHeadlessController { newValue.setKey(LAST_RUN_TIME); newValue.setValue(LocalDateTime.now().toString()); configsRepository.save(newValue); - } - else { + } else { Configs oldValue = configs.get(0); oldValue.setValue(LocalDateTime.now().toString()); configsRepository.save(oldValue); @@ -164,8 +170,8 @@ public class DownloadHeadlessController { String sourceFile = now.toString(); try (FileOutputStream fos = new FileOutputStream(now + ".zip"); ZipOutputStream zipOut = new ZipOutputStream(fos)) { - File fileToZip = new File(sourceFile); - zipFile(fileToZip, fileToZip.getName(), zipOut); + File fileToZip = new File(sourceFile); + zipFile(fileToZip, fileToZip.getName(), zipOut); } catch (Exception ex) { logger.error("in createArchive" + ex.getMessage()); } @@ -199,4 +205,30 @@ public class DownloadHeadlessController { } fis.close(); } + + @SneakyThrows + public void richData() { + List all = missedDataRepository.findAll(PageRequest.of(0, 10)).toList(); + DownloadHeadlessService downloadHeadlessService = new DownloadHeadlessService(); + for (MissedData missedCadastral : all) { + Map currentContent = downloadHeadlessService.downloadExcel(0, String.format(CUSTOM_EXCEL_LINK, missedCadastral.getCadastral())); + logger.error("get content by cadastral {} {}", missedCadastral.getCadastral(), currentContent.size()); + if (!currentContent.isEmpty()) { + List newEvents = new ArrayList<>(); + Set contentByCadastral = new HashSet<>(); + eventRepository.findAllByFullInfoLikeOrderByCreatedAtDesc("%" + missedCadastral.getCadastral() + "%").forEach(event -> contentByCadastral.add(event.getKey())); + + for (String key : currentContent.keySet()) { + if (!contentByCadastral.contains(key)) { + newEvents.add(new TorgiEvent(key, currentContent.get(key))); + content.add(key); + logger.error("added new missed content"); + } + } + eventRepository.saveAll(newEvents); + } + missedDataRepository.delete(missedCadastral); + Thread.sleep(5000); + } + } } diff --git a/src/main/java/com/pledge/torgi/controller/storage/FileServiceImpl.java b/src/main/java/com/pledge/torgi/controller/storage/FileServiceImpl.java index 49ba83b..4195701 100644 --- a/src/main/java/com/pledge/torgi/controller/storage/FileServiceImpl.java +++ b/src/main/java/com/pledge/torgi/controller/storage/FileServiceImpl.java @@ -1,6 +1,8 @@ package com.pledge.torgi.controller.storage; import com.pledge.torgi.controller.DownloadHeadlessController; +import com.pledge.torgi.model.MissedData; +import com.pledge.torgi.repositories.MissedDataRepository; import com.pledge.torgi.utils.StartParams; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -14,7 +16,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; -import java.io.InputStream; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; @@ -25,6 +27,7 @@ public class FileServiceImpl implements FileService { private final DownloadHeadlessController headlessController; private final StartParams params; + private final MissedDataRepository missedDataRepository; @Override public String processAsync(MultipartFile file, String email) { @@ -53,11 +56,23 @@ public class FileServiceImpl implements FileService { log.error("Extract cadastrals: " + cadastrals.size()); headlessController.sendNewContent(params, cadastrals, email); - + saveCadastralToGetMissedData(cadastrals); log.error("Complete request"); return "Обработано " + cadastrals.size() + " кадастровых номеров"; } + private void saveCadastralToGetMissedData(List cadastrals) { + List all = missedDataRepository.findAll(); + List allExistedCadastrals = all.stream().map(MissedData::getCadastral).toList(); + LocalDateTime now = LocalDateTime.now(); + missedDataRepository.saveAll(cadastrals.stream().filter(cad -> !allExistedCadastrals.contains(cad)).map(cad -> { + MissedData md = new MissedData(); + md.setCadastral(cad); + md.setCreatedTime(now); + return md; + }).toList()); + } + private List extractCadastral(Workbook wb) { List result = new ArrayList<>(); Sheet firstSheet = wb.getSheetAt(0); diff --git a/src/main/java/com/pledge/torgi/model/MissedData.java b/src/main/java/com/pledge/torgi/model/MissedData.java new file mode 100644 index 0000000..05170df --- /dev/null +++ b/src/main/java/com/pledge/torgi/model/MissedData.java @@ -0,0 +1,27 @@ +package com.pledge.torgi.model; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import java.time.LocalDateTime; + +@Entity +@Getter +@Setter +@EntityListeners(AuditingEntityListener.class) +@AllArgsConstructor +@NoArgsConstructor +public class MissedData { + + @Id + @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "MissedDataId") + @SequenceGenerator(name = "MissedDataId", sequenceName = "MISSED_DATA_SEQ", allocationSize = 1) + private Long id; + + private String cadastral; + private LocalDateTime createdTime; +} diff --git a/src/main/java/com/pledge/torgi/repositories/MissedDataRepository.java b/src/main/java/com/pledge/torgi/repositories/MissedDataRepository.java new file mode 100644 index 0000000..370d45b --- /dev/null +++ b/src/main/java/com/pledge/torgi/repositories/MissedDataRepository.java @@ -0,0 +1,7 @@ +package com.pledge.torgi.repositories; + +import com.pledge.torgi.model.MissedData; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface MissedDataRepository extends JpaRepository { +} diff --git a/src/main/java/com/pledge/torgi/service/DownloadHeadlessService.java b/src/main/java/com/pledge/torgi/service/DownloadHeadlessService.java index 86074c9..377b40d 100644 --- a/src/main/java/com/pledge/torgi/service/DownloadHeadlessService.java +++ b/src/main/java/com/pledge/torgi/service/DownloadHeadlessService.java @@ -29,7 +29,6 @@ import java.util.*; import java.util.regex.Pattern; import static com.pledge.torgi.utils.AppUtils.EMPTY; -import static com.pledge.torgi.utils.AppUtils.EXCEL_LINK; import static java.util.Objects.nonNull; public class DownloadHeadlessService { @@ -40,9 +39,9 @@ public class DownloadHeadlessService { Pattern vinPattern = Pattern.compile("\\b[A-HJ-NPR-Z0-9-]{12,17}\\b"); @SneakyThrows - public Map downloadExcel(int retryCount) { + public Map downloadExcel(int retryCount, String link) { try { - HttpURLConnection connection = (HttpURLConnection) new URL(EXCEL_LINK).openConnection(); + HttpURLConnection connection = (HttpURLConnection) new URL(link).openConnection(); connection.setRequestMethod("GET"); if (connection.getResponseCode() == HttpURLConnection.HTTP_OK) { Map data = new HashMap<>(); @@ -53,11 +52,11 @@ public class DownloadHeadlessService { List cadastrals = extractCadastral(sheetWithData.getRow(currentRow).getCell(29).getStringCellValue()); if (cadastrals.isEmpty()) { Lot extractedContent = extractDataFromRow(sheetWithData.getRow(currentRow), ""); - data.put(extractedContent.getNotice()+ "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus(), extractedContent); + data.put(extractedContent.getNotice() + "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus(), extractedContent); } else { for (String cadastral : cadastrals) { Lot extractedContent = extractDataFromRow(sheetWithData.getRow(currentRow), cadastral); - data.put(extractedContent.getNotice()+ "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus() + "_" + cadastral, extractedContent); + data.put(extractedContent.getNotice() + "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus() + "_" + cadastral, extractedContent); } } @@ -67,7 +66,7 @@ public class DownloadHeadlessService { } else { logger.error("Error in line 99 "); Thread.sleep(1000); - return downloadExcel(retryCount + 1); + return downloadExcel(retryCount + 1, link); } } catch (Exception e) { logger.error("Error in line 103 " + e); @@ -76,7 +75,7 @@ public class DownloadHeadlessService { } } Thread.sleep(1000); - return downloadExcel(retryCount + 1); + return downloadExcel(retryCount + 1, link); } private Lot extractDataFromRow(Row row, String cadastral) { @@ -84,7 +83,7 @@ public class DownloadHeadlessService { lot.setBidStart(row.getCell(6).getStringCellValue()); lot.setBidEnd(row.getCell(7).getStringCellValue()); lot.setCadastral(cadastral); - lot.setLotNumber((int)row.getCell(16).getNumericCellValue()); + lot.setLotNumber((int) row.getCell(16).getNumericCellValue()); lot.setFinalPrice(row.getCell(27).getStringCellValue()); lot.setNotice(row.getCell(3).getStringCellValue()); lot.setInitMoney(row.getCell(24).getStringCellValue()); @@ -189,7 +188,7 @@ public class DownloadHeadlessService { } private void setRowHead(XSSFSheet sheet) { - XSSFRow rowHead = sheet.createRow( 0); + XSSFRow rowHead = sheet.createRow(0); rowHead.createCell(0).setCellValue("Статус торгов"); rowHead.createCell(1).setCellValue("Дата начала подачи заявок"); rowHead.createCell(2).setCellValue("Дата окончания подачи заявок"); @@ -236,7 +235,7 @@ public class DownloadHeadlessService { private File getFileToSave(LocalDate now) { File dir = new File(now.toString()); - if(!dir.exists()) { + if (!dir.exists()) { dir.mkdir(); } return new File(dir.getAbsolutePath() + FileSystems.getDefault().getSeparator() + now + ".xlsx"); diff --git a/src/main/java/com/pledge/torgi/service/MainService.java b/src/main/java/com/pledge/torgi/service/MainService.java index e7b7aae..20c2f0e 100644 --- a/src/main/java/com/pledge/torgi/service/MainService.java +++ b/src/main/java/com/pledge/torgi/service/MainService.java @@ -1,16 +1,12 @@ package com.pledge.torgi.service; import com.pledge.torgi.controller.DownloadHeadlessController; -import com.pledge.torgi.controller.storage.FileService; import com.pledge.torgi.utils.StartParams; import lombok.AllArgsConstructor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; -import java.util.List; import java.util.concurrent.TimeUnit; @Service @@ -24,6 +20,7 @@ public class MainService { @Scheduled(timeUnit = TimeUnit.SECONDS, fixedRate = 120) public void getNewData() { headlessController.process(); + headlessController.richData(); } @Scheduled(timeUnit = TimeUnit.SECONDS, fixedRate = 600) diff --git a/src/main/java/com/pledge/torgi/utils/AppUtils.java b/src/main/java/com/pledge/torgi/utils/AppUtils.java index f6664ee..9bd4866 100644 --- a/src/main/java/com/pledge/torgi/utils/AppUtils.java +++ b/src/main/java/com/pledge/torgi/utils/AppUtils.java @@ -30,4 +30,6 @@ public final class AppUtils { public static final String SEARCH_URL = "https://torgi.gov.ru/new/api/public/lotcards/search?"; //example https://torgi.gov.ru/new/api/public/lotcards/search?pubFrom=2025-02-04&pubTo=2025-02-04&byFirstVersion=true&withFacets=true&page=0&size=10&sort=firstVersionPublicationDate,desc public static final Format DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); public static final String EXCEL_LINK = "https://torgi.gov.ru/new/api/public/lotcards/export/excel?byFirstVersion=true&sort=updateDate,desc"; + public static final String CUSTOM_EXCEL_LINK = "https://torgi.gov.ru/new/api/public/lotcards/export/excel?text=%s&matchPhrase=false&byFirstVersion=true&sort=firstVersionPublicationDate,desc"; + } diff --git a/src/main/resources/db/migration/V4__add_missed_data.sql b/src/main/resources/db/migration/V4__add_missed_data.sql new file mode 100644 index 0000000..fd28cdf --- /dev/null +++ b/src/main/resources/db/migration/V4__add_missed_data.sql @@ -0,0 +1,9 @@ +CREATE TABLE missed_data +( + id BIGINT NOT NULL, + cadastral VARCHAR(255), + created_time TIMESTAMP, + CONSTRAINT pk_missed_data PRIMARY KEY (id) +); + +CREATE SEQUENCE missed_data_seq; \ No newline at end of file