RM/torgi2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add processing for missed data

Browse Source
This commit is contained in:
RM
2026-06-16 21:19:21 +03:00
parent d4acc6ae5b
commit 5e344c60df
8 changed files with 111 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/main/java/com/pledge/torgi/controller/DownloadHeadlessController.java
View File

@@ -1,13 +1,16 @@
package com.pledge.torgi.controller;
import com.pledge.torgi.model.Configs;
import com.pledge.torgi.model.MissedData;
import com.pledge.torgi.model.TorgiEvent;
import com.pledge.torgi.repositories.ConfigsRepository;
import com.pledge.torgi.repositories.MissedDataRepository;
import com.pledge.torgi.repositories.TorgiEventRepository;
import com.pledge.torgi.service.DownloadHeadlessService;
import com.pledge.torgi.service.SendEmailService;
import com.pledge.torgi.service.Lot;
import com.pledge.torgi.service.SendEmailService;
import com.pledge.torgi.utils.StartParams;
import lombok.SneakyThrows;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.domain.PageRequest;
@@ -25,6 +28,8 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import static com.pledge.torgi.model.Configs.LAST_RUN_TIME;
import static com.pledge.torgi.utils.AppUtils.CUSTOM_EXCEL_LINK;
import static com.pledge.torgi.utils.AppUtils.EXCEL_LINK;
@Service
public class DownloadHeadlessController {
@@ -33,10 +38,12 @@ public class DownloadHeadlessController {
private final Set<String> content = new HashSet<>();
private final TorgiEventRepository eventRepository;
private final ConfigsRepository configsRepository;
private final MissedDataRepository missedDataRepository;
public DownloadHeadlessController(TorgiEventRepository eventRepository, ConfigsRepository configsRepository) {
public DownloadHeadlessController(TorgiEventRepository eventRepository, ConfigsRepository configsRepository, MissedDataRepository missedDataRepository) {
this.eventRepository = eventRepository;
this.configsRepository = configsRepository;
this.missedDataRepository = missedDataRepository;
eventRepository.findAllByOrderByCreatedAtDesc(PageRequest.of(0, 10000)).forEach(event -> content.add(event.getKey()));
}
@@ -47,7 +54,7 @@ public class DownloadHeadlessController {
private void addNewToDiff() {
logger.error("startProcessing");
DownloadHeadlessService downloadHeadlessService = new DownloadHeadlessService();
Map<String, Lot> currentContent = downloadHeadlessService.downloadExcel(0);
Map<String, Lot> currentContent = downloadHeadlessService.downloadExcel(0, EXCEL_LINK);
List<TorgiEvent> newEvents = new ArrayList<>();
for (String key : currentContent.keySet()) {
@@ -102,8 +109,7 @@ public class DownloadHeadlessController {
newValue.setKey(LAST_RUN_TIME);
newValue.setValue(LocalDateTime.now().toString());
configsRepository.save(newValue);
}
else {
} else {
Configs oldValue = configs.get(0);
oldValue.setValue(LocalDateTime.now().toString());
configsRepository.save(oldValue);
@@ -164,8 +170,8 @@ public class DownloadHeadlessController {
String sourceFile = now.toString();
try (FileOutputStream fos = new FileOutputStream(now + ".zip");
ZipOutputStream zipOut = new ZipOutputStream(fos)) {
File fileToZip = new File(sourceFile);
zipFile(fileToZip, fileToZip.getName(), zipOut);
File fileToZip = new File(sourceFile);
zipFile(fileToZip, fileToZip.getName(), zipOut);
} catch (Exception ex) {
logger.error("in createArchive" + ex.getMessage());
}
@@ -199,4 +205,30 @@ public class DownloadHeadlessController {
}
fis.close();
}
@SneakyThrows
public void richData() {
List<MissedData> all = missedDataRepository.findAll(PageRequest.of(0, 10)).toList();
DownloadHeadlessService downloadHeadlessService = new DownloadHeadlessService();
for (MissedData missedCadastral : all) {
Map<String, Lot> currentContent = downloadHeadlessService.downloadExcel(0, String.format(CUSTOM_EXCEL_LINK, missedCadastral.getCadastral()));
logger.error("get content by cadastral {} {}", missedCadastral.getCadastral(), currentContent.size());
if (!currentContent.isEmpty()) {
List<TorgiEvent> newEvents = new ArrayList<>();
Set<String> contentByCadastral = new HashSet<>();
eventRepository.findAllByFullInfoLikeOrderByCreatedAtDesc("%" + missedCadastral.getCadastral() + "%").forEach(event -> contentByCadastral.add(event.getKey()));
for (String key : currentContent.keySet()) {
if (!contentByCadastral.contains(key)) {
newEvents.add(new TorgiEvent(key, currentContent.get(key)));
content.add(key);
logger.error("added new missed content");
}
}
eventRepository.saveAll(newEvents);
}
missedDataRepository.delete(missedCadastral);
Thread.sleep(5000);
}
}
}

19
src/main/java/com/pledge/torgi/controller/storage/FileServiceImpl.java
View File

@@ -1,6 +1,8 @@
package com.pledge.torgi.controller.storage;
import com.pledge.torgi.controller.DownloadHeadlessController;
import com.pledge.torgi.model.MissedData;
import com.pledge.torgi.repositories.MissedDataRepository;
import com.pledge.torgi.utils.StartParams;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -14,7 +16,7 @@ import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.io.InputStream;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
@@ -25,6 +27,7 @@ public class FileServiceImpl implements FileService {
private final DownloadHeadlessController headlessController;
private final StartParams params;
private final MissedDataRepository missedDataRepository;
@Override
public String processAsync(MultipartFile file, String email) {
@@ -53,11 +56,23 @@ public class FileServiceImpl implements FileService {
log.error("Extract cadastrals: " + cadastrals.size());
headlessController.sendNewContent(params, cadastrals, email);
saveCadastralToGetMissedData(cadastrals);
log.error("Complete request");
return "Обработано " + cadastrals.size() + " кадастровых номеров";
}
private void saveCadastralToGetMissedData(List<String> cadastrals) {
List<MissedData> all = missedDataRepository.findAll();
List<String> allExistedCadastrals = all.stream().map(MissedData::getCadastral).toList();
LocalDateTime now = LocalDateTime.now();
missedDataRepository.saveAll(cadastrals.stream().filter(cad -> !allExistedCadastrals.contains(cad)).map(cad -> {
MissedData md = new MissedData();
md.setCadastral(cad);
md.setCreatedTime(now);
return md;
}).toList());
}
private List<String> extractCadastral(Workbook wb) {
List<String> result = new ArrayList<>();
Sheet firstSheet = wb.getSheetAt(0);

27
src/main/java/com/pledge/torgi/model/MissedData.java Normal file
View File

@@ -0,0 +1,27 @@
package com.pledge.torgi.model;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import java.time.LocalDateTime;
@Entity
@Getter
@Setter
@EntityListeners(AuditingEntityListener.class)
@AllArgsConstructor
@NoArgsConstructor
public class MissedData {
@Id
@GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "MissedDataId")
@SequenceGenerator(name = "MissedDataId", sequenceName = "MISSED_DATA_SEQ", allocationSize = 1)
private Long id;
private String cadastral;
private LocalDateTime createdTime;
}

7
src/main/java/com/pledge/torgi/repositories/MissedDataRepository.java Normal file
View File

@@ -0,0 +1,7 @@
package com.pledge.torgi.repositories;
import com.pledge.torgi.model.MissedData;
import org.springframework.data.jpa.repository.JpaRepository;
public interface MissedDataRepository extends JpaRepository<MissedData, Long> {
}

19
src/main/java/com/pledge/torgi/service/DownloadHeadlessService.java
View File

@@ -29,7 +29,6 @@ import java.util.*;
import java.util.regex.Pattern;
import static com.pledge.torgi.utils.AppUtils.EMPTY;
import static com.pledge.torgi.utils.AppUtils.EXCEL_LINK;
import static java.util.Objects.nonNull;
public class DownloadHeadlessService {
@@ -40,9 +39,9 @@ public class DownloadHeadlessService {
Pattern vinPattern = Pattern.compile("\\b[A-HJ-NPR-Z0-9-]{12,17}\\b");
@SneakyThrows
public Map<String, Lot> downloadExcel(int retryCount) {
public Map<String, Lot> downloadExcel(int retryCount, String link) {
try {
HttpURLConnection connection = (HttpURLConnection) new URL(EXCEL_LINK).openConnection();
HttpURLConnection connection = (HttpURLConnection) new URL(link).openConnection();
connection.setRequestMethod("GET");
if (connection.getResponseCode() == HttpURLConnection.HTTP_OK) {
Map<String, Lot> data = new HashMap<>();
@@ -53,11 +52,11 @@ public class DownloadHeadlessService {
List<String> cadastrals = extractCadastral(sheetWithData.getRow(currentRow).getCell(29).getStringCellValue());
if (cadastrals.isEmpty()) {
Lot extractedContent = extractDataFromRow(sheetWithData.getRow(currentRow), "");
data.put(extractedContent.getNotice()+ "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus(), extractedContent);
data.put(extractedContent.getNotice() + "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus(), extractedContent);
} else {
for (String cadastral : cadastrals) {
Lot extractedContent = extractDataFromRow(sheetWithData.getRow(currentRow), cadastral);
data.put(extractedContent.getNotice()+ "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus() + "_" + cadastral, extractedContent);
data.put(extractedContent.getNotice() + "_" + extractedContent.getLotNumber() + "_" + extractedContent.getStatus() + "_" + cadastral, extractedContent);
}
}
@@ -67,7 +66,7 @@ public class DownloadHeadlessService {
} else {
logger.error("Error in line 99 ");
Thread.sleep(1000);
return downloadExcel(retryCount + 1);
return downloadExcel(retryCount + 1, link);
}
} catch (Exception e) {
logger.error("Error in line 103 " + e);
@@ -76,7 +75,7 @@ public class DownloadHeadlessService {
}
}
Thread.sleep(1000);
return downloadExcel(retryCount + 1);
return downloadExcel(retryCount + 1, link);
}
private Lot extractDataFromRow(Row row, String cadastral) {
@@ -84,7 +83,7 @@ public class DownloadHeadlessService {
lot.setBidStart(row.getCell(6).getStringCellValue());
lot.setBidEnd(row.getCell(7).getStringCellValue());
lot.setCadastral(cadastral);
lot.setLotNumber((int)row.getCell(16).getNumericCellValue());
lot.setLotNumber((int) row.getCell(16).getNumericCellValue());
lot.setFinalPrice(row.getCell(27).getStringCellValue());
lot.setNotice(row.getCell(3).getStringCellValue());
lot.setInitMoney(row.getCell(24).getStringCellValue());
@@ -189,7 +188,7 @@ public class DownloadHeadlessService {
}
private void setRowHead(XSSFSheet sheet) {
XSSFRow rowHead = sheet.createRow( 0);
XSSFRow rowHead = sheet.createRow(0);
rowHead.createCell(0).setCellValue("Статус торгов");
rowHead.createCell(1).setCellValue("Дата начала подачи заявок");
rowHead.createCell(2).setCellValue("Дата окончания подачи заявок");
@@ -236,7 +235,7 @@ public class DownloadHeadlessService {
private File getFileToSave(LocalDate now) {
File dir = new File(now.toString());
if(!dir.exists()) {
if (!dir.exists()) {
dir.mkdir();
}
return new File(dir.getAbsolutePath() + FileSystems.getDefault().getSeparator() + now + ".xlsx");

5
src/main/java/com/pledge/torgi/service/MainService.java
View File

@@ -1,16 +1,12 @@
package com.pledge.torgi.service;
import com.pledge.torgi.controller.DownloadHeadlessController;
import com.pledge.torgi.controller.storage.FileService;
import com.pledge.torgi.utils.StartParams;
import lombok.AllArgsConstructor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.concurrent.TimeUnit;
@Service
@@ -24,6 +20,7 @@ public class MainService {
@Scheduled(timeUnit = TimeUnit.SECONDS, fixedRate = 120)
public void getNewData() {
headlessController.process();
headlessController.richData();
}
@Scheduled(timeUnit = TimeUnit.SECONDS, fixedRate = 600)

2
src/main/java/com/pledge/torgi/utils/AppUtils.java
View File

@@ -30,4 +30,6 @@ public final class AppUtils {
public static final String SEARCH_URL = "https://torgi.gov.ru/new/api/public/lotcards/search?"; //example https://torgi.gov.ru/new/api/public/lotcards/search?pubFrom=2025-02-04&pubTo=2025-02-04&byFirstVersion=true&withFacets=true&page=0&size=10&sort=firstVersionPublicationDate,desc
public static final Format DATE_FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
public static final String EXCEL_LINK = "https://torgi.gov.ru/new/api/public/lotcards/export/excel?byFirstVersion=true&sort=updateDate,desc";
public static final String CUSTOM_EXCEL_LINK = "https://torgi.gov.ru/new/api/public/lotcards/export/excel?text=%s&matchPhrase=false&byFirstVersion=true&sort=firstVersionPublicationDate,desc";
}

9
src/main/resources/db/migration/V4__add_missed_data.sql Normal file
View File

@@ -0,0 +1,9 @@
CREATE TABLE missed_data
(
id BIGINT NOT NULL,
cadastral VARCHAR(255),
created_time TIMESTAMP,
CONSTRAINT pk_missed_data PRIMARY KEY (id)
);
CREATE SEQUENCE missed_data_seq;