Merge remote-tracking branch 'origin/uat-temp-castle-2502241152-fordFetchWeb' into uat-temp-wj-chongqingdaqu-prod

modules/i3plus-ext-mes-pcn-api/src/main/java/cn/estsh/i3plus/ext/mes/pcn/api/busi/ford/IFordFetchWebService.java

@@ -0,0 +1,23 @@
+package cn.estsh.i3plus.ext.mes.pcn.api.busi.ford;
+
+import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordFetchParameter;
+import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordJsaSortInfoWithFetchWeb;
+
+import java.util.List;
+
+public interface IFordFetchWebService {
+
+    void insertParams(MesFordFetchParameter parameter);
+
+
+    void updateParams(MesFordFetchParameter parameter);
+
+
+    void deleteParams(Long id,String UserName);
+
+    MesFordFetchParameter queryParams(Long id);
+
+    List<MesFordFetchParameter> queryParams(String organizeCode);
+
+    List<MesFordJsaSortInfoWithFetchWeb> doFetchData(Long id, String organizeCode);
+}

modules/i3plus-ext-mes-pcn-apiservice/pom.xml

@@ -14,6 +14,10 @@
     <packaging>jar</packaging>
 
     <dependencies>
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+        </dependency>
         <!-- CXF依赖 -->
         <dependency>
             <groupId>org.apache.cxf</groupId>

modules/i3plus-ext-mes-pcn-apiservice/src/main/java/cn/estsh/i3plus/ext/mes/pcn/apiservice/controller/busi/ford/MesFordFetchWebController.java

@@ -0,0 +1,97 @@
+package cn.estsh.i3plus.ext.mes.pcn.apiservice.controller.busi.ford;
+
+import cn.estsh.i3plus.ext.mes.pcn.api.busi.ford.IFordFetchWebService;
+import cn.estsh.i3plus.ext.mes.pcn.pojo.constant.MesCommonConstant;
+import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordFetchParameter;
+import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordJsaSortInfoWithFetchWeb;
+import cn.estsh.impp.framework.boot.util.ResultBean;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@RestController
+@Api(tags = "福特web抓单")
+@RequestMapping(MesCommonConstant.MES_YANFEN + "/fetch-web")
+@Slf4j
+public class MesFordFetchWebController {
+
+    @Autowired
+    private IFordFetchWebService fordFetchWebService;
+
+    @PostMapping(value = "/param/save")
+    @ApiOperation(value = "保存抓单参数")
+    public ResultBean saveParams(@RequestBody MesFordFetchParameter params) {
+        try {
+            fordFetchWebService.insertParams(params);
+        } catch (Exception e) {
+            return ResultBean.fail(e.getMessage());
+        }
+        return ResultBean.success("保存成功!");
+    }
+
+    @PostMapping(value = "/param/update")
+    @ApiOperation(value = "更新抓单参数")
+    public ResultBean updateParams(@RequestBody MesFordFetchParameter params) {
+        try {
+            fordFetchWebService.updateParams(params);
+        } catch (Exception e) {
+            return ResultBean.fail(e.getMessage());
+        }
+        return ResultBean.success("更新成功!");
+    }
+
+    @GetMapping(value = "/param/del/{id}/{userName}")
+    @ApiOperation(value = "删除抓单参数")
+    public ResultBean updateParams(@PathVariable Long id, @PathVariable String userName) {
+        try {
+            fordFetchWebService.deleteParams(id, userName);
+        } catch (Exception e) {
+            return ResultBean.fail(e.getMessage());
+        }
+        return ResultBean.success("删除成功!");
+    }
+
+    @GetMapping(value = "/param/query/{id}")
+    @ApiOperation(value = "查询抓单参数")
+    public ResultBean queryParams(@PathVariable Long id) {
+        MesFordFetchParameter parameter;
+        try {
+            parameter = fordFetchWebService.queryParams(id);
+        } catch (Exception e) {
+            return ResultBean.fail(e.getMessage());
+        }
+        return ResultBean.success("查询成功!").setResultObject(parameter);
+    }
+
+
+    @GetMapping(value = "/param/query/list/{organizeCode}")
+    @ApiOperation(value = "查询抓单参数")
+    public ResultBean queryParamList(@PathVariable String organizeCode) {
+        List<MesFordFetchParameter> mesFordFetchParameters;
+        try {
+            mesFordFetchParameters = fordFetchWebService.queryParams(organizeCode);
+        } catch (Exception e) {
+            return ResultBean.fail(e.getMessage());
+        }
+        return ResultBean.success("查询成功!").setResultList(mesFordFetchParameters);
+    }
+
+    @GetMapping(value = "/param/data/{id}/{organizeCode}")
+    @ApiOperation(value = "定时抓单")
+    public ResultBean queryData(@PathVariable Long id,@PathVariable String organizeCode) {
+        List<MesFordJsaSortInfoWithFetchWeb> mesFordJsaSortInfoWithFetchWebs;
+        try {
+            mesFordJsaSortInfoWithFetchWebs = fordFetchWebService.doFetchData(id, organizeCode);
+        } catch (Exception e) {
+            return ResultBean.fail(e.getMessage());
+        }
+        return ResultBean.success("查询成功").setResultList(mesFordJsaSortInfoWithFetchWebs);
+    }
+
+
+}

modules/i3plus-ext-mes-pcn-apiservice/src/main/java/cn/estsh/i3plus/ext/mes/pcn/apiservice/serviceimpl/ford/MesFordFetchWebServiceImpl.java

@@ -0,0 +1,182 @@
+package cn.estsh.i3plus.ext.mes.pcn.apiservice.serviceimpl.ford;
+
+import cn.estsh.i3plus.ext.mes.pcn.api.busi.ford.IFordFetchWebService;
+import cn.estsh.i3plus.platform.common.convert.ConvertBean;
+import cn.estsh.i3plus.pojo.base.bean.DdlPackBean;
+import cn.estsh.i3plus.pojo.base.tool.DdlPreparedPack;
+import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordFetchParameter;
+import cn.estsh.i3plus.pojo.mes.bean.ford.MesFordJsaSortInfoWithFetchWeb;
+import cn.estsh.i3plus.pojo.mes.repository.ford.MesFordFetchParameterRepository;
+import cn.estsh.i3plus.pojo.mes.repository.ford.MesFordJsaSortInfoWithFetchWebRepository;
+import cn.estsh.impp.framework.boot.exception.ImppExceptionBuilder;
+import cn.hutool.http.HttpRequest;
+import cn.hutool.http.HttpResponse;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.net.HttpCookie;
+import java.util.*;
+
+@Service
+@Slf4j
+public class MesFordFetchWebServiceImpl implements IFordFetchWebService {
+    @Autowired
+    private MesFordFetchParameterRepository paramRao;
+
+    @Autowired
+    private MesFordJsaSortInfoWithFetchWebRepository fetchWebRao;
+
+    @Override
+    public void insertParams(MesFordFetchParameter parameter) {
+       ConvertBean.saveOrUpdate(parameter, parameter.getCreateUser());
+        paramRao.insert(parameter);
+
+    }
+
+    @Override
+    public void updateParams(MesFordFetchParameter parameter) {
+        ConvertBean.saveOrUpdate(parameter, parameter.getCreateUser());
+        paramRao.update(parameter);
+    }
+
+    @Override
+    public void deleteParams(Long id,String userName) {
+        paramRao.deleteWeaklyById(id,userName);
+    }
+
+    @Override
+    public MesFordFetchParameter queryParams(Long id) {
+        Optional<MesFordFetchParameter> params = paramRao.findById(id);
+        return params.orElseGet(MesFordFetchParameter::new);
+
+    }
+
+    @Override
+    public List<MesFordFetchParameter> queryParams(String organizeCode) {
+        DdlPackBean ddlPackBean = DdlPackBean.getDdlPackBean(organizeCode);
+        return paramRao.findByHqlWhere(ddlPackBean);
+    }
+
+    @Override
+    public List<MesFordJsaSortInfoWithFetchWeb> doFetchData(Long id, String organizeCode) {
+        List<MesFordJsaSortInfoWithFetchWeb> infoList = new ArrayList<>();
+        DdlPackBean ddlPackBean = DdlPackBean.getDdlPackBean(organizeCode);
+        DdlPreparedPack.getNumEqualPack(id,"id",ddlPackBean);
+        List<MesFordFetchParameter> list = paramRao.findByHqlWhere(ddlPackBean);
+        if (list.isEmpty()) {
+            throw ImppExceptionBuilder.newInstance().setErrorDetail("未查询出当前配置,请检查数据!").build();
+        }
+        MesFordFetchParameter parameter = list.get(0);
+        //尝试登录
+        Map<String, Object> paramMap = new HashMap<>();
+        paramMap.put("username", parameter.getUid());
+        paramMap.put("password", parameter.getPwd());
+        paramMap.put("t", String.valueOf(System.currentTimeMillis()));
+        String body = "username="+parameter.getUid()+"&password="+parameter.getPwd()+"&t="+ System.currentTimeMillis();
+        String loginUrl = parameter.getLoginUrl();
+        HttpResponse loginResponse = HttpRequest.post(loginUrl).body(body)
+                .header("Content-Type","application/x-www-form-urlencoded").timeout(60000).execute();
+        log.info("doFetchData-打印的返回值是:{}",loginResponse.body());
+        if (loginResponse.getStatus() != 200){
+            log.info("doFetchData-登录失败,需要重新登录!返回内容为:{}",loginResponse.body());
+        }
+        List<HttpCookie> cookies = loginResponse.getCookies();
+        //拼接请求参数
+        //time=0.9656002469020775&maxpage=5&trigger=B100VR12&st=2023-05-12%2005:00:00&et=2023-05-12%2023:59:59&catacode=0&itemcode=0
+        Map<String,Object> map = new HashMap<>();
+        map.put("maxpage",parameter.getPageSize() != null ? parameter.getPageSize().toString() : "1");
+        map.put("trigger",parameter.getInfoPointCode());
+        map.put("st",parameter.getStartDateTime());
+        map.put("et",parameter.getEndDateTime());
+        map.put("catacode","0");
+        map.put("itemcode","0");
+        //校验最后一条数据是否已经存在数据库中， 特殊逻辑page如果为空，则只查最后一页的内容
+        boolean result = checkLastPage(parameter, cookies, organizeCode);
+        if (!result){
+            //分页查询
+            map.put("page",parameter.getCurrentPageIndex());
+            HttpResponse response = HttpRequest.get(parameter.getFetchUrl()).cookie(cookies).timeout(60000).form(map).execute();
+            log.info("doFetchData-查询出来的内容为:{}",response.body());
+            Document doc = Jsoup.parse(response.body());
+            int tempCount = 0;
+            Elements trList = doc.select("tr");
+            for (int i = 1; i < trList.size(); i++) {
+                Elements tdList = trList.get(i).select("td");
+                MesFordJsaSortInfoWithFetchWeb webInfo = new MesFordJsaSortInfoWithFetchWeb();
+                if (tdList.get(0).text().equals("无数据") || StringUtils.isBlank(tdList.get(0).text())){
+                    break;
+                }
+                tempCount++;
+                webInfo.setSeq(Long.parseLong(tdList.get(0).text()));
+                webInfo.setFetchDate(tdList.get(1).text());
+                webInfo.setFetchTime(tdList.get(2).text());
+                webInfo.setVinCode(tdList.get(3).text());
+                webInfo.setPartNo(tdList.get(4).text());
+                webInfo.setPartName(tdList.get(5).text());
+                webInfo.setColorCode(tdList.get(6).text());
+                webInfo.setInfo0008(tdList.get(7).text());
+                webInfo.setSeqContext(tdList.get(8).text());
+                webInfo.setRemark(tdList.get(9).text());
+                webInfo.setInfoPointDesc(tdList.get(10).text());
+                webInfo.setFetchType(1); //抓取类型
+                webInfo.setDataStatus(200);
+                webInfo.setOrganizeCode(organizeCode);
+                webInfo.setInfoPointCode(parameter.getInfoPointCode());
+                webInfo.setCustOrganizeCode(parameter.getCustOrganizeCode());
+                ConvertBean.saveOrUpdate(webInfo,"doFetchData");
+                //校验是否已经查询过
+                DdlPackBean webInfoPackBean = DdlPackBean.getDdlPackBean(organizeCode);
+                DdlPreparedPack.getNumEqualPack(webInfo.getSeq(),"seq",webInfoPackBean);
+                DdlPreparedPack.getStringEqualPack(webInfo.getFetchDate(),"fetchDate",webInfoPackBean);
+                DdlPreparedPack.getStringEqualPack(webInfo.getFetchTime(),"fetchTime",webInfoPackBean);
+                int count = fetchWebRao.findByHqlWhereCount(webInfoPackBean);
+                if (count > 0){
+                    continue;
+                }
+                fetchWebRao.insert(webInfo);
+                infoList.add(webInfo);
+            }
+            if (tempCount == parameter.getPageSize()){
+                parameter.setCurrentPageIndex(parameter.getCurrentPageIndex()+1);
+                ConvertBean.saveOrUpdate(parameter,"doFetchData-index");
+                paramRao.update(parameter);
+            }
+        }
+        return  infoList;
+    }
+
+    private boolean checkLastPage(MesFordFetchParameter parameter, List<HttpCookie> cookies,String organizeCode) {
+        Map<String,Object> map = new HashMap<>();
+        map.put("maxpage",parameter.getPageSize() != null ? parameter.getPageSize().toString() : "1");
+        map.put("trigger",parameter.getInfoPointCode());
+        map.put("st",parameter.getStartDateTime());
+        map.put("et",parameter.getEndDateTime());
+        map.put("catacode","0");
+        map.put("itemcode","0");
+        //校验最后一条数据是否已经存在数据库中， 特殊逻辑page如果为空，则只查最后一页的内容
+        HttpResponse response = HttpRequest.get(parameter.getFetchUrl()).cookie(cookies).timeout(60000).form(map).execute();
+        log.info("doFetchData-查询出来的内容为:{}",response.body());
+        Document doc = Jsoup.parse(response.body());
+        Elements trList = doc.select("tr");
+        Elements tdList = trList.get(trList.size()-1).select("td");
+        if (!tdList.isEmpty()){
+            if (tdList.get(0).text().equals("无数据") || StringUtils.isBlank(tdList.get(0).text())) {
+                return false;
+            }
+            //校验是否已经查询过
+            DdlPackBean webInfoPackBean = DdlPackBean.getDdlPackBean(organizeCode);
+            DdlPreparedPack.getNumEqualPack(Long.parseLong(tdList.get(0).text()),"seq",webInfoPackBean);
+            DdlPreparedPack.getStringEqualPack(tdList.get(1).text(),"fetchDate",webInfoPackBean);
+            DdlPreparedPack.getStringEqualPack(tdList.get(2).text(),"fetchTime",webInfoPackBean);
+            int count = fetchWebRao.findByHqlWhereCount(webInfoPackBean);
+            return count > 0;
+        }
+        return false;
+
+    }
+}

pom.xml

@@ -41,6 +41,11 @@
 
     <dependencyManagement>
         <dependencies>
+            <dependency>
+                <groupId>org.jsoup</groupId>
+                <artifactId>jsoup</artifactId>
+                <version>1.11.3</version>
+            </dependency>
             <!-- CXF依赖 -->
             <dependency>
                 <groupId>org.apache.cxf</groupId>