diff --git a/pom.xml b/pom.xml index 914e219..1516852 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,12 @@ commons-lang3 3.12.0 + + + com.qniao + xxl-job-core + 2.3.0 + org.springframework.boot diff --git a/src/main/java/com/qniao/zsh/api/command/paddle/admin/PaddleAdminCommandController.java b/src/main/java/com/qniao/zsh/api/command/paddle/admin/PaddleAdminCommandController.java new file mode 100644 index 0000000..4c15b0e --- /dev/null +++ b/src/main/java/com/qniao/zsh/api/command/paddle/admin/PaddleAdminCommandController.java @@ -0,0 +1,33 @@ +package com.qniao.zsh.api.command.paddle.admin; + +import com.qniao.zsh.api.command.paddle.admin.request.PaddleDto; +import com.qniao.zsh.api.command.paddle.admin.response.PaddleVo; +import com.qniao.zsh.application.service.paddle.PaddleApplicationService; +import io.swagger.annotations.ApiOperation; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; +import java.io.IOException; + +/** + * @author wh + * @date 2023/4/11 + */ +@RestController +@RequestMapping("admin/paddle") +public class PaddleAdminCommandController { + + @Resource + private PaddleApplicationService paddleApplicationService; + + @ApiOperation("飞桨文本解析") + @PostMapping("/analysis") + public PaddleVo analysis(@RequestBody PaddleDto paddleDto) throws IOException { + return paddleApplicationService.runPaddle(paddleDto); + + } + +} diff --git a/src/main/java/com/qniao/zsh/api/command/paddle/admin/request/PaddleDto.java b/src/main/java/com/qniao/zsh/api/command/paddle/admin/request/PaddleDto.java new file mode 100644 index 0000000..8bd2633 --- /dev/null +++ b/src/main/java/com/qniao/zsh/api/command/paddle/admin/request/PaddleDto.java @@ -0,0 +1,40 @@ +package com.qniao.zsh.api.command.paddle.admin.request; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.constraints.NotNull; + +/** + * @author wh + * @date 2023/4/11 + */ +@Data +public class PaddleDto { + @NotNull + @JsonSerialize(using = ToStringSerializer.class) + @TableId(type = IdType.ASSIGN_ID) + @ApiModelProperty("文章ID") + private String articleId; + + @NotNull + @JsonSerialize(using = ToStringSerializer.class) + @TableId(type = IdType.ASSIGN_ID) + @ApiModelProperty("造纸厂ID") + private String paperMillId; + + @NotNull + @ApiModelProperty("文章标题") + private String title; + + @NotNull + @ApiModelProperty("文章正文") + private String content; + + + +} diff --git a/src/main/java/com/qniao/zsh/api/command/paddle/admin/response/PaddleVo.java b/src/main/java/com/qniao/zsh/api/command/paddle/admin/response/PaddleVo.java new file mode 100644 index 0000000..0105b53 --- /dev/null +++ b/src/main/java/com/qniao/zsh/api/command/paddle/admin/response/PaddleVo.java @@ -0,0 +1,24 @@ +package com.qniao.zsh.api.command.paddle.admin.response; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import javax.validation.constraints.NotNull; +import java.util.List; + +/** + * @author wh + * @date 2023/4/11 + */ +@Data +public class PaddleVo { + + @NotNull + @ApiModelProperty("纸厂名字") + private String factory; + + @ApiModelProperty("纸类分类") + private List paperInfo; + + @ApiModelProperty("纸类价格") + private List paperPrice; +} diff --git a/src/main/java/com/qniao/zsh/api/validator/SpiderStopped.java b/src/main/java/com/qniao/zsh/api/validator/SpiderStopped.java index 46f88c9..844cb37 100644 --- a/src/main/java/com/qniao/zsh/api/validator/SpiderStopped.java +++ b/src/main/java/com/qniao/zsh/api/validator/SpiderStopped.java @@ -1,6 +1,5 @@ package com.qniao.zsh.api.validator; -import cn.hutool.core.util.ObjectUtil; import com.qniao.zsh.domain.aggregate.spiderstate.constant.SpiderStateEnum; import com.qniao.zsh.domain.aggregate.spiderstate.entity.SpiderState; import com.qniao.zsh.infrastructure.dao.SpiderStateDao; diff --git a/src/main/java/com/qniao/zsh/application/service/paddle/PaddleApplicationService.java b/src/main/java/com/qniao/zsh/application/service/paddle/PaddleApplicationService.java new file mode 100644 index 0000000..4599efe --- /dev/null +++ b/src/main/java/com/qniao/zsh/application/service/paddle/PaddleApplicationService.java @@ -0,0 +1,42 @@ +package com.qniao.zsh.application.service.paddle; + +import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import com.qniao.zsh.api.command.paddle.admin.request.PaddleDto; +import com.qniao.zsh.api.command.paddle.admin.response.PaddleVo; +import com.qniao.zsh.infrastructure.config.SpiderDirectoryConfig; +import org.mc.ddd.infrastructure.util.ScriptRunnerHelper; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.List; + +/** + * @author wh + * @date 2023/4/11 + */ +@Service +public class PaddleApplicationService { + + public PaddleVo runPaddle(PaddleDto paddleDto) throws IOException { + //linux +// String result = ScriptRunnerHelper.handle("python3", + //win + String result = String.valueOf(ScriptRunnerHelper.handle("python", + SpiderDirectoryConfig.PADDLEDIR, + paddleDto.getTitle(), + paddleDto.getArticleId(), + paddleDto.getPaperMillId(), + paddleDto.getContent())); + String[] strings = result.split("\r\n"); + result = strings[strings.length - 1]; + JSONObject jsonResult = JSONUtil.parseObj(result); + PaddleVo paddleVo = new PaddleVo(); + paddleVo.setFactory(jsonResult.getStr("factory")); + paddleVo.setPaperInfo((List) jsonResult.getObj("paper_info")); + paddleVo.setPaperPrice((List) jsonResult.getObj("paper_price")); + return paddleVo; + } + +} diff --git a/src/main/java/com/qniao/zsh/domain/aggregate/paddle/entity/SpiderArticle.java b/src/main/java/com/qniao/zsh/domain/aggregate/paddle/entity/SpiderArticle.java new file mode 100644 index 0000000..d09d951 --- /dev/null +++ b/src/main/java/com/qniao/zsh/domain/aggregate/paddle/entity/SpiderArticle.java @@ -0,0 +1,27 @@ +package com.qniao.zsh.domain.aggregate.paddle.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; +import io.swagger.annotations.ApiModelProperty; + +/** + * @author wh + * @date 2023/4/11 + */ +public class SpiderArticle { + @JsonSerialize(using = ToStringSerializer.class) + @TableId(type = IdType.ASSIGN_ID) + @ApiModelProperty("文章ID") + private Long articleId; + + @JsonSerialize(using = ToStringSerializer.class) + @TableId(type = IdType.ASSIGN_ID) + @ApiModelProperty("造纸厂ID") + private Long paperMillId; + + + + +} diff --git a/src/main/java/com/qniao/zsh/domain/aggregate/spiderstate/entity/SpiderState.java b/src/main/java/com/qniao/zsh/domain/aggregate/spiderstate/entity/SpiderState.java index 442b41f..ba1e723 100644 --- a/src/main/java/com/qniao/zsh/domain/aggregate/spiderstate/entity/SpiderState.java +++ b/src/main/java/com/qniao/zsh/domain/aggregate/spiderstate/entity/SpiderState.java @@ -1,21 +1,41 @@ package com.qniao.zsh.domain.aggregate.spiderstate.entity; -import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.annotation.*; +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; import com.qniao.zsh.domain.aggregate.spiderstate.constant.SpiderStateEnum; import lombok.Data; import lombok.EqualsAndHashCode; import io.swagger.annotations.ApiModelProperty; import org.mc.ddd.infrastructure.persistent.Entity; +import java.time.LocalDateTime; + /** * @author Administrator * @date 2023/2/21 *爬虫状态数据库 */ @Data -@EqualsAndHashCode(callSuper = true) @TableName("spider_state") -public class SpiderState extends Entity { +public class SpiderState { + + @JsonSerialize(using = ToStringSerializer.class) + @TableId(type = IdType.ASSIGN_ID) + @ApiModelProperty("标识") + private Long id; + + @ApiModelProperty("是否删除 1-删除 0-未删除") + private Boolean isDeleted; + + @ApiModelProperty("创建时间") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS") + private LocalDateTime createdTime; + + @ApiModelProperty("更新时间") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS") + private LocalDateTime updatedTime; @ApiModelProperty("爬虫名字") private String spiderName; diff --git a/src/main/java/com/qniao/zsh/domain/aggregate/xxljob/entity/XxlJobInfo.java b/src/main/java/com/qniao/zsh/domain/aggregate/xxljob/entity/XxlJobInfo.java new file mode 100644 index 0000000..0e750cb --- /dev/null +++ b/src/main/java/com/qniao/zsh/domain/aggregate/xxljob/entity/XxlJobInfo.java @@ -0,0 +1,39 @@ +package com.qniao.zsh.domain.aggregate.xxljob.entity; + +import java.util.Date; + +public class XxlJobInfo { + + private int id; // 主键ID + + private int jobGroup; // 执行器主键ID + private String jobDesc; + + private Date addTime; + private Date updateTime; + + private String author; // 负责人 + private String alarmEmail; // 报警邮件 + + private String scheduleType; // 调度类型 + private String scheduleConf; // 调度配置，值含义取决于调度类型 + private String misfireStrategy; // 调度过期策略 + + private String executorRouteStrategy; // 执行器路由策略 + private String executorHandler; // 执行器，任务Handler名称 + private String executorParam; // 执行器，任务参数 + private String executorBlockStrategy; // 阻塞处理策略 + private int executorTimeout; // 任务执行超时时间，单位秒 + private int executorFailRetryCount; // 失败重试次数 + + private String glueType; // GLUE类型 #com.xxl.job.core.glue.GlueTypeEnum + private String glueSource; // GLUE源代码 + private String glueRemark; // GLUE备注 + private Date glueUpdatetime; // GLUE更新时间 + + private String childJobId; // 子任务ID，多个逗号分隔 + + private int triggerStatus; // 调度状态：0-停止，1-运行 + private long triggerLastTime; // 上次调度时间 + private long triggerNextTime; // 下次调度时间 +} diff --git a/src/main/java/com/qniao/zsh/infrastructure/config/SpiderDirectoryConfig.java b/src/main/java/com/qniao/zsh/infrastructure/config/SpiderDirectoryConfig.java index b0ff33d..4e17882 100644 --- a/src/main/java/com/qniao/zsh/infrastructure/config/SpiderDirectoryConfig.java +++ b/src/main/java/com/qniao/zsh/infrastructure/config/SpiderDirectoryConfig.java @@ -15,13 +15,19 @@ import org.springframework.context.annotation.Configuration; public class SpiderDirectoryConfig implements InitializingBean { @Value("${spier-dirdirectory}") - private String dir; + private String spider; - public static String DIR; + @Value("${paddle-dirdirectory}") + private String paddle; + + + public static String SCRAPYDIR; + public static String PADDLEDIR; @Override public void afterPropertiesSet() { - DIR = dir; + SCRAPYDIR = spider; + PADDLEDIR = paddle; } diff --git a/src/main/java/com/qniao/zsh/infrastructure/config/XxlJobConfig.java b/src/main/java/com/qniao/zsh/infrastructure/config/XxlJobConfig.java new file mode 100644 index 0000000..067dfc4 --- /dev/null +++ b/src/main/java/com/qniao/zsh/infrastructure/config/XxlJobConfig.java @@ -0,0 +1,43 @@ +package com.qniao.zsh.infrastructure.config; + +import com.xxl.job.core.executor.impl.XxlJobSpringExecutor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@Slf4j +public class XxlJobConfig { + @Value("${xxl.job.admin.addresses}") + private String adminAddresses; + @Value("${xxl.job.accessToken:}") + private String accessToken; + @Value("${xxl.job.executor.appname:}") + private String appname; + @Value("${xxl.job.executor.address:}") + private String address; + @Value("${xxl.job.executor.ip:}") + private String ip; + @Value("${xxl.job.executor.port:}") + private int port; + @Value("${xxl.job.executor.logpath:}") + private String logPath; + @Value("${xxl.job.executor.logretentiondays:0}") + private int logRetentionDays; + + @Bean + public XxlJobSpringExecutor xxlJobExecutor() { + log.info(">>>>>>>>>>> xxl-job config init."); + XxlJobSpringExecutor xxlJobSpringExecutor = new XxlJobSpringExecutor(); + xxlJobSpringExecutor.setAdminAddresses(adminAddresses); + xxlJobSpringExecutor.setAppname(appname); + xxlJobSpringExecutor.setAddress(address); + xxlJobSpringExecutor.setIp(ip); + xxlJobSpringExecutor.setPort(port); + xxlJobSpringExecutor.setAccessToken(accessToken); + xxlJobSpringExecutor.setLogPath(logPath); + xxlJobSpringExecutor.setLogRetentionDays(logRetentionDays); + return xxlJobSpringExecutor; + } +} \ No newline at end of file diff --git a/src/main/java/com/qniao/zsh/infrastructure/util/LinuxCommandUtil.java b/src/main/java/com/qniao/zsh/infrastructure/util/LinuxCommandUtil.java index acf9f77..ffc842d 100644 --- a/src/main/java/com/qniao/zsh/infrastructure/util/LinuxCommandUtil.java +++ b/src/main/java/com/qniao/zsh/infrastructure/util/LinuxCommandUtil.java @@ -32,7 +32,7 @@ public class LinuxCommandUtil { // 创建ProcessBuilder对象 ProcessBuilder builder = new ProcessBuilder("/bin/sh", "-c", command); //配置工作目录 - builder.directory(new File(SpiderDirectoryConfig.DIR)); + builder.directory(new File(SpiderDirectoryConfig.SCRAPYDIR)); // 重定向标准输出和标准错误输出到Java进程 builder.redirectErrorStream(true); diff --git a/src/main/java/com/qniao/zsh/infrastructure/util/xxl/JobHandlerService.java b/src/main/java/com/qniao/zsh/infrastructure/util/xxl/JobHandlerService.java new file mode 100644 index 0000000..c3d3aa3 --- /dev/null +++ b/src/main/java/com/qniao/zsh/infrastructure/util/xxl/JobHandlerService.java @@ -0,0 +1,15 @@ +package com.qniao.zsh.infrastructure.util.xxl; +import com.qniao.zsh.domain.aggregate.xxljob.entity.XxlJobInfo; +import com.xxl.job.core.enums.ExecutorBlockStrategyEnum; +import com.xxl.job.core.biz.model.TriggerParam; +import com.xxl.job.core.executor.XxlJobExecutor; +import java.sql.Timestamp; + + +public class JobHandlerService { + public void updateJob() { + XxlJobExecutor executor = new XxlJobExecutor(); + XxlJobInfo jobInfo = new XxlJobInfo(); + + } +} diff --git a/src/main/java/com/qniao/zsh/infrastructure/util/xxl/XxlJobDemo.java b/src/main/java/com/qniao/zsh/infrastructure/util/xxl/XxlJobDemo.java new file mode 100644 index 0000000..8164cd0 --- /dev/null +++ b/src/main/java/com/qniao/zsh/infrastructure/util/xxl/XxlJobDemo.java @@ -0,0 +1,16 @@ +package com.qniao.zsh.infrastructure.util.xxl; + +import com.xxl.job.core.biz.model.ReturnT; +import com.xxl.job.core.handler.annotation.XxlJob; +import org.springframework.stereotype.Component; + +@Component +public class XxlJobDemo { + + @XxlJob("myJobHandler") + public ReturnT myJobHandler(String param) throws Exception { + // your jobHandler code here + return ReturnT.SUCCESS; + } +} + diff --git a/src/main/java/com/qniao/zsh/infrastructure/util/xxl/xxl.java b/src/main/java/com/qniao/zsh/infrastructure/util/xxl/xxl.java new file mode 100644 index 0000000..3f38bd9 --- /dev/null +++ b/src/main/java/com/qniao/zsh/infrastructure/util/xxl/xxl.java @@ -0,0 +1,11 @@ +package com.qniao.zsh.infrastructure.util.xxl; + +import com.xxl.job.core.handler.annotation.XxlJob; + +public class xxl { + @XxlJob("spiderTest") + public static void main(String[] args) { + System.out.println("调用成功"); + + } +} diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index 1f20e07..22761ee 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -34,7 +34,7 @@ xxl.job.admin.addresses: http://xxl-job-admin-dev.qniao.cn/xxl-job-admin # 执行器AppName [选填]：执行器心跳注册分组依据；为空则关闭自动注册 xxl.job.executor.appname: ztb-factory-dev # 执行器端口号 [选填]：小于等于0则自动获取；默认端口为9999，单机部署多个执行器时，注意要配置不同执行器端口； -xxl.job.executor.port: 9999 +xxl.job.executor.port: 3856 ### xxl-job executor log-path #xxl.job.executor.logpath: /data/applogs/xxl-job/jobhandler ### xxl-job executor log-retention-days @@ -45,6 +45,11 @@ xxl.job.executor.logretentiondays: 30 #spier-dirdirectory: D:\Project\papernews_spider #Linux spier-dirdirectory: /opt/ztb-information-spider/papernews_spider +#winpaddle +paddle-dirdirectory: E:\Python\ztb-information-spider - 副本\ztb-information-spider\papernews_spider\Module\PaddleRun.py +#Linuxpaddle +#paddle-dirdirectory: /opt/ztb-information-spider/papernews_spider/Module/PaddleRun.py + #登录请求url login-request: https://api-ops-ztb-test.qniao.cn/ztb-supply-chain-service/admin/login