接入xxl-job，飞桨解析模块

pom.xml

@@ -94,6 +94,12 @@
             <artifactId>commons-lang3</artifactId>
             <version>3.12.0</version>
         </dependency>
+        <!--xxl-job-->
+        <dependency>
+            <groupId>com.qniao</groupId>
+            <artifactId>xxl-job-core</artifactId>
+            <version>2.3.0</version>
+        </dependency>
         <!-- Test -->
         <dependency>
             <groupId>org.springframework.boot</groupId>

src/main/java/com/qniao/zsh/api/command/paddle/admin/PaddleAdminCommandController.java

@@ -0,0 +1,33 @@
+package com.qniao.zsh.api.command.paddle.admin;
+
+import com.qniao.zsh.api.command.paddle.admin.request.PaddleDto;
+import com.qniao.zsh.api.command.paddle.admin.response.PaddleVo;
+import com.qniao.zsh.application.service.paddle.PaddleApplicationService;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.annotation.Resource;
+import java.io.IOException;
+
+/**
+ * @author wh
+ * @date 2023/4/11
+ */
+@RestController
+@RequestMapping("admin/paddle")
+public class PaddleAdminCommandController {
+
+    @Resource
+    private PaddleApplicationService paddleApplicationService;
+
+    @ApiOperation("飞桨文本解析")
+    @PostMapping("/analysis")
+    public PaddleVo analysis(@RequestBody PaddleDto paddleDto) throws IOException {
+        return paddleApplicationService.runPaddle(paddleDto);
+
+    }
+
+}

src/main/java/com/qniao/zsh/api/command/paddle/admin/request/PaddleDto.java

@@ -0,0 +1,40 @@
+package com.qniao.zsh.api.command.paddle.admin.request;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+
+import javax.validation.constraints.NotNull;
+
+/**
+ * @author wh
+ * @date 2023/4/11
+ */
+@Data
+public class PaddleDto {
+    @NotNull
+    @JsonSerialize(using = ToStringSerializer.class)
+    @TableId(type = IdType.ASSIGN_ID)
+    @ApiModelProperty("文章ID")
+    private String articleId;
+
+    @NotNull
+    @JsonSerialize(using = ToStringSerializer.class)
+    @TableId(type = IdType.ASSIGN_ID)
+    @ApiModelProperty("造纸厂ID")
+    private String paperMillId;
+
+    @NotNull
+    @ApiModelProperty("文章标题")
+    private String title;
+
+    @NotNull
+    @ApiModelProperty("文章正文")
+    private String content;
+
+
+
+}

src/main/java/com/qniao/zsh/api/command/paddle/admin/response/PaddleVo.java

@@ -0,0 +1,24 @@
+package com.qniao.zsh.api.command.paddle.admin.response;
+
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+/**
+ * @author wh
+ * @date 2023/4/11
+ */
+@Data
+public class PaddleVo {
+
+    @NotNull
+    @ApiModelProperty("纸厂名字")
+    private String factory;
+
+    @ApiModelProperty("纸类分类")
+    private List<String> paperInfo;
+
+    @ApiModelProperty("纸类价格")
+    private List<String> paperPrice;
+}

src/main/java/com/qniao/zsh/api/validator/SpiderStopped.java

@@ -1,6 +1,5 @@
 package com.qniao.zsh.api.validator;
 
-import cn.hutool.core.util.ObjectUtil;
 import com.qniao.zsh.domain.aggregate.spiderstate.constant.SpiderStateEnum;
 import com.qniao.zsh.domain.aggregate.spiderstate.entity.SpiderState;
 import com.qniao.zsh.infrastructure.dao.SpiderStateDao;

src/main/java/com/qniao/zsh/application/service/paddle/PaddleApplicationService.java

@@ -0,0 +1,42 @@
+package com.qniao.zsh.application.service.paddle;
+
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.json.JSONObject;
+import cn.hutool.json.JSONUtil;
+import com.qniao.zsh.api.command.paddle.admin.request.PaddleDto;
+import com.qniao.zsh.api.command.paddle.admin.response.PaddleVo;
+import com.qniao.zsh.infrastructure.config.SpiderDirectoryConfig;
+import org.mc.ddd.infrastructure.util.ScriptRunnerHelper;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * @author wh
+ * @date 2023/4/11
+ */
+@Service
+public class PaddleApplicationService {
+
+    public PaddleVo runPaddle(PaddleDto paddleDto) throws IOException {
+        //linux
+//        String result = ScriptRunnerHelper.handle("python3",
+        //win
+        String result = String.valueOf(ScriptRunnerHelper.handle("python",
+                SpiderDirectoryConfig.PADDLEDIR,
+                paddleDto.getTitle(),
+                paddleDto.getArticleId(),
+                paddleDto.getPaperMillId(),
+                paddleDto.getContent()));
+        String[] strings = result.split("\r\n");
+        result = strings[strings.length - 1];
+        JSONObject jsonResult = JSONUtil.parseObj(result);
+        PaddleVo paddleVo = new PaddleVo();
+        paddleVo.setFactory(jsonResult.getStr("factory"));
+        paddleVo.setPaperInfo((List<String>) jsonResult.getObj("paper_info"));
+        paddleVo.setPaperPrice((List<String>) jsonResult.getObj("paper_price"));
+        return paddleVo;
+    }
+
+}

src/main/java/com/qniao/zsh/domain/aggregate/paddle/entity/SpiderArticle.java

@@ -0,0 +1,27 @@
+package com.qniao.zsh.domain.aggregate.paddle.entity;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
+import io.swagger.annotations.ApiModelProperty;
+
+/**
+ * @author wh
+ * @date 2023/4/11
+ */
+public class SpiderArticle {
+    @JsonSerialize(using = ToStringSerializer.class)
+    @TableId(type = IdType.ASSIGN_ID)
+    @ApiModelProperty("文章ID")
+    private Long articleId;
+
+    @JsonSerialize(using = ToStringSerializer.class)
+    @TableId(type = IdType.ASSIGN_ID)
+    @ApiModelProperty("造纸厂ID")
+    private Long paperMillId;
+
+
+
+
+}

src/main/java/com/qniao/zsh/domain/aggregate/spiderstate/entity/SpiderState.java

@@ -1,21 +1,41 @@
 package com.qniao.zsh.domain.aggregate.spiderstate.entity;
 
-import com.baomidou.mybatisplus.annotation.TableName;
+import com.baomidou.mybatisplus.annotation.*;
+import com.fasterxml.jackson.annotation.JsonFormat;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
 import com.qniao.zsh.domain.aggregate.spiderstate.constant.SpiderStateEnum;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import io.swagger.annotations.ApiModelProperty;
 import org.mc.ddd.infrastructure.persistent.Entity;
 
+import java.time.LocalDateTime;
+
 /**
  * @author Administrator
  * @date 2023/2/21
  *爬虫状态数据库
  */
 @Data
-@EqualsAndHashCode(callSuper = true)
 @TableName("spider_state")
-public class SpiderState extends Entity<SpiderState> {
+public class SpiderState  {
+
+    @JsonSerialize(using = ToStringSerializer.class)
+    @TableId(type = IdType.ASSIGN_ID)
+    @ApiModelProperty("标识")
+    private Long id;
+
+    @ApiModelProperty("是否删除 1-删除 0-未删除")
+    private Boolean isDeleted;
+
+    @ApiModelProperty("创建时间")
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS")
+    private LocalDateTime createdTime;
+
+    @ApiModelProperty("更新时间")
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS")
+    private LocalDateTime updatedTime;
 
     @ApiModelProperty("爬虫名字")
     private String spiderName;

src/main/java/com/qniao/zsh/domain/aggregate/xxljob/entity/XxlJobInfo.java

@@ -0,0 +1,39 @@
+package com.qniao.zsh.domain.aggregate.xxljob.entity;
+
+import java.util.Date;
+
+public class XxlJobInfo {
+
+    private int id;                // 主键ID
+
+    private int jobGroup;        // 执行器主键ID
+    private String jobDesc;
+
+    private Date addTime;
+    private Date updateTime;
+
+    private String author;        // 负责人
+    private String alarmEmail;    // 报警邮件
+
+    private String scheduleType;            // 调度类型
+    private String scheduleConf;            // 调度配置，值含义取决于调度类型
+    private String misfireStrategy;            // 调度过期策略
+
+    private String executorRouteStrategy;    // 执行器路由策略
+    private String executorHandler;            // 执行器，任务Handler名称
+    private String executorParam;            // 执行器，任务参数
+    private String executorBlockStrategy;    // 阻塞处理策略
+    private int executorTimeout;            // 任务执行超时时间，单位秒
+    private int executorFailRetryCount;        // 失败重试次数
+
+    private String glueType;        // GLUE类型	#com.xxl.job.core.glue.GlueTypeEnum
+    private String glueSource;        // GLUE源代码
+    private String glueRemark;        // GLUE备注
+    private Date glueUpdatetime;    // GLUE更新时间
+
+    private String childJobId;        // 子任务ID，多个逗号分隔
+
+    private int triggerStatus;        // 调度状态：0-停止，1-运行
+    private long triggerLastTime;    // 上次调度时间
+    private long triggerNextTime;    // 下次调度时间
+}

src/main/java/com/qniao/zsh/infrastructure/config/SpiderDirectoryConfig.java

@@ -15,13 +15,19 @@ import org.springframework.context.annotation.Configuration;
 public class SpiderDirectoryConfig implements InitializingBean {
 
     @Value("${spier-dirdirectory}")
-    private String dir;
+    private String spider;
 
-    public static String DIR;
+    @Value("${paddle-dirdirectory}")
+    private String paddle;
+
+
+    public static String SCRAPYDIR;
+    public static String PADDLEDIR;
 
     @Override
     public void afterPropertiesSet() {
-        DIR = dir;
+        SCRAPYDIR = spider;
+        PADDLEDIR = paddle;
 
 
     }

src/main/java/com/qniao/zsh/infrastructure/config/XxlJobConfig.java

@@ -0,0 +1,43 @@
+package com.qniao.zsh.infrastructure.config;
+
+import com.xxl.job.core.executor.impl.XxlJobSpringExecutor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@Slf4j
+public class XxlJobConfig {
+    @Value("${xxl.job.admin.addresses}")
+    private String adminAddresses;
+    @Value("${xxl.job.accessToken:}")
+    private String accessToken;
+    @Value("${xxl.job.executor.appname:}")
+    private String appname;
+    @Value("${xxl.job.executor.address:}")
+    private String address;
+    @Value("${xxl.job.executor.ip:}")
+    private String ip;
+    @Value("${xxl.job.executor.port:}")
+    private int port;
+    @Value("${xxl.job.executor.logpath:}")
+    private String logPath;
+    @Value("${xxl.job.executor.logretentiondays:0}")
+    private int logRetentionDays;
+
+    @Bean
+    public XxlJobSpringExecutor xxlJobExecutor() {
+        log.info(">>>>>>>>>>> xxl-job config init.");
+        XxlJobSpringExecutor xxlJobSpringExecutor = new XxlJobSpringExecutor();
+        xxlJobSpringExecutor.setAdminAddresses(adminAddresses);
+        xxlJobSpringExecutor.setAppname(appname);
+        xxlJobSpringExecutor.setAddress(address);
+        xxlJobSpringExecutor.setIp(ip);
+        xxlJobSpringExecutor.setPort(port);
+        xxlJobSpringExecutor.setAccessToken(accessToken);
+        xxlJobSpringExecutor.setLogPath(logPath);
+        xxlJobSpringExecutor.setLogRetentionDays(logRetentionDays);
+        return xxlJobSpringExecutor;
+    }
+}

src/main/java/com/qniao/zsh/infrastructure/util/LinuxCommandUtil.java

@@ -32,7 +32,7 @@ public class LinuxCommandUtil {
         // 创建ProcessBuilder对象
         ProcessBuilder builder = new ProcessBuilder("/bin/sh", "-c", command);
         //配置工作目录
-        builder.directory(new File(SpiderDirectoryConfig.DIR));
+        builder.directory(new File(SpiderDirectoryConfig.SCRAPYDIR));
         // 重定向标准输出和标准错误输出到Java进程
         builder.redirectErrorStream(true);
 

src/main/java/com/qniao/zsh/infrastructure/util/xxl/JobHandlerService.java

@@ -0,0 +1,15 @@
+package com.qniao.zsh.infrastructure.util.xxl;
+import com.qniao.zsh.domain.aggregate.xxljob.entity.XxlJobInfo;
+import com.xxl.job.core.enums.ExecutorBlockStrategyEnum;
+import com.xxl.job.core.biz.model.TriggerParam;
+import com.xxl.job.core.executor.XxlJobExecutor;
+import java.sql.Timestamp;
+
+
+public class JobHandlerService {
+    public void updateJob() {
+        XxlJobExecutor executor = new XxlJobExecutor();
+        XxlJobInfo jobInfo = new XxlJobInfo();
+
+    }
+}

src/main/java/com/qniao/zsh/infrastructure/util/xxl/XxlJobDemo.java

@@ -0,0 +1,16 @@
+package com.qniao.zsh.infrastructure.util.xxl;
+
+import com.xxl.job.core.biz.model.ReturnT;
+import com.xxl.job.core.handler.annotation.XxlJob;
+import org.springframework.stereotype.Component;
+
+@Component
+public class XxlJobDemo {
+
+    @XxlJob("myJobHandler")
+    public ReturnT<String> myJobHandler(String param) throws Exception {
+        // your jobHandler code here
+        return ReturnT.SUCCESS;
+    }
+}
+

src/main/java/com/qniao/zsh/infrastructure/util/xxl/xxl.java

@@ -0,0 +1,11 @@
+package com.qniao.zsh.infrastructure.util.xxl;
+
+import com.xxl.job.core.handler.annotation.XxlJob;
+
+public class xxl {
+    @XxlJob("spiderTest")
+    public static void main(String[] args) {
+        System.out.println("调用成功");
+
+    }
+}

src/main/resources/application-local.yml

@@ -34,7 +34,7 @@ xxl.job.admin.addresses: http://xxl-job-admin-dev.qniao.cn/xxl-job-admin
 # 执行器AppName [选填]：执行器心跳注册分组依据；为空则关闭自动注册
 xxl.job.executor.appname: ztb-factory-dev
 # 执行器端口号 [选填]：小于等于0则自动获取；默认端口为9999，单机部署多个执行器时，注意要配置不同执行器端口；
-xxl.job.executor.port: 9999
+xxl.job.executor.port: 3856
 ### xxl-job executor log-path
 #xxl.job.executor.logpath: /data/applogs/xxl-job/jobhandler
 ### xxl-job executor log-retention-days
@@ -45,6 +45,11 @@ xxl.job.executor.logretentiondays: 30
 #spier-dirdirectory: D:\Project\papernews_spider
 #Linux
 spier-dirdirectory: /opt/ztb-information-spider/papernews_spider
+#winpaddle
+paddle-dirdirectory: E:\Python\ztb-information-spider - 副本\ztb-information-spider\papernews_spider\Module\PaddleRun.py
+#Linuxpaddle
+#paddle-dirdirectory: /opt/ztb-information-spider/papernews_spider/Module/PaddleRun.py
+
 
 #登录请求url
 login-request: https://api-ops-ztb-test.qniao.cn/ztb-supply-chain-service/admin/login