diff --git a/src/main/java/com/qniao/zsh/api/command/paddle/admin/PaddleAdminCommandController.java b/src/main/java/com/qniao/zsh/api/command/paddle/admin/PaddleAdminCommandController.java index 4c15b0e..ed8f89e 100644 --- a/src/main/java/com/qniao/zsh/api/command/paddle/admin/PaddleAdminCommandController.java +++ b/src/main/java/com/qniao/zsh/api/command/paddle/admin/PaddleAdminCommandController.java @@ -25,7 +25,7 @@ public class PaddleAdminCommandController { @ApiOperation("飞桨文本解析") @PostMapping("/analysis") - public PaddleVo analysis(@RequestBody PaddleDto paddleDto) throws IOException { + public PaddleVo analysis(@RequestBody PaddleDto paddleDto) throws IOException, InterruptedException { return paddleApplicationService.runPaddle(paddleDto); } diff --git a/src/main/java/com/qniao/zsh/api/command/xxljob/admin/XxlJobAdminCommandController.java b/src/main/java/com/qniao/zsh/api/command/xxljob/admin/XxlJobAdminCommandController.java new file mode 100644 index 0000000..59725fd --- /dev/null +++ b/src/main/java/com/qniao/zsh/api/command/xxljob/admin/XxlJobAdminCommandController.java @@ -0,0 +1,14 @@ +package com.qniao.zsh.api.command.xxljob.admin; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * @author wh + * @date 2023/4/23 + */ +@RestController +@RequestMapping("admin/xxl-job") +public class XxlJobAdminCommandController { + +} diff --git a/src/main/java/com/qniao/zsh/application/service/createspider/CreateSpiderApplicationService.java b/src/main/java/com/qniao/zsh/application/service/createspider/CreateSpiderApplicationService.java index e15c5eb..661813e 100644 --- a/src/main/java/com/qniao/zsh/application/service/createspider/CreateSpiderApplicationService.java +++ b/src/main/java/com/qniao/zsh/application/service/createspider/CreateSpiderApplicationService.java @@ -1,5 +1,6 @@ package com.qniao.zsh.application.service.createspider; +import com.qniao.zsh.application.service.xxljob.XxlJobApplicationService; import com.qniao.zsh.domain.aggregate.spiderstate.entity.SpiderState; import com.qniao.zsh.domain.aggregate.spiderstate.repository.SpiderStateRepository; import lombok.extern.slf4j.Slf4j; @@ -16,11 +17,15 @@ public class CreateSpiderApplicationService { @Resource private SpiderStateRepository spiderStateRepository; + @Resource + private XxlJobApplicationService xxlJobApplicationService; public Long create(String spiderName){ SpiderState spiderState = new SpiderState(); spiderState.setSpiderName(spiderName); - return spiderStateRepository.save(spiderState); + Long id = spiderStateRepository.save(spiderState); + xxlJobApplicationService.add(spiderName); + return id; } diff --git a/src/main/java/com/qniao/zsh/application/service/paddle/PaddleApplicationService.java b/src/main/java/com/qniao/zsh/application/service/paddle/PaddleApplicationService.java index 4599efe..fcce3ce 100644 --- a/src/main/java/com/qniao/zsh/application/service/paddle/PaddleApplicationService.java +++ b/src/main/java/com/qniao/zsh/application/service/paddle/PaddleApplicationService.java @@ -19,7 +19,7 @@ import java.util.List; @Service public class PaddleApplicationService { - public PaddleVo runPaddle(PaddleDto paddleDto) throws IOException { + public PaddleVo runPaddle(PaddleDto paddleDto) throws IOException, InterruptedException { //linux // String result = ScriptRunnerHelper.handle("python3", //win @@ -29,7 +29,7 @@ public class PaddleApplicationService { paddleDto.getArticleId(), paddleDto.getPaperMillId(), paddleDto.getContent())); - String[] strings = result.split("\r\n"); + String[] strings = result.split("\n"); result = strings[strings.length - 1]; JSONObject jsonResult = JSONUtil.parseObj(result); PaddleVo paddleVo = new PaddleVo(); diff --git a/src/main/java/com/qniao/zsh/application/service/xxljob/XxlJobApplicationService.java b/src/main/java/com/qniao/zsh/application/service/xxljob/XxlJobApplicationService.java new file mode 100644 index 0000000..d08b6c4 --- /dev/null +++ b/src/main/java/com/qniao/zsh/application/service/xxljob/XxlJobApplicationService.java @@ -0,0 +1,91 @@ +package com.qniao.zsh.application.service.xxljob; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import cn.hutool.http.HttpStatus; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import com.qniao.zsh.infrastructure.config.LoginRequestConfig; +import com.xxl.job.core.biz.AdminBiz; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author wh + * @date 2023/4/23 + */ + +@Service +public class XxlJobApplicationService { + public int login() { + Map paramMap = new HashMap<>(2); + paramMap.put("userName", "admin"); + paramMap.put("password", "123456"); + paramMap.put("ifRemember", "on"); + HttpResponse response = HttpRequest.post("http://localhost:8080/xxl-job-admin/login/") + .form(paramMap) + .timeout(20000) + .execute(); + + + String body = response.body(); + JSONObject jsonInfo = JSONUtil.parseObj(body); + + return Integer.parseInt(jsonInfo.getStr("code")); + + + } + + + public int add(String jobDesc) { + login(); + String cron = "* 0/15 * * * ?"; + Map paramMap = new HashMap<>(2); + paramMap.put("jobGroup", 2); + paramMap.put("jobDesc", jobDesc); + paramMap.put("author", "wh"); + paramMap.put("scheduleType", "CRON"); + paramMap.put("scheduleConf", cron); + paramMap.put("misfireStrategy", "DO_NOTHING"); + paramMap.put("executorRouteStrategy", "FIRST"); + paramMap.put("executorHandler", "spiderTest"); + paramMap.put("executorBlockStrategy", "SERIAL_EXECUTION"); + paramMap.put("glueType", "BEAN"); + HttpResponse response = HttpRequest.post("http://localhost:8080/xxl-job-admin/jobinfo/add") + .form(paramMap) + .timeout(20000) + .execute(); + String body = response.body(); + JSONObject jsonInfo = JSONUtil.parseObj(body); + + return Integer.parseInt(jsonInfo.getStr("code")); + } + + public int update(String jobDesc) { + login(); + String cron = "* 0/20 * * * ?"; + Map paramMap = new HashMap<>(2); + paramMap.put("jobGroup", 2); + paramMap.put("jobDesc", jobDesc); + paramMap.put("author", "wh"); + paramMap.put("scheduleType", "CRON"); + paramMap.put("scheduleConf", cron); + paramMap.put("misfireStrategy", "DO_NOTHING"); + paramMap.put("executorRouteStrategy", "FIRST"); + paramMap.put("executorHandler", "spiderTest"); + paramMap.put("executorBlockStrategy", "SERIAL_EXECUTION"); + paramMap.put("glueType", "BEAN"); + HttpResponse response = HttpRequest.post("http://localhost:8080/xxl-job-admin/jobinfo/add") + .form(paramMap) + .timeout(20000) + .execute(); + String body = response.body(); + JSONObject jsonInfo = JSONUtil.parseObj(body); + + return Integer.parseInt(jsonInfo.getStr("code")); + } + + +} diff --git a/src/main/java/com/qniao/zsh/infrastructure/util/LinuxCommandUtil.java b/src/main/java/com/qniao/zsh/infrastructure/util/LinuxCommandUtil.java index ffc842d..a5364fc 100644 --- a/src/main/java/com/qniao/zsh/infrastructure/util/LinuxCommandUtil.java +++ b/src/main/java/com/qniao/zsh/infrastructure/util/LinuxCommandUtil.java @@ -4,7 +4,10 @@ import cn.hutool.core.lang.Snowflake; import cn.hutool.core.text.StrBuilder; import cn.hutool.core.util.StrUtil; import com.qniao.zsh.infrastructure.config.SpiderDirectoryConfig; +import com.xxl.job.core.context.XxlJobHelper; +import com.xxl.job.core.handler.annotation.XxlJob; import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Scope; import java.io.BufferedReader; import java.io.File; @@ -18,17 +21,23 @@ import java.nio.charset.StandardCharsets; */ @Slf4j +@Scope("prototype") public class LinuxCommandUtil { /** * 运行scrapy的工具类 + * * @param spiderName 爬虫名 * @return 控制台输出 * @throws IOException 线程 */ + @XxlJob("runScrapy") public static String startScrapy(String spiderName) throws IOException { + if (spiderName == null) { + spiderName = XxlJobHelper.getJobParam(); + } // 要执行的shell命令 String scrapy = " scrapy crawl {} -s LOG_FILE=spider-debug.log "; - String command = StrUtil.format(scrapy,spiderName); + String command = StrUtil.format(scrapy, spiderName); // 创建ProcessBuilder对象 ProcessBuilder builder = new ProcessBuilder("/bin/sh", "-c", command); //配置工作目录 @@ -53,6 +62,5 @@ public class LinuxCommandUtil { return retStrBuilder.toString(); - } } diff --git a/src/main/java/com/qniao/zsh/infrastructure/util/xxl/XxlJobDemo.java b/src/main/java/com/qniao/zsh/infrastructure/util/xxl/XxlJobDemo.java index 8164cd0..7209216 100644 --- a/src/main/java/com/qniao/zsh/infrastructure/util/xxl/XxlJobDemo.java +++ b/src/main/java/com/qniao/zsh/infrastructure/util/xxl/XxlJobDemo.java @@ -1,6 +1,7 @@ package com.qniao.zsh.infrastructure.util.xxl; import com.xxl.job.core.biz.model.ReturnT; +import com.xxl.job.core.context.XxlJobHelper; import com.xxl.job.core.handler.annotation.XxlJob; import org.springframework.stereotype.Component; @@ -8,9 +9,9 @@ import org.springframework.stereotype.Component; public class XxlJobDemo { @XxlJob("myJobHandler") - public ReturnT myJobHandler(String param) throws Exception { - // your jobHandler code here - return ReturnT.SUCCESS; + public void test() { + String jobParamStr = XxlJobHelper.getJobParam(); + System.out.println("成功调取：" + jobParamStr); } } diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index 22761ee..ede58e3 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -30,7 +30,7 @@ xxl-job-executor-ip: 127.0.0.1 ### xxl-job admin address list, such as "http://address" or "http://address01,http://address02" # 调度中心部署跟地址 [选填]：如调度中心集群部署存在多个地址则用逗号分隔。执行器将会使用该地址进行"执行器心跳注册"和"任务结果回调"；为空则关闭自动注册； #xxl.job.admin.addresses: http://127.0.0.1:8808/xxl-job-admin -xxl.job.admin.addresses: http://xxl-job-admin-dev.qniao.cn/xxl-job-admin +xxl.job.admin.addresses: http://192.168.10.198/xxl-job-admin # 执行器AppName [选填]：执行器心跳注册分组依据；为空则关闭自动注册 xxl.job.executor.appname: ztb-factory-dev # 执行器端口号 [选填]：小于等于0则自动获取；默认端口为9999，单机部署多个执行器时，注意要配置不同执行器端口；