更改了包名

.gitignore

@@ -0,0 +1,37 @@
+HELP.md
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
+/.mvn/
+/logs/
+*.cmd
+mvnw

pom.xml

@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.qniao</groupId>
+        <artifactId>java-dependency</artifactId>
+        <version>0.0.1-SNAPSHOT</version>
+    </parent>
+    <artifactId>ztb-spider-handle</artifactId>
+    <version>0.0.1-SNAPSHOT</version>
+    <name>ztb-spider-handle</name>
+    <description>ztb-spider-handle</description>
+    <properties>
+        <java.version>1.8</java.version>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>com.qniao</groupId>
+            <artifactId>spring-boot-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.qniao</groupId>
+            <artifactId>mybatis-plus-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.qniao</groupId>
+            <artifactId>xxl-job-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.swagger</groupId>
+            <artifactId>swagger-annotations</artifactId>
+        </dependency>
+        <!-- Hu Tool -->
+        <dependency>
+            <groupId>cn.hutool</groupId>
+            <artifactId>hutool-all</artifactId>
+            <version>5.8.12</version>
+        </dependency>
+        <dependency>
+            <groupId>cn.hutool</groupId>
+            <artifactId>hutool-json</artifactId>
+            <version>5.8.12</version>
+        </dependency>
+    </dependencies>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+    <distributionManagement>
+        <repository>
+            <id>maven-releases</id>
+            <name>Nexus releases Repository</name>
+            <url>http://120.78.76.88:8081/repository/maven-snapshots/</url>
+        </repository>
+
+    </distributionManagement>
+
+</project>

src/main/java/com/qniao/zsh/ZtbSpiderHandleApplication.java

@@ -0,0 +1,20 @@
+package com.qniao.zsh;
+
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+/**
+ * @author wh
+ * @date 2023/2/21
+ * 启动主类
+ */
+@SpringBootApplication(scanBasePackages = {"com.qniao.zsh"})
+@MapperScan("com.qniao.**.dao.**")
+public class ZtbSpiderHandleApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(ZtbSpiderHandleApplication.class, args);
+    }
+
+}

src/main/java/com/qniao/zsh/api/command/createspidera/admin/CreateSpiderAdminCommandController.java

@@ -0,0 +1,29 @@
+package com.qniao.zsh.api.command.createspidera.admin;
+
+import com.qniao.zsh.application.service.createspider.CreateSpiderApplicationService;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.annotation.Resource;
+
+/**
+ * @author wh
+ * @date 2023/3/1
+ */
+@RestController
+@RequestMapping("admin")
+public class CreateSpiderAdminCommandController {
+    @Resource
+    private CreateSpiderApplicationService createSpiderApplicationService;
+
+    @ApiOperation("创建爬虫")
+    @PostMapping ("/create/spider/{spiderName}")
+    public String create(@PathVariable String spiderName){
+
+        return createSpiderApplicationService.create(spiderName);
+
+    }
+}

src/main/java/com/qniao/zsh/api/command/startspider/admin/SpiderStateAdminCommandController.java

@@ -0,0 +1,30 @@
+package com.qniao.zsh.api.command.startspider.admin;
+
+import com.qniao.zsh.application.service.startspider.SpiderStateApplicationService;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.web.bind.annotation.*;
+
+import javax.annotation.Resource;
+
+
+/**
+ * @author wh
+ * @date 2023/2/21
+ */
+@RestController
+@RequestMapping("admin")
+public class SpiderStateAdminCommandController {
+
+    @Resource
+    private SpiderStateApplicationService spiderStateApplicationService;
+
+    @ApiOperation("启动爬虫")
+    @PostMapping("/run/spider/{spiderName}")
+    public String run(@PathVariable String spiderName) throws Exception {
+
+        return spiderStateApplicationService.run(spiderName);
+
+    }
+}
+
+

src/main/java/com/qniao/zsh/application/service/createspider/CreateSpiderApplicationService.java

@@ -0,0 +1,27 @@
+package com.qniao.zsh.application.service.createspider;
+
+
+import cn.hutool.json.JSONUtil;
+import com.qniao.zsh.domain.aggregate.spiderstate.entity.SpiderState;
+import com.qniao.zsh.domain.aggregate.spiderstate.repository.SpiderStateRepository;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+
+/**
+ * @author wh
+ * @date 2023/3/1
+ */
+@Service
+public class CreateSpiderApplicationService {
+
+    @Resource
+    private SpiderStateRepository spiderStateRepository;
+    public String create(String spiderName){
+        SpiderState spiderState = new SpiderState();
+        spiderState.setSpiderName(spiderName);
+        spiderStateRepository.save(spiderState);
+        return JSONUtil.toJsonStr(spiderStateRepository.load(spiderName));
+    }
+
+}

src/main/java/com/qniao/zsh/application/service/startspider/SpiderStateApplicationService.java

@@ -0,0 +1,50 @@
+package com.qniao.zsh.application.service.startspider;
+
+import cn.hutool.json.JSONObject;
+import cn.hutool.json.JSONUtil;
+import com.qniao.zsh.domain.aggregate.spiderstate.entity.SpiderState;
+import com.qniao.zsh.domain.aggregate.spiderstate.repository.SpiderStateRepository;
+import com.qniao.zsh.infrastructure.util.LinuxCommandUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+
+
+
+/**
+ * @author wh
+ * @date 2023/2/21
+ * Linux 启动爬虫
+ */
+@Slf4j
+@Service
+public class SpiderStateApplicationService {
+
+
+
+    @Resource
+    private SpiderStateRepository spiderStateRepository;
+
+
+    public String run(String spiderName) throws Exception {
+
+        //获取实体
+        SpiderState spiderState = spiderStateRepository.load(spiderName);
+        //运行爬虫并返回结果
+        String retStrBuilder = LinuxCommandUtil.startScrapy(spiderName);
+        //设置爬虫状态
+        if (JSONUtil.isTypeJSON(retStrBuilder)) {
+            JSONObject jsonObject = JSONUtil.parseObj(retStrBuilder);
+            log.info(String.valueOf(jsonObject));
+            spiderState.setStatus(1);
+        } else {
+            spiderState.setStatus(2);
+            log.info(retStrBuilder);
+        }
+        //更新数据库
+        spiderStateRepository.save(spiderState);
+
+        return retStrBuilder;
+    }
+}

src/main/java/com/qniao/zsh/domain/aggregate/spiderstate/SpiderStateAggregate.java

@@ -0,0 +1,23 @@
+package com.qniao.zsh.domain.aggregate.spiderstate;
+
+import com.qniao.zsh.domain.aggregate.spiderstate.entity.SpiderState;
+import com.qniao.zsh.domain.aggregate.spiderstate.repository.SpiderStateRepository;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+
+/**
+ * @author wh
+ * 2023/2/22
+ */
+@Service
+public class SpiderStateAggregate {
+
+    @Resource
+    private SpiderStateRepository spiderStateRepository;
+
+    public void save (SpiderState spiderState) {
+        spiderStateRepository.save(spiderState);
+    }
+
+}

src/main/java/com/qniao/zsh/domain/aggregate/spiderstate/entity/SpiderState.java

@@ -0,0 +1,29 @@
+package com.qniao.zsh.domain.aggregate.spiderstate.entity;
+
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.qniao.domain.Entity;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import io.swagger.annotations.ApiModelProperty;
+
+/**
+ * @author Administrator
+ * @date 2023/2/21
+ *爬虫状态数据库
+ */
+@Data
+@EqualsAndHashCode(callSuper = true)
+@TableName("spider_state")
+public class SpiderState extends Entity<SpiderState> {
+
+    @ApiModelProperty("爬虫名字")
+    private String  spiderName;
+
+    @ApiModelProperty("爬取周期 默认 15 分钟")
+    private Integer cycle;
+
+    @ApiModelProperty("状态 1-爬取成功 2-爬取失败 3-停止")
+    private Integer status;
+
+
+}

src/main/java/com/qniao/zsh/domain/aggregate/spiderstate/repository/SpiderStateRepository.java

@@ -0,0 +1,19 @@
+package com.qniao.zsh.domain.aggregate.spiderstate.repository;
+
+import com.qniao.zsh.domain.aggregate.spiderstate.entity.SpiderState;
+import com.qniao.domain.Repository;
+
+
+/**
+ * @author wh
+ * @date 2023/2/21
+ */
+public interface SpiderStateRepository extends Repository<SpiderState, Long> {
+
+    /**
+     * 用爬虫名字来读取数据库爬虫
+     * @param spiderName 爬虫名字
+     * @return 爬虫实体
+     */
+    SpiderState load(String spiderName);
+}

src/main/java/com/qniao/zsh/infrastructure/config/SpiderDirectoryConfig.java

@@ -0,0 +1,27 @@
+package com.qniao.zsh.infrastructure.config;
+
+
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * @author wh
+ * @date 2023/2/28
+ */
+
+@Configuration
+public class SpiderDirectoryConfig implements InitializingBean {
+
+    @Value("${spier-dirdirectory}")
+    private String dir;
+
+    public static String DIR;
+
+    @Override
+    public void afterPropertiesSet() {
+        DIR = dir;
+
+
+    }
+}

src/main/java/com/qniao/zsh/infrastructure/persistence/dao/SpiderStateDao.java

@@ -0,0 +1,14 @@
+package com.qniao.zsh.infrastructure.persistence.dao;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.qniao.zsh.domain.aggregate.spiderstate.entity.SpiderState;
+
+
+/**
+ * @author wh
+ * @date 2023/2/22
+ */
+public interface SpiderStateDao extends BaseMapper<SpiderState> {
+
+
+}

src/main/java/com/qniao/zsh/infrastructure/persistence/repository/impl/SpiderState/SpiderStateRepositoryImpl.java

@@ -0,0 +1,44 @@
+package com.qniao.zsh.infrastructure.persistence.repository.impl.SpiderState;
+
+
+import com.baomidou.mybatisplus.core.toolkit.Wrappers;
+import com.qniao.zsh.domain.aggregate.spiderstate.entity.SpiderState;
+import com.qniao.zsh.domain.aggregate.spiderstate.repository.SpiderStateRepository;
+import com.qniao.zsh.infrastructure.persistence.dao.SpiderStateDao;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+import java.util.Objects;
+
+/**
+ * @author wh
+ * @date 2023/2/22
+ */
+@Service
+public class SpiderStateRepositoryImpl implements SpiderStateRepository {
+
+    @Resource
+    private SpiderStateDao spiderStateDao;
+
+
+    @Override
+    public SpiderState load(Long id) {
+        return spiderStateDao.selectById(id);
+    }
+    @Override
+    public SpiderState load(String spiderName) {
+        return spiderStateDao.selectOne(Wrappers.lambdaQuery(new SpiderState()).eq(SpiderState::getSpiderName, spiderName));
+    }
+
+    @Override
+    public Long save(SpiderState entity) {
+        if (!Objects.isNull(entity.getSpiderName())) {
+            spiderStateDao.updateById(entity);
+        } else {
+            spiderStateDao.insert(entity);
+        }
+        return entity.getId();
+    }
+
+
+}

src/main/java/com/qniao/zsh/infrastructure/util/LinuxCommandUtil.java

@@ -0,0 +1,57 @@
+package com.qniao.zsh.infrastructure.util;
+
+import cn.hutool.core.text.StrBuilder;
+import cn.hutool.core.util.StrUtil;
+import com.qniao.zsh.infrastructure.config.SpiderDirectoryConfig;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * @author wh
+ * @date 2023/3/1
+ */
+
+@Slf4j
+public class LinuxCommandUtil {
+    /**
+     * 运行scrapy的工具类
+     * @param spiderName 爬虫名
+     * @return 控制台输出
+     * @throws IOException 线程
+     */
+    public static String startScrapy(String spiderName) throws IOException {
+        // 要执行的shell命令
+        String scrapy = " scrapy crawl {} -s LOG_FILE=spider-debug.log ";
+        String command = StrUtil.format(scrapy,spiderName);
+        // 创建ProcessBuilder对象
+        ProcessBuilder builder = new ProcessBuilder("/bin/sh", "-c", command);
+        //配置工作目录
+        builder.directory(new File(SpiderDirectoryConfig.DIR));
+        // 重定向标准输出和标准错误输出到Java进程
+        builder.redirectErrorStream(true);
+
+        // 启动进程
+        Process process = builder.start();
+
+
+        // 读取进程输出
+        StrBuilder retStrBuilder = StrBuilder.create();
+        BufferedReader processOutputReader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8));
+        String readLine;
+        while ((readLine = processOutputReader.readLine()) != null) {
+            retStrBuilder.append(readLine).append(System.lineSeparator());
+            log.info(readLine);
+
+        }
+
+        return retStrBuilder.toString();
+
+
+
+    }
+}

src/main/resources/application-local.yml

@@ -0,0 +1,56 @@
+mysql-database-name: springboot
+mysql-database-url: 52.81.89.46:3306
+mysql-user-name: root
+mysql-user-password: 704177820
+redis-database-index: 2
+redis-host: 8.135.8.221
+redis-port: 6379
+eureka-enable: false
+register-eureka-url: http://eureka:123456@127.0.0.1:8000/eureka/
+message-push-host: http://8.135.8.221:7050
+redis-password: qnredis
+fake_captcha_enable: true
+ rabbitmq
+rabbitmq-virtual-host: /
+rabbitmq-host: 8.135.8.221
+rabbitmq-port: 5672
+rabbitmq-username: qniao
+rabbitmq-password: Qianniao2020
+eureka.instance.ip-address: 47.113.118.47
+
+# 服务器文件存放路径
+server-file-path: D:\Qniao
+
+xxl-job-executor-ip: 127.0.0.1
+
+
+
+
+
+#xxl-job
+#logging.config: classpath:logback.xml
+### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
+# 调度中心部署跟地址 [选填]：如调度中心集群部署存在多个地址则用逗号分隔。执行器将会使用该地址进行"执行器心跳注册"和"任务结果回调"；为空则关闭自动注册；
+#xxl.job.admin.addresses: http://127.0.0.1:8808/xxl-job-admin
+xxl.job.admin.addresses: http://xxl-job-admin-dev.qniao.cn/xxl-job-admin
+# 执行器AppName [选填]：执行器心跳注册分组依据；为空则关闭自动注册
+xxl.job.executor.appname: ztb-factory-dev
+# 执行器端口号 [选填]：小于等于0则自动获取；默认端口为9999，单机部署多个执行器时，注意要配置不同执行器端口；
+xxl.job.executor.port: 9999
+### xxl-job executor log-path
+#xxl.job.executor.logpath: /data/applogs/xxl-job/jobhandler
+### xxl-job executor log-retention-days
+xxl.job.executor.logretentiondays: 30
+
+#开启驼峰命名规范自动映射
+configuration:
+ map-underscore-to-camel-case: true
+
+#win
+#spier-dirdirectory: D:\Project\papernews_spider
+#Linux
+spier-dirdirectory: /opt/ztb-information-spider/papernews_spider
+
+
+
+

src/main/resources/application.yml

@@ -0,0 +1,42 @@
+server:
+  port: 7023
+
+
+server-file-path: /home/files/
+
+xxl:
+  job:
+    admin:
+      # 调度中心部署跟地址 [选填]：如调度中心集群部署存在多个地址则用逗号分隔。执行器将会使用该地址进行"执行器心跳注册"和"任务结果回调"；为空则关闭自动注册；
+      addresses: http://xxl-job-admin-dev.qniao.cn/xxl-job-admin/
+    executor:
+      # 执行器AppName [选填]：执行器心跳注册分组依据；为空则关闭自动注册
+      appname: ztb-factory-test
+      # 执行器端口号 [选填]：小于等于0则自动获取；默认端口为9999，单机部署多个执行器时，注意要配置不同执行器端口；
+      port: 9788
+      # 执行器日志文件保存天数 [选填] ： 过期日志自动清理, 限制值大于等于3时生效; 否则, 如-1, 关闭自动清理功能；
+      logretentiondays: 30
+      # 当前服务器IP
+      #      ip: 120.79.198.246
+      ip: 120.79.170.129
+
+
+spring:
+  profiles:
+    active: local
+  datasource:
+    url: jdbc:mysql://52.81.89.46:3306/springboot?autoReconnect=true&useUnicode=true&characterEncoding=utf8&useTimezone=true&serverTimezone=GMT%2B8&useSSL=false&allowMultiQueries=true
+    #useSSL安全连接 useUnicode编码 characterEncoding编码格式 serverTimezone时区
+    username: root
+    password: 704177820
+    driver-class-name: com.mysql.cj.jdbc.Driver
+#配置日志
+mybatis-plus:
+  configuration:
+    log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
+
+#开启驼峰命名规范自动映射
+configuration:
+  map-underscore-to-camel-case: true
+
+

src/main/resources/db.setting

@@ -0,0 +1,13 @@
+url = jdbc:mysql://52.81.89.46:3306/springboot
+user = root
+pass = 704177820
+
+## 可选配置
+# 是否在日志中显示执行的SQL
+showSql = true
+# 是否格式化显示的SQL
+formatSql = false
+# 是否显示SQL参数
+showParams = true
+# 打印SQL的日志等级，默认debug，可以是info、warn、error
+sqlLevel = debug

src/main/resources/logback.xml

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+
+    <appender name="system"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>./logs/log.log</file>
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>INFO</level>
+        </filter>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>./logs/log.%d{yyyy-MM-dd}.log</fileNamePattern>
+            <maxHistory>15</maxHistory>
+        </rollingPolicy>
+        <encoder>
+            <pattern>[%d{yyyy-MM-dd HH:mm:ss:SSS}] %thread %level %logger{36} %L - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="errorFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <!-- ThresholdFilter:临界值过滤器，过滤掉 TRACE 和 DEBUG 级别的日志 -->
+        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+            <level>ERROR</level>
+        </filter>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>./logs/error.%d{yyyy-MM-dd}.log</fileNamePattern>
+            <maxHistory>15</maxHistory><!--保存最近30天的日志-->
+        </rollingPolicy>
+        <encoder>
+            <charset>UTF-8</charset>
+            <pattern>[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%thread] %-5level %logger{36} : %msg%n</pattern>
+        </encoder>
+    </appender>
+    <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>[%d{yyyy-MM-dd HH:mm:ss:SSS}] %thread %level %logger{36} %L - %msg%n</pattern>
+            <charset>UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <logger name="com.qniao" level="info" additivity="false">
+        <appender-ref ref="stdout"/>
+        <appender-ref ref="system"/>
+        <appender-ref ref="errorFile"/>
+
+    </logger>
+
+    <logger name="org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping" level="info"
+            additivity="false">
+        <appender-ref ref="stdout"/>
+    </logger>
+
+    <logger name="org.springframework.orm.jpa.JpaTransactionManager" level="debug"
+            additivity="false">
+        <appender-ref ref="stdout"/>
+    </logger>
+
+    <logger name="org.hibernate.type.descriptor.sql.BasicBinder" level="trace"
+            additivity="false">
+        <appender-ref ref="stdout"/>
+    </logger>
+
+    <root level="info">
+        <appender-ref ref="stdout"/>
+
+    </root>
+</configuration>

src/test/java/com/qniao/zsh/ZtbSpiderHandleApplicationTests.java

@@ -0,0 +1,13 @@
+package com.qniao.zsh;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+
+@SpringBootTest
+class ZtbSpiderHandleApplicationTests {
+
+    @Test
+    void contextLoads() {
+    }
+
+}