diff --git a/README.md b/README.md new file mode 100644 index 0000000..7e8ed3c --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +## 爬虫管理server +### 需要配置启动设置 +#### 1.VM options: +-Ddev_meta=http://8.135.8.221:5000 -Dapollo.configService=http://8.135.8.221:5000 + +#### 2.Program arguments: +--spring.profiles.active=test --eureka.client.register-with-eureka=false \ No newline at end of file diff --git a/pom.xml b/pom.xml index eeb3a3b..914e219 100644 --- a/pom.xml +++ b/pom.xml @@ -2,12 +2,18 @@ 4.0.0 - + + <!– lookup parent from repository –> + --> + + com.qniao + java-dependency + 0.0.1-SNAPSHOT + ztb-spider-handle 0.0.1-SNAPSHOT ztb-spider-handle @@ -121,11 +127,23 @@ 0.0.1-SNAPSHOT compile + + + + com.ctrip.framework.apollo + apollo-client + + com.qniao - employee-admin-common - 1.0.12-SNAPSHOT + spring-boot-starter + + com.qniao + spring-cloud-starter + + + diff --git a/src/main/java/com/qniao/zsh/ZtbSpiderHandleApplication.java b/src/main/java/com/qniao/zsh/ZtbSpiderHandleApplication.java index 9c1eab2..2580709 100644 --- a/src/main/java/com/qniao/zsh/ZtbSpiderHandleApplication.java +++ b/src/main/java/com/qniao/zsh/ZtbSpiderHandleApplication.java @@ -1,14 +1,18 @@ package com.qniao.zsh; +import com.ctrip.framework.apollo.spring.annotation.EnableApolloConfig; import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.cloud.netflix.eureka.EnableEurekaClient; /** * @author wh * @date 2023/2/21 * 启动主类 */ +@EnableApolloConfig +@EnableEurekaClient @SpringBootApplication(scanBasePackages = {"org.mc.ddd", "com.qniao.zsh"}) @MapperScan("com.qniao.zsh.infrastructure.dao") public class ZtbSpiderHandleApplication { diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index f34d929..8ab4fa7 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -1,25 +1,23 @@ mysql-database-name: springboot -mysql-database-url: 52.81.89.46:3306 +mysql-database-url: jdbc:mysql://52.81.89.46:3306/springboot?autoReconnect=true&useUnicode=true&characterEncoding=utf8&useTimezone=true&serverTimezone=GMT%2B8&useSSL=false&allowMultiQueries=true mysql-user-name: root mysql-user-password: 704177820 redis-database-index: 2 redis-host: 8.135.8.221 redis-port: 6379 -eureka-enable: false -register-eureka-url: http://eureka:123456@127.0.0.1:8000/eureka/ +eureka-enable : true +register-eureka-url : http://eureka:123456@8.135.8.221:8000/eureka/ message-push-host: http://8.135.8.221:7050 redis-password: qnredis fake_captcha_enable: true - rabbitmq -rabbitmq-virtual-host: / +rabbitmq-virtual-host: /rabbitmq rabbitmq-host: 8.135.8.221 rabbitmq-port: 5672 rabbitmq-username: qniao rabbitmq-password: Qianniao2020 eureka.instance.ip-address: 47.113.118.47 -# 服务器文件存放路径 -server-file-path: D:\Qniao + xxl-job-executor-ip: 127.0.0.1 @@ -42,9 +40,6 @@ xxl.job.executor.port: 9999 ### xxl-job executor log-retention-days xxl.job.executor.logretentiondays: 30 -#开启驼峰命名规范自动映射 -configuration: - map-underscore-to-camel-case: true #win #spier-dirdirectory: D:\Project\papernews_spider diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 59c92dc..c96241c 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -6,32 +6,29 @@ application: doc: enabled: true selector: org.mc.oas.api -server: - port: 7023 -server-file-path: /home/files/ -xxl: - job: - admin: - # 调度中心部署跟地址 [选填]：如调度中心集群部署存在多个地址则用逗号分隔。执行器将会使用该地址进行"执行器心跳注册"和"任务结果回调"；为空则关闭自动注册； - addresses: http://xxl-job-admin-dev.qniao.cn/xxl-job-admin/ - executor: - # 执行器AppName [选填]：执行器心跳注册分组依据；为空则关闭自动注册 - appname: ztb-factory-test - # 执行器端口号 [选填]：小于等于0则自动获取；默认端口为9999，单机部署多个执行器时，注意要配置不同执行器端口； - port: 9788 - # 执行器日志文件保存天数 [选填] ： 过期日志自动清理, 限制值大于等于3时生效; 否则, 如-1, 关闭自动清理功能； - logretentiondays: 30 - # 当前服务器IP - # ip: 120.79.198.246 - ip: 120.79.170.129 +#开启驼峰命名规范自动映射 +configuration: + map-underscore-to-camel-case: true + + +server: + port: 7023 + servlet: + context-path: /ztb-spider-service + spring: profiles: active: local + application: + name: eureka-client-ztb-spider-service + version: 1.0 + description: 千鸟数据中心 + api-doc-enabled: true datasource: driver-class-name: com.mysql.cj.jdbc.Driver dynamic: @@ -48,7 +45,47 @@ spring: maximum-pool-size: 10 max-lifetime: 30000 connection-test-query: SELECT 1 + hikari: + minimum-idle: 3 + maximum-pool-size: 10 + max-lifetime: 30000 + connection-test-query: SELECT 1 + + redis: + database: ${redis-database-index} + host: ${redis-host} + port: ${redis-port} + password: ${redis-password} + timeout: 3000 + lettuce: + pool: + max-active: 200 + max-idle: 20 + min-idle: 10 + max-wait: -1 + servlet: + multipart: + enabled: true + max-file-size: 100MB + max-request-size: 105MB + rabbitmq: + virtual-host: ${rabbitmq-virtual-host} + host: ${rabbitmq-host} + port: ${rabbitmq-port} + username: ${rabbitmq-username} + password: ${rabbitmq-password} + listener: + simple: + retry: + enabled: true + max-attempts: 3 + initial-interval: 1000 + default-requeue-rejected: false + +mybatis: + mapper-locations: classpath:mapping/*Mapper.xml + type-aliases-package: com.qniao.**.entity # mybatis-plus配置控制台打印完整带参数SQL语句 mybatis-plus: @@ -61,13 +98,125 @@ mybatis-plus: configuration: log-impl: org.apache.ibatis.logging.stdout.StdOutImpl default-enum-type-handler: org.apache.ibatis.type.EnumOrdinalTypeHandler #支持数据库的枚举型转换 + map-underscore-to-camel-case: true + + +eureka: + client: + register-with-eureka: ${eureka-enable} + fetch-registry: ${eureka-enable} + service-url: + defaultZone: ${register-eureka-url} + instance: + prefer-ip-address: true + instance-id: ${spring.application.name}:${spring.cloud.client.ip-address}:${server.port} + lease-expiration-duration-in-seconds: 15 + lease-renewal-interval-in-seconds: 5 + + +feign: + hystrix: + enabled: false + client: + config: + default: + connectTimeout: 10000 + readTimeout: 60000 + +# DefaultClientConfigImpl +ribbon: + ConnectTimeout: 8000 + ReadTimeout: 20000 + +management: + endpoints: + web: + exposure: + include: "*" + endpoint: + shutdown: + enabled: true + health: + enabled: true + show-details: always + +logging: + config: classpath:logback.xml + snow-flake: datacenter-id: 1 - worker-id: 1 -debug: false + machine-id: 1 + +app: + id: ztb-spider-service +# Custom.Security +employee: + event: + listenerQueue: ${employee-event-listenerQueue} + exchageCode: ${employee-event-exchageCode} + custom: + security: + loginUrl: ${employee-Custom-loginUrl} + loginProcessingUrl: ${employee-Custom-loginProcessingUrl} + logout: ${employee-Custom-logout} + tokenAuthorizeRequests: ${employee-Custom-tokenAuthorizeRequests} + + +# 不需要拦截的请求url +request-pass-url: /swagger-ui.html||/webjars/**||/v2/**||/swagger-resources** + +x-app-id-pass-url: /utils/uploadImage||/utils/uploadImages||/utils/uploadFile||/delete/department-member-role/by-enterprise-member-id||/get/factory/user-permission-type||/recycle-service-feign/add/enterprise-profile + +ding-talk-alarm-robot-access-token: e10f76d44fa2b01a0d00e88ac256366577f6ead9511a0a92c40fbe32a2687d94 +message-push-host: ${message-push-host} + +event-bus: + thread-pool: + core-pool-size: 2 + max-pool-size: 2 + keep-alive-time: 2 # 单位: 秒 + allow-core-thread-timeOut: false + +xxl: + job: + admin: + # 调度中心部署跟地址 [选填]：如调度中心集群部署存在多个地址则用逗号分隔。执行器将会使用该地址进行"执行器心跳注册"和"任务结果回调"；为空则关闭自动注册； + addresses: ${xxl-job-admin-addresses} + executor: + # 执行器AppName [选填]：执行器心跳注册分组依据；为空则关闭自动注册 + appname: ${xxl-job-executor-appname} + # 执行器端口号 [选填]：小于等于0则自动获取；默认端口为9999，单机部署多个执行器时，注意要配置不同执行器端口； + port: ${xxl-job-executor-port} + # 执行器日志文件保存天数 [选填] ： 过期日志自动清理, 限制值大于等于3时生效; 否则, 如-1, 关闭自动清理功能； + logretentiondays: 30 + # 当前服务器IP + ip: ${xxl-job-executor-ip} + +# 服务器文件存放路径 +server-file-path: /home/files/ #开启驼峰命名规范自动映射 -configuration: - map-underscore-to-camel-case: true +#spring: +# profiles: +# active: local +# datasource: +# driver-class-name: com.mysql.cj.jdbc.Driver +# dynamic: +# primary: oas #设置默认的数据源或者数据源组 +# strict: true #严格匹配数据源,默认false. true未匹配到指定数据源时抛异常,false使用默认数据源 +# datasource: +# oas: +# url: jdbc:mysql://52.81.89.46:3306/springboot?autoReconnect=true&useUnicode=true&characterEncoding=utf8&useTimezone=true&serverTimezone=GMT%2B8&useSSL=false&allowMultiQueries=true +# #useSSL安全连接 useUnicode编码 characterEncoding编码格式 serverTimezone时区 +# username: root +# password: 704177820 +# hikari: +# minimum-idle: 3 +# maximum-pool-size: 10 +# max-lifetime: 30000 +# connection-test-query: SELECT 1 + + +debug: false \ No newline at end of file