Skip to content

E
etl-ruoshui-bigdata-server
提交 0dcaf7ce authored 作者: 吕本才's avatar 吕本才
浏览文件
操作

feat(告警通知): 修改邮件配置为飞书企业邮箱

优化Druid配置默认值和日志配置 清理.gitignore文件并添加.vscode目录
上级 1d4ea963
显示空白字符变更
内嵌 并排
正在显示 包含 142 行增加 和 43 行删除
.gitignore
浏览文件 @ 0dcaf7ce
...@@ -3,8 +3,9 @@ applogs/** ...@@ -3,8 +3,9 @@ applogs/**
.idea/** .idea/**
.idea .idea
.history/** .history/**
.vscode/**
**/target/ **/target/
/build/* /build/*
/derby.log /derby.log
**/**.log
ruoshui-admin/src/main/resources/application-druid.yml
浏览文件 @ 0dcaf7ce
# 开发环境配置
server:
# 服务器的HTTP端口，默认为8080
port: 10001
# 数据源配置 # 数据源配置
spring: spring:
datasource: datasource:
...@@ -9,9 +13,9 @@ spring: ...@@ -9,9 +13,9 @@ spring:
url: jdbc:mysql://192.168.100.40:3306/ruoshui_bigdata?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8 url: jdbc:mysql://192.168.100.40:3306/ruoshui_bigdata?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8
username: root username: root
password: Wxl@325Pa91 password: Wxl@325Pa91
# url: jdbc:mysql://127.0.0.1:3306/test_ruoshui_bigdata_db?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8 # url: jdbc:mysql://127.0.0.1:3306/test_ruoshui_bigdata_db?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8
# username: root # username: root
# password: 123456 # password: 123456
# 从库数据源 # 从库数据源
slave: slave:
# 从数据源开关/默认关闭 # 从数据源开关/默认关闭
...@@ -59,8 +63,6 @@ spring: ...@@ -59,8 +63,6 @@ spring:
config: config:
multi-statement-allow: true multi-statement-allow: true
# 日志配置 # 日志配置
logging: logging:
level: level:
......
ruoshui-admin/src/main/resources/application-master.yml
浏览文件 @ 0dcaf7ce
# 开发环境配置 # 开发环境配置
server: server:
# 服务器的HTTP端口，38环境10001，阿里云 10002 # 服务器的HTTP端口，38环境10001，阿里云 10002
port: 10002 port: 10001
# 数据源配置 # 数据源配置
spring: spring:
datasource: datasource:
...@@ -70,16 +70,19 @@ spring: ...@@ -70,16 +70,19 @@ spring:
port: 6379 port: 6379
# 数据库索引 # 数据库索引
# database: 3 # database: 3
database: 4 database:
4
# 密码 # 密码
# password: Wxl2025!@#$ # password: Wxl2025!@#$
password: QjL6H5nH password:
QjL6H5nH
# 连接超时时间 # 连接超时时间
timeout: 3000s timeout: 3000s
lettuce: lettuce:
pool: pool:
# 连接池中的最小空闲连接 # 连接池中的最小空闲连接
min-idle: 0 min-idle:
0
# 连接池中的最大空闲连接 # 连接池中的最大空闲连接
max-idle: 8 max-idle: 8
# 连接池的最大数据库连接数 # 连接池的最大数据库连接数
......
ruoshui-admin/src/main/resources/application-qa.yml
浏览文件 @ 0dcaf7ce
# 开发环境配置 # 开发环境配置
server: server:
# 服务器的HTTP端口，默认为8080 # 服务器的HTTP端口，默认为8080
port: 10001 port: 10002
# 数据源配置 # 数据源配置
spring: spring:
datasource: datasource:
...@@ -65,15 +65,18 @@ spring: ...@@ -65,15 +65,18 @@ spring:
# 端口，默认为6379 # 端口，默认为6379
port: 6379 port: 6379
# 数据库索引 # 数据库索引
database: 4 database:
4
# 密码 # 密码
password: QjL6H5nH password:
QjL6H5nH
# 连接超时时间 # 连接超时时间
timeout: 3000s timeout: 3000s
lettuce: lettuce:
pool: pool:
# 连接池中的最小空闲连接 # 连接池中的最小空闲连接
min-idle: 0 min-idle:
0
# 连接池中的最大空闲连接 # 连接池中的最大空闲连接
max-idle: 8 max-idle: 8
# 连接池的最大数据库连接数 # 连接池的最大数据库连接数
......
ruoshui-admin/src/main/resources/application.yml
浏览文件 @ 0dcaf7ce
...@@ -49,11 +49,11 @@ spring: ...@@ -49,11 +49,11 @@ spring:
driver-class-name: com.mysql.cj.jdbc.Driver driver-class-name: com.mysql.cj.jdbc.Driver
# datax-web email # datax-web email
mail: mail:
host: smtp host: smtp.feishu.cn
port: 465 port: 465
username: 2217343704@qq.com username: lvbencai@wangxiaolu.com.cn
password: lsx19980707.. password: mcRafgxSR1oocC42
authorization: kwzfbpemiqzwebgf authorization: mcRafgxSR1oocC42
properties: properties:
mail: mail:
smtp: smtp:
...@@ -131,8 +131,6 @@ token: ...@@ -131,8 +131,6 @@ token:
# 令牌有效期（默认30分钟） # 令牌有效期（默认30分钟）
expireTime: 720 expireTime: 720
#datax-job, access token #datax-job, access token
datax: datax:
job: job:
...@@ -151,7 +149,6 @@ datax: ...@@ -151,7 +149,6 @@ datax:
dataxPyHome: /Users/tumaxiao/soft/大数据etl-datx-flink/datax/bin/datax.py dataxPyHome: /Users/tumaxiao/soft/大数据etl-datx-flink/datax/bin/datax.py
dataxHome: /Users/tumaxiao/soft/大数据etl-datx-flink/datax dataxHome: /Users/tumaxiao/soft/大数据etl-datx-flink/datax
## MyBatis配置 ## MyBatis配置
#mybatis: #mybatis:
# # 搜索指定包别名 # # 搜索指定包别名
...@@ -189,7 +186,6 @@ mybatis-plus: ...@@ -189,7 +186,6 @@ mybatis-plus:
jdbc-type-for-null: 'null' jdbc-type-for-null: 'null'
type-handlers-package: com.ruoshui.core.handler type-handlers-package: com.ruoshui.core.handler
# PageHelper分页插件 # PageHelper分页插件
pagehelper: pagehelper:
helperDialect: mysql helperDialect: mysql
...@@ -215,4 +211,7 @@ xss: ...@@ -215,4 +211,7 @@ xss:
datasource: datasource:
aes: aes:
key: AD42F6697B035B75 key: AD42F6697B035B75
# 飞书配置
feishu:
webhook:
url: https://open.feishu.cn/open-apis/bot/v2/hook/your_webhook_url_here
ruoshui-admin/src/main/resources/logback.xml
浏览文件 @ 0dcaf7ce
...@@ -76,12 +76,9 @@ ...@@ -76,12 +76,9 @@
<!-- Spring日志级别控制 --> <!-- Spring日志级别控制 -->
<logger name="org.springframework" level="warn" /> <logger name="org.springframework" level="warn" />
<root level="info">
<appender-ref ref="console" />
</root>
<!--系统操作日志--> <!--系统操作日志-->
<root level="info"> <root level="info">
<appender-ref ref="console" />
<appender-ref ref="file_info" /> <appender-ref ref="file_info" />
<appender-ref ref="file_error" /> <appender-ref ref="file_error" />
</root> </root>
......
ruoshui-datax-executor/src/main/resources/application.yml
浏览文件 @ 0dcaf7ce
...@@ -14,15 +14,15 @@ datax: ...@@ -14,15 +14,15 @@ datax:
admin: admin:
### datax admin address list, such as "http://address" or "http://address01,http://address02" ### datax admin address list, such as "http://address" or "http://address01,http://address02"
addresses: http://127.0.0.1:10001 addresses: http://127.0.0.1:10001
# addresses: http://192.168.172.173:8080 # addresses: http://192.168.172.173:8080
executor: executor:
appname: mendale_executor appname: mendale_executor
ip: 127.0.0.1 ip: 127.0.0.1
port: 10002 port: 10002
# port: 9998 # port: 9998
### job log path ### job log path
logpath: ./applogs/executor/jobhandler logpath: ./applogs/executor/jobhandler
# logpath: /lsx/bigdata/datax-executor/data/applogs/executor/jobhandler # logpath: /lsx/bigdata/datax-executor/data/applogs/executor/jobhandler
### job log retention days ### job log retention days
logretentiondays: 30 logretentiondays: 30
### job, access token ### job, access token
......
ruoshui-framework/src/main/java/com/ruoshui/framework/config/properties/DruidProperties.java
浏览文件 @ 0dcaf7ce
...@@ -12,37 +12,37 @@ import com.alibaba.druid.pool.DruidDataSource; ...@@ -12,37 +12,37 @@ import com.alibaba.druid.pool.DruidDataSource;
@Configuration @Configuration
public class DruidProperties public class DruidProperties
{ {
@Value("${spring.datasource.druid.initialSize}") @Value("${spring.datasource.druid.initialSize:5}")
private int initialSize; private int initialSize;
@Value("${spring.datasource.druid.minIdle}") @Value("${spring.datasource.druid.minIdle:10}")
private int minIdle; private int minIdle;
@Value("${spring.datasource.druid.maxActive}") @Value("${spring.datasource.druid.maxActive:20}")
private int maxActive; private int maxActive;
@Value("${spring.datasource.druid.maxWait}") @Value("${spring.datasource.druid.maxWait:60000}")
private int maxWait; private int maxWait;
@Value("${spring.datasource.druid.timeBetweenEvictionRunsMillis}") @Value("${spring.datasource.druid.timeBetweenEvictionRunsMillis:60000}")
private int timeBetweenEvictionRunsMillis; private int timeBetweenEvictionRunsMillis;
@Value("${spring.datasource.druid.minEvictableIdleTimeMillis}") @Value("${spring.datasource.druid.minEvictableIdleTimeMillis:300000}")
private int minEvictableIdleTimeMillis; private int minEvictableIdleTimeMillis;
@Value("${spring.datasource.druid.maxEvictableIdleTimeMillis}") @Value("${spring.datasource.druid.maxEvictableIdleTimeMillis:900000}")
private int maxEvictableIdleTimeMillis; private int maxEvictableIdleTimeMillis;
@Value("${spring.datasource.druid.validationQuery}") @Value("${spring.datasource.druid.validationQuery:SELECT 1 FROM DUAL}")
private String validationQuery; private String validationQuery;
@Value("${spring.datasource.druid.testWhileIdle}") @Value("${spring.datasource.druid.testWhileIdle:true}")
private boolean testWhileIdle; private boolean testWhileIdle;
@Value("${spring.datasource.druid.testOnBorrow}") @Value("${spring.datasource.druid.testOnBorrow:false}")
private boolean testOnBorrow; private boolean testOnBorrow;
@Value("${spring.datasource.druid.testOnReturn}") @Value("${spring.datasource.druid.testOnReturn:false}")
private boolean testOnReturn; private boolean testOnReturn;
public DruidDataSource dataSource(DruidDataSource datasource) public DruidDataSource dataSource(DruidDataSource datasource)
......
ruoshui-system/src/main/java/com/ruoshui/bigdata/core/conf/JobAdminConfig.java
浏览文件 @ 0dcaf7ce
...@@ -65,6 +65,9 @@ public class JobAdminConfig implements InitializingBean, DisposableBean { ...@@ -65,6 +65,9 @@ public class JobAdminConfig implements InitializingBean, DisposableBean {
@Value("${spring.mail.authorization}") @Value("${spring.mail.authorization}")
private String emailAuthorization; private String emailAuthorization;
@Value("${feishu.webhook.url:}")
private String feishuWebhookUrl;
@Value("${datax.job.triggerpool.fast.max}") @Value("${datax.job.triggerpool.fast.max}")
private int triggerPoolFastMax; private int triggerPoolFastMax;
...@@ -164,4 +167,8 @@ public class JobAdminConfig implements InitializingBean, DisposableBean { ...@@ -164,4 +167,8 @@ public class JobAdminConfig implements InitializingBean, DisposableBean {
public String getEmailAuthorization() { public String getEmailAuthorization() {
return emailAuthorization; return emailAuthorization;
} }
public String getFeishuWebhookUrl() {
return feishuWebhookUrl;
}
} }
ruoshui-system/src/main/java/com/ruoshui/bigdata/core/thread/FeishuUtil.java 0 → 100644
浏览文件 @ 0dcaf7ce
package com.ruoshui.bigdata.core.thread;
import com.ruoshui.bigdata.core.conf.JobAdminConfig;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FeishuUtil {
private static Logger logger = LoggerFactory.getLogger(FeishuUtil.class);
public static void send(String alarmContent, String title, String content) {
String webhookUrl = JobAdminConfig.getAdminConfig().getFeishuWebhookUrl();
if (webhookUrl == null || webhookUrl.trim().isEmpty()) {
logger.warn(">>>>>>>>>>> ruoshui-ground, feishu webhook url is empty, skip send feishu notification.");
return;
}
try {
JSONObject card = new JSONObject();
card.set("config", new JSONObject().set("wide_screen_mode", true));
JSONObject header = new JSONObject();
JSONObject titleObj = new JSONObject();
titleObj.set("tag", "plain_text");
titleObj.set("content", title);
header.set("title", titleObj);
header.set("template", "red");
card.set("header", header);
JSONObject element = new JSONObject();
element.set("tag", "markdown");
element.set("content", formatMarkdownContent(alarmContent, content));
card.set("elements", new Object[]{element});
JSONObject requestBody = new JSONObject();
requestBody.set("msg_type", "interactive");
requestBody.set("card", card);
String result = HttpUtil.post(webhookUrl, requestBody.toString());
logger.info(">>>>>>>>>>> ruoshui-ground, job fail alarm feishu send result:{}", result);
JSONObject resultJson = JSONUtil.parseObj(result);
if (resultJson != null && resultJson.getInt("code") != null && resultJson.getInt("code") == 0) {
logger.info(">>>>>>>>>>> ruoshui-ground, job fail alarm feishu send success.");
} else {
logger.error(">>>>>>>>>>> ruoshui-ground, job fail alarm feishu send error. Response: {}", result);
}
} catch (Exception e) {
logger.error(">>>>>>>>>>> ruoshui-ground, job fail alarm feishu send exception.", e);
}
}
private static String formatMarkdownContent(String alarmContent, String content) {
StringBuilder markdown = new StringBuilder();
markdown.append("**告警信息**\n");
markdown.append("------------------------------\n");
if (alarmContent != null && !alarmContent.isEmpty()) {
String cleanAlarmContent = alarmContent.replace("<br>", "\n")
.replace("<br/>", "\n")
.replace("<", "&lt;")
.replace(">", "&gt;");
markdown.append(cleanAlarmContent).append("\n");
}
markdown.append("------------------------------\n");
if (content != null && !content.isEmpty()) {
String cleanContent = content.replace("<br>", "\n")
.replace("<br/>", "\n")
.replace("<", "&lt;")
.replace(">", "&gt;")
.replaceAll("<[^>]*>", "");
markdown.append(cleanContent).append("\n");
}
return markdown.toString();
}
}
\ No newline at end of file
ruoshui-system/src/main/java/com/ruoshui/bigdata/core/thread/JobFailMonitorHelper.java
浏览文件 @ 0dcaf7ce
...@@ -184,14 +184,15 @@ public class JobFailMonitorHelper { ...@@ -184,14 +184,15 @@ public class JobFailMonitorHelper {
try { try {
EmailUtil.send(JobAdminConfig.getAdminConfig().getEmailUserName(), JobAdminConfig.getAdminConfig().getEmailPassword(),JobAdminConfig.getAdminConfig().getEmailAuthorization(),email,title,content); EmailUtil.send(JobAdminConfig.getAdminConfig().getEmailUserName(), JobAdminConfig.getAdminConfig().getEmailPassword(),JobAdminConfig.getAdminConfig().getEmailAuthorization(),email,title,content);
} catch (Exception e) { } catch (Exception e) {
logger.error(">>>>>>>>>>> ruoshui-ground, job fail alarm email send error, JobLogId:{}", jobLog.getId(), e); logger.error(">>>>>>>>>>> ruoshui-ground, job fail alarm email send error, JobLogId:"+ jobLog.getId(), e);
alarmResult = false; alarmResult = false;
} }
} }
} }
// 增加飞书发送通知
// FeishuUtil.send(alarmContent,title,content);
// do something, custom alarm strategy, such as sms // do something, custom alarm strategy, such as sms
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论