Skip to content

E
etl-ruoshui-bigdata-server
提交 0dcaf7ce authored 作者: 吕本才's avatar 吕本才
浏览文件
操作

feat(告警通知): 修改邮件配置为飞书企业邮箱

优化Druid配置默认值和日志配置 清理.gitignore文件并添加.vscode目录
上级 1d4ea963
显示空白字符变更
内嵌 并排
正在显示 包含 142 行增加 和 43 行删除
.gitignore
浏览文件 @ 0dcaf7ce
......@@ -3,8 +3,9 @@ applogs/**
.idea/**
.idea
.history/**
.vscode/**
**/target/
/build/*
/derby.log
**/**.log
ruoshui-admin/src/main/resources/application-druid.yml
浏览文件 @ 0dcaf7ce
# 开发环境配置
server:
# 服务器的HTTP端口，默认为8080
port: 10001
# 数据源配置
spring:
datasource:
......@@ -9,9 +13,9 @@ spring:
url: jdbc:mysql://192.168.100.40:3306/ruoshui_bigdata?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8
username: root
password: Wxl@325Pa91
# url: jdbc:mysql://127.0.0.1:3306/test_ruoshui_bigdata_db?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8
# username: root
# password: 123456
# url: jdbc:mysql://127.0.0.1:3306/test_ruoshui_bigdata_db?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8
# username: root
# password: 123456
# 从库数据源
slave:
# 从数据源开关/默认关闭
......@@ -59,8 +63,6 @@ spring:
config:
multi-statement-allow: true
# 日志配置
logging:
level:
......
ruoshui-admin/src/main/resources/application-master.yml
浏览文件 @ 0dcaf7ce
# 开发环境配置
server:
# 服务器的HTTP端口，38环境10001，阿里云 10002
port: 10002
port: 10001
# 数据源配置
spring:
datasource:
......@@ -70,16 +70,19 @@ spring:
port: 6379
# 数据库索引
# database: 3
database: 4
database:
4
# 密码
# password: Wxl2025!@#$
password: QjL6H5nH
password:
QjL6H5nH
# 连接超时时间
timeout: 3000s
lettuce:
pool:
# 连接池中的最小空闲连接
min-idle: 0
min-idle:
0
# 连接池中的最大空闲连接
max-idle: 8
# 连接池的最大数据库连接数
......
ruoshui-admin/src/main/resources/application-qa.yml
浏览文件 @ 0dcaf7ce
# 开发环境配置
server:
# 服务器的HTTP端口，默认为8080
port: 10001
port: 10002
# 数据源配置
spring:
datasource:
......@@ -65,15 +65,18 @@ spring:
# 端口，默认为6379
port: 6379
# 数据库索引
database: 4
database:
4
# 密码
password: QjL6H5nH
password:
QjL6H5nH
# 连接超时时间
timeout: 3000s
lettuce:
pool:
# 连接池中的最小空闲连接
min-idle: 0
min-idle:
0
# 连接池中的最大空闲连接
max-idle: 8
# 连接池的最大数据库连接数
......
ruoshui-admin/src/main/resources/application.yml
浏览文件 @ 0dcaf7ce
......@@ -49,11 +49,11 @@ spring:
driver-class-name: com.mysql.cj.jdbc.Driver
# datax-web email
mail:
host: smtp
host: smtp.feishu.cn
port: 465
username: 2217343704@qq.com
password: lsx19980707..
authorization: kwzfbpemiqzwebgf
username: lvbencai@wangxiaolu.com.cn
password: mcRafgxSR1oocC42
authorization: mcRafgxSR1oocC42
properties:
mail:
smtp:
......@@ -131,8 +131,6 @@ token:
# 令牌有效期（默认30分钟）
expireTime: 720
#datax-job, access token
datax:
job:
......@@ -151,7 +149,6 @@ datax:
dataxPyHome: /Users/tumaxiao/soft/大数据etl-datx-flink/datax/bin/datax.py
dataxHome: /Users/tumaxiao/soft/大数据etl-datx-flink/datax
## MyBatis配置
#mybatis:
# # 搜索指定包别名
......@@ -189,7 +186,6 @@ mybatis-plus:
jdbc-type-for-null: 'null'
type-handlers-package: com.ruoshui.core.handler
# PageHelper分页插件
pagehelper:
helperDialect: mysql
......@@ -215,4 +211,7 @@ xss:
datasource:
aes:
key: AD42F6697B035B75
# 飞书配置
feishu:
webhook:
url: https://open.feishu.cn/open-apis/bot/v2/hook/your_webhook_url_here
ruoshui-admin/src/main/resources/logback.xml
浏览文件 @ 0dcaf7ce
......@@ -76,12 +76,9 @@
<!-- Spring日志级别控制 -->
<logger name="org.springframework" level="warn" />
<root level="info">
<appender-ref ref="console" />
</root>
<!--系统操作日志-->
<root level="info">
<appender-ref ref="console" />
<appender-ref ref="file_info" />
<appender-ref ref="file_error" />
</root>
......
ruoshui-datax-executor/src/main/resources/application.yml
浏览文件 @ 0dcaf7ce
......@@ -14,15 +14,15 @@ datax:
admin:
### datax admin address list, such as "http://address" or "http://address01,http://address02"
addresses: http://127.0.0.1:10001
# addresses: http://192.168.172.173:8080
# addresses: http://192.168.172.173:8080
executor:
appname: mendale_executor
ip: 127.0.0.1
port: 10002
# port: 9998
# port: 9998
### job log path
logpath: ./applogs/executor/jobhandler
# logpath: /lsx/bigdata/datax-executor/data/applogs/executor/jobhandler
# logpath: /lsx/bigdata/datax-executor/data/applogs/executor/jobhandler
### job log retention days
logretentiondays: 30
### job, access token
......
ruoshui-framework/src/main/java/com/ruoshui/framework/config/properties/DruidProperties.java
浏览文件 @ 0dcaf7ce
......@@ -12,37 +12,37 @@ import com.alibaba.druid.pool.DruidDataSource;
@Configuration
public class DruidProperties
{
@Value("${spring.datasource.druid.initialSize}")
@Value("${spring.datasource.druid.initialSize:5}")
private int initialSize;
@Value("${spring.datasource.druid.minIdle}")
@Value("${spring.datasource.druid.minIdle:10}")
private int minIdle;
@Value("${spring.datasource.druid.maxActive}")
@Value("${spring.datasource.druid.maxActive:20}")
private int maxActive;
@Value("${spring.datasource.druid.maxWait}")
@Value("${spring.datasource.druid.maxWait:60000}")
private int maxWait;
@Value("${spring.datasource.druid.timeBetweenEvictionRunsMillis}")
@Value("${spring.datasource.druid.timeBetweenEvictionRunsMillis:60000}")
private int timeBetweenEvictionRunsMillis;
@Value("${spring.datasource.druid.minEvictableIdleTimeMillis}")
@Value("${spring.datasource.druid.minEvictableIdleTimeMillis:300000}")
private int minEvictableIdleTimeMillis;
@Value("${spring.datasource.druid.maxEvictableIdleTimeMillis}")
@Value("${spring.datasource.druid.maxEvictableIdleTimeMillis:900000}")
private int maxEvictableIdleTimeMillis;
@Value("${spring.datasource.druid.validationQuery}")
@Value("${spring.datasource.druid.validationQuery:SELECT 1 FROM DUAL}")
private String validationQuery;
@Value("${spring.datasource.druid.testWhileIdle}")
@Value("${spring.datasource.druid.testWhileIdle:true}")
private boolean testWhileIdle;
@Value("${spring.datasource.druid.testOnBorrow}")
@Value("${spring.datasource.druid.testOnBorrow:false}")
private boolean testOnBorrow;
@Value("${spring.datasource.druid.testOnReturn}")
@Value("${spring.datasource.druid.testOnReturn:false}")
private boolean testOnReturn;
public DruidDataSource dataSource(DruidDataSource datasource)
......
ruoshui-system/src/main/java/com/ruoshui/bigdata/core/conf/JobAdminConfig.java
浏览文件 @ 0dcaf7ce
......@@ -65,6 +65,9 @@ public class JobAdminConfig implements InitializingBean, DisposableBean {
@Value("${spring.mail.authorization}")
private String emailAuthorization;
@Value("${feishu.webhook.url:}")
private String feishuWebhookUrl;
@Value("${datax.job.triggerpool.fast.max}")
private int triggerPoolFastMax;
......@@ -164,4 +167,8 @@ public class JobAdminConfig implements InitializingBean, DisposableBean {
public String getEmailAuthorization() {
return emailAuthorization;
}
public String getFeishuWebhookUrl() {
return feishuWebhookUrl;
}
}
ruoshui-system/src/main/java/com/ruoshui/bigdata/core/thread/FeishuUtil.java 0 → 100644
浏览文件 @ 0dcaf7ce
package com.ruoshui.bigdata.core.thread;
import com.ruoshui.bigdata.core.conf.JobAdminConfig;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FeishuUtil {
private static Logger logger = LoggerFactory.getLogger(FeishuUtil.class);
public static void send(String alarmContent, String title, String content) {
String webhookUrl = JobAdminConfig.getAdminConfig().getFeishuWebhookUrl();
if (webhookUrl == null || webhookUrl.trim().isEmpty()) {
logger.warn(">>>>>>>>>>> ruoshui-ground, feishu webhook url is empty, skip send feishu notification.");
return;
}
try {
JSONObject card = new JSONObject();
card.set("config", new JSONObject().set("wide_screen_mode", true));
JSONObject header = new JSONObject();
JSONObject titleObj = new JSONObject();
titleObj.set("tag", "plain_text");
titleObj.set("content", title);
header.set("title", titleObj);
header.set("template", "red");
card.set("header", header);
JSONObject element = new JSONObject();
element.set("tag", "markdown");
element.set("content", formatMarkdownContent(alarmContent, content));
card.set("elements", new Object[]{element});
JSONObject requestBody = new JSONObject();
requestBody.set("msg_type", "interactive");
requestBody.set("card", card);
String result = HttpUtil.post(webhookUrl, requestBody.toString());
logger.info(">>>>>>>>>>> ruoshui-ground, job fail alarm feishu send result:{}", result);
JSONObject resultJson = JSONUtil.parseObj(result);
if (resultJson != null && resultJson.getInt("code") != null && resultJson.getInt("code") == 0) {
logger.info(">>>>>>>>>>> ruoshui-ground, job fail alarm feishu send success.");
} else {
logger.error(">>>>>>>>>>> ruoshui-ground, job fail alarm feishu send error. Response: {}", result);
}
} catch (Exception e) {
logger.error(">>>>>>>>>>> ruoshui-ground, job fail alarm feishu send exception.", e);
}
}
private static String formatMarkdownContent(String alarmContent, String content) {
StringBuilder markdown = new StringBuilder();
markdown.append("**告警信息**\n");
markdown.append("------------------------------\n");
if (alarmContent != null && !alarmContent.isEmpty()) {
String cleanAlarmContent = alarmContent.replace("<br>", "\n")
.replace("<br/>", "\n")
.replace("<", "&lt;")
.replace(">", "&gt;");
markdown.append(cleanAlarmContent).append("\n");
}
markdown.append("------------------------------\n");
if (content != null && !content.isEmpty()) {
String cleanContent = content.replace("<br>", "\n")
.replace("<br/>", "\n")
.replace("<", "&lt;")
.replace(">", "&gt;")
.replaceAll("<[^>]*>", "");
markdown.append(cleanContent).append("\n");
}
return markdown.toString();
}
}
\ No newline at end of file
ruoshui-system/src/main/java/com/ruoshui/bigdata/core/thread/JobFailMonitorHelper.java
浏览文件 @ 0dcaf7ce
......@@ -184,14 +184,15 @@ public class JobFailMonitorHelper {
try {
EmailUtil.send(JobAdminConfig.getAdminConfig().getEmailUserName(), JobAdminConfig.getAdminConfig().getEmailPassword(),JobAdminConfig.getAdminConfig().getEmailAuthorization(),email,title,content);
} catch (Exception e) {
logger.error(">>>>>>>>>>> ruoshui-ground, job fail alarm email send error, JobLogId:{}", jobLog.getId(), e);
logger.error(">>>>>>>>>>> ruoshui-ground, job fail alarm email send error, JobLogId:"+ jobLog.getId(), e);
alarmResult = false;
}
}
}
// 增加飞书发送通知
// FeishuUtil.send(alarmContent,title,content);
// do something, custom alarm strategy, such as sms
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论