背景
本文主要讲述的是如何实现动态切换数据源,数据源信息是存储在数据库表里,与在配置文件中写好数据库信息然后用@DS("XX")注解不同。
前言
本文是为了解决多数据源切换执行任务等,希望对大家有所帮助,有问题的可以在评论区留言,看到一定回复您!
一、准备工作
1.依赖
<dependency> <groupId>com.baomidou</groupId> <artifactId>dynamic-datasource-spring-boot-starter</artifactId> <version>3.4.0</version> </dependency>
2.数据库表(脚本)
SET NAMES utf8mb4; SET FOREIGN_KEY_CHECKS = 0; -- ---------------------------- -- Table structure for sys_data_resource -- ---------------------------- DROP TABLE IF EXISTS `sys_data_resource`; CREATE TABLE `sys_data_resource` ( `uuid` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '主键', `data_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '数据源别名', `ip_address` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '服务器地址(支持ip和域名)', `port` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '端口号', `data_resource_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '数据库名称', `user_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '用户名', `password` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '密码', `parent_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT '0' COMMENT '父id,默认0,0:代表父级 用于树形结构', `data_resource_type` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT 'mysql' COMMENT '数据库类型,默认:mysql 用于区分不同数据库驱动', `create_user_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '创建用户id', `create_time` datetime NULL DEFAULT NULL COMMENT '创建时间', `update_user_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '修改用户id', `update_time` datetime NULL DEFAULT NULL COMMENT '修改时间', `url` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '数据库链接信息', `status` int(2) NULL DEFAULT 0 COMMENT '0:未删除,1:已删除', `data_desc` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '数据描述', PRIMARY KEY (`uuid`) USING BTREE ) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = '动态数据源管理' ROW_FORMAT = Dynamic; SET FOREIGN_KEY_CHECKS = 1;
3.配置文件
#配置数据源 spring: datasource: dynamic: primary: master #设置默认的数据源,默认值为master strict: false datasource: master: # 数据源1配置 url: jdbc:log4jdbc:mysql://192.168.1.202:3306/xxxxxx?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowPublicKeyRetrieval=true username: xxxxx password: xxx123! driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy druid: db-type: com.alibaba.druid.pool.DruidDataSource break-after-acquire-failure: true # 初始化配置 initial-size: 3 # 最小连接数 min-idle: 3 # 最大连接数 max-active: 15 # 获取连接超时时间 max-wait: 5000 # 连接有效性检测时间 time-between-eviction-runs-millis: 90000 # 最大空闲时间 min-evictable-idle-time-millis: 1800000 test-while-idle: true test-on-borrow: false test-on-return: false validation-query: select 1 # 配置监控统计拦截的filters filters: stat stat-view-servlet: url-pattern: /druid/* reset-enable: false web-stat-filter: url-pattern: /* exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*" redis: #数据库索引 database: 0 host: 192.168.1.202 port: 6379 password: xxxx23!
url和driver-class-name 大家可以自行换成mysql的驱动就行。
4.自定义注解@ChangeDB
当方法上使用@ChangeDB时,代表切换数据源,不加时,默认走主数据源。
import java.lang.annotation.*; /** * 自定义多数据源切换注解 * <p> * 优先级:先方法,后类,如果方法覆盖了类上的数据源类型,以方法的为准,否则以类上的为准 * 如果传递value,则切换到value对应的数据源 * * @author 曹震 */ @Target({ElementType.METHOD, ElementType.TYPE}) @Retention(RetentionPolicy.RUNTIME) @Documented @Inherited public @interface ChangeDB { String value() default ""; }
5.定义AOP切面类
import com.alibaba.fastjson.JSONObject; import com.baomidou.dynamic.datasource.DynamicRoutingDataSource; import com.baomidou.dynamic.datasource.creator.DefaultDataSourceCreator; import com.baomidou.dynamic.datasource.spring.boot.autoconfigure.DataSourceProperty; import com.baomidou.dynamic.datasource.toolkit.DynamicDataSourceContextHolder; import com.hvit.data_governance.dataComparison.dbhelp.DataSourceHelp; import com.hvit.data_governance.dataComparison.entity.SysDataResource; import com.hvit.data_governance.dataComparison.service.SysDataResourceService; import com.hvit.data_governance.utils.DBUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.annotation.Pointcut; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.sql.DataSource; import java.io.BufferedReader; import java.io.IOException; import java.util.Set; /** * 动态数据源切面拦截 * 请注意:这里order一定要小于tx:annotation-driven的order,即先执行DynamicDataSourceAspectAdvice切面,再执行事务切面,才能获取到最终的数据源 * * @author 曹震 * @version 1.0 */ @Slf4j @Aspect @Component @Order(1) @EnableAspectJAutoProxy(proxyTargetClass = true) public class DynamicDataSourceAspect { @Autowired private DataSource dataSource; @Resource private DefaultDataSourceCreator defaultDataSourceCreator; @Autowired private SysDataResourceService sysDataResourceService; @Pointcut("@annotation(com.hvit.data_governance.dataComparison.dbhelp.ChangeDB) || @within(com.hvit.data_governance.dataComparison.dbhelp.ChangeDB)") public void dsPointCut() { } @Around("dsPointCut()") public Object doAround(ProceedingJoinPoint joinPoint) throws Throwable { ServletRequestAttributes servletRequestAttributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); HttpServletRequest request = servletRequestAttributes.getRequest(); DynamicRoutingDataSource ds = (DynamicRoutingDataSource) dataSource; String dynamicDataSource = ""; try { //获取参数,切换数据源参数固定都传poolName参数 //get请求获取参数 dynamicDataSource = request.getParameter(DBUtils.DATA_RESOURCE_NAME); if (StringUtils.isEmpty(dynamicDataSource)) { //如果get请求没获取到参数,就那用post请求body获取参数 dynamicDataSource = getDataResourceName(request); } //如果get、post都没获取到,支持动态传参切换数据源 if (StringUtils.isEmpty(dynamicDataSource)) { dynamicDataSource = (String) request.getAttribute("poolName"); } if (StringUtils.isNotEmpty(dynamicDataSource)) { //先获取传入的数据源是不是存在数据池当中 Set<String> sets = ds.getCurrentDataSources().keySet(); boolean flag = false; for (String str : sets) { //首先排除主数据源 if (!"master".equals(str)) { if (str.equals(dynamicDataSource)) { DynamicDataSourceContextHolder.push(dynamicDataSource); flag = true; break; } } } //如果数据池中没有找到当前数据源,那么开始进行新增数据源 //poolName其实就是数据源id if (!flag) { SysDataResource sysDataResource = sysDataResourceService.getById(dynamicDataSource); if (sysDataResource != null) { manualAddSource(sysDataResource); DynamicDataSourceContextHolder.push(dynamicDataSource); } } return joinPoint.proceed(); } return joinPoint.proceed(); } finally { //每次使用完成,清空数据源 //DynamicDataSourceContextHolder.poll(); if (StringUtils.isNotEmpty(dynamicDataSource)) { ds.removeDataSource(dynamicDataSource); } } } /*** * post请求获取HttpServletRequest 内 poolName参数 * @param request * @return */ public String getDataResourceName(HttpServletRequest request) throws IOException { StringBuilder data = new StringBuilder(); String line; BufferedReader reader; try { reader = request.getReader(); while (null != (line = reader.readLine())) { data.append(line); } } catch (IOException e) { return null; } JSONObject jsonObject = JSONObject.parseObject(data.toString()); if (jsonObject == null) { return ""; } return jsonObject.getString(DBUtils.DATA_RESOURCE_NAME); } /*** * 手动加入数据源池 * @param sysDataResource */ public void manualAddSource(SysDataResource sysDataResource) { DataSourceHelp dataSourceHelp = new DataSourceHelp(); dataSourceHelp.setDriverClassName(DBUtils.getDriverClassName(sysDataResource.getDataResourceType())); dataSourceHelp.setPassword(sysDataResource.getPassword()); dataSourceHelp.setPoolName(sysDataResource.getUuid()); dataSourceHelp.setUsername(sysDataResource.getUserName()); dataSourceHelp.setUrl(sysDataResource.getUrl()); addSource(dataSourceHelp); } public Set<String> addSource(DataSourceHelp dataSourceHelp) { DataSourceProperty dataSourceProperty = new DataSourceProperty(); BeanUtils.copyProperties(dataSourceHelp, dataSourceProperty); DynamicRoutingDataSource source = (DynamicRoutingDataSource) dataSource; DataSource dataSource = defaultDataSourceCreator.createDataSource(dataSourceProperty); source.addDataSource(dataSourceHelp.getPoolName(), dataSource); return source.getCurrentDataSources().keySet(); } }
这里解释下,我们会先做页面中保存数据库链接信息,然后在树结构中点击查询时,会固定传一个参数poolName,我们传的是记录数据库信息的id,作为poolName,用于我们后续切换数据源的时候的一个标识。
注意: 切面中,我们会在get请求里拿到poolName进行数据源切换,但是在日常考虑中,有post请求也需要切换数据源,所以代码中也加入了从body里取得poolName进行切换参数。为了进一步实现自定义切换参数,我们也可以在 request.getAttribute("poolName");取得参数进行切换!
6.DataSourceHelp类
import lombok.Data; @Data public class DataSourceHelp { private String poolName; private String driverClassName; private String url; private String username; private String password; }
二、开发代码
1.controller层
import com.hvit.data_governance.dataComparison.request.*; import com.hvit.data_governance.dataComparison.service.data.DBService; import com.hvit.data_governance.dataComparison.service.data.SysDataResourceDataService; import com.hvit.data_governance.utils.R; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.sql.SQLException; /** * <p> * 动态数据源管理 前端控制器 * </p> * * @author 曹震 * @since 2022-12-06 */ @Api(tags = "动态数据源管理") @RestController @RequestMapping("/hvit/dataResource/") public class SysDataResourceController { @Autowired private SysDataResourceDataService sysDataResourceDataService; @Autowired private DBService dbService; @ApiOperation("获取数据集的字段信息") @GetMapping("/getDataTableColumns") public ResponseEntity getDataTableColumns(String tableName, String poolName) { return ResponseEntity.ok(sysDataResourceDataService.getDataTableColumns(tableName, poolName)); } }
2.service层
@Slf4j @Service public class SysDataResourceDataService { @Autowired private SysDataResourceService sysDataResourceService; @Autowired private DBService dbService; public R getDataTableColumns(String tableName, String poolName) { if (StringUtils.isAnyEmpty(tableName, poolName)) { return R.error("缺少参数!"); } getDataTableFramework(tableName, poolName); QueryWrapper<SysTableStructure> queryWrapper = new QueryWrapper<>(); queryWrapper.lambda().eq(SysTableStructure::getDataResourceUuid, poolName); queryWrapper.lambda().eq(SysTableStructure::getTableName, tableName); queryWrapper.lambda().orderByAsc(SysTableStructure::getSort); return R.ok().put("data", sysTableStructureService.list(queryWrapper)); } /*** * 注意点:当方法内调用其他需要数据源切换的方法时,需要将类本身注入进来,然后调用才能生效(数据源切换) * 处理字段入库 * @param tableName * @param poolName */ @Transactional public void getDataTableFramework(String tableName, String poolName) { //获取表字段结构信息 List<TableFramework> tableFrameworks = dbService.getTableFramework(tableName); //获取不在表中的字段信息 List<String> stringList = getDifferenceSet(tableFrameworks, tableName, poolName); //装备剔除没用的字段 if (!CollectionUtils.isEmpty(stringList)) { QueryWrapper<SysTableStructure> queryWrapper = new QueryWrapper<>(); queryWrapper.lambda().eq(SysTableStructure::getDataResourceUuid, poolName); queryWrapper.lambda().eq(SysTableStructure::getTableName, tableName); queryWrapper.lambda().in(SysTableStructure::getField, stringList); sysTableStructureService.remove(queryWrapper); } AtomicInteger i = new AtomicInteger(); if (!CollectionUtils.isEmpty(tableFrameworks)) { tableFrameworks.forEach(x -> { QueryWrapper<SysTableStructure> wrapper = new QueryWrapper<>(); wrapper.lambda().eq(SysTableStructure::getDataResourceUuid, poolName); wrapper.lambda().eq(SysTableStructure::getTableName, tableName); wrapper.lambda().eq(SysTableStructure::getField, x.getField()); if (sysTableStructureService.count(wrapper) == 0) { SysTableStructure sysTableStructure = new SysTableStructure(); sysTableStructure.setField(x.getField()); sysTableStructure.setTableName(tableName); sysTableStructure.setHidden(1); sysTableStructure.setType(x.getType()); sysTableStructure.setSort(i.get()); sysTableStructure.setLength(DBUtils.getContent(x.getType())); sysTableStructure.setDataResourceUuid(poolName); sysTableStructure.setPriKey(x.getKey()); sysTableStructureService.save(sysTableStructure); } i.getAndIncrement(); }); } } /*** * 智能匹配字段差集 * 当表字段存在记录表中,而实际数据库表字段通过其他手动修改过时,智能剔除。 * @param tableFrameworks * @param tableName * @return */ public List<String> getDifferenceSet(List<TableFramework> tableFrameworks, String tableName, String poolName) { QueryWrapper<SysTableStructure> queryWrapper = new QueryWrapper<>(); queryWrapper.lambda().eq(SysTableStructure::getTableName, tableName); queryWrapper.lambda().eq(SysTableStructure::getDataResourceUuid, poolName); List<SysTableStructure> structureList = sysTableStructureService.list(queryWrapper); List<String> retList = new ArrayList<>(); if (!CollectionUtils.isEmpty(structureList)) { List<String> strings = sysTableStructureService.list(queryWrapper).stream().map(SysTableStructure::getField).collect(Collectors.toList()); List<String> stringList = tableFrameworks.stream().map(TableFramework::getField).collect(Collectors.toList()); retList = strings.stream().filter(e -> { return !stringList.contains(e); }).collect(Collectors.toList()); } return retList; } }
3. dbService类
import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.hvit.data_governance.dataComparison.dao.SysDataResourceMapper; import com.hvit.data_governance.dataComparison.dbhelp.ChangeDB; import com.hvit.data_governance.dataComparison.entity.SysDataResource; import com.hvit.data_governance.dataComparison.entity.TableFramework; import com.hvit.data_governance.dataComparison.request.HandleMianDataReq; import com.hvit.data_governance.dataComparison.request.RowDataReq; import com.hvit.data_governance.dataComparison.service.SysDataResourceService; import com.hvit.data_governance.utils.DBUtils; import com.hvit.data_governance.utils.R; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.ibatis.annotations.Param; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @Slf4j @Service public class DBService { @Resource private SysDataResourceMapper sysDataResourceMapper; @Autowired private SysDataResourceDataService sysDataResourceDataService; /*** * 获取表结构信息 * @param tableName * @return */ @ChangeDB public List<TableFramework> getTableFramework(String tableName) { return sysDataResourceMapper.getTableFramework(tableName); } }
4.sysDataResourceMapper类
import com.hvit.data_governance.dataComparison.entity.SysDataResource; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.hvit.data_governance.dataComparison.entity.TableFramework; import org.apache.ibatis.annotations.*; import java.util.LinkedHashMap; import java.util.List; /** * <p> * 动态数据源管理 Mapper 接口 * </p> * * @author 曹震 * @since 2022-12-06 */ @Mapper public interface SysDataResourceMapper extends BaseMapper<SysDataResource> { /*** * 获取表结构信息 * @param tableName * @return */ @Select("DESC `${tableName}`") List<TableFramework> getTableFramework(@Param("tableName") String tableName); }
5.看下调用过程
我们可以看到日志输出中,已经进行的数据源的切换,在调用完后也及时的清除了切换的数据源。
可以看到表的结构信息已经获取到了。同样postman中也有,给大家看下。
那么到此处就已经做好了动态数据源的切换了。
如果大家喜欢的话,点个关注,点个赞,谢谢。