liuchang/mall-ebtp-cloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(uboot-core): 分表功能

Browse Source 
1、通过项目创建年份分表
2、修改redis配置类转换localdatetime序列化异常
3、查询项目信息通过redis缓存减少数据库压力
This commit is contained in:
chuhang
2021-09-10 16:39:43 +08:00
parent 01711f7477
commit 49969bf57e
14 changed files with 510 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
uboot-common/src/main/java/com/chinaunicom/mall/ebtp/common/config/RedisLettuceConfig.java
View File

@ -3,7 +3,9 @@ package com.chinaunicom.mall.ebtp.common.config;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.jsontype.impl.LaissezFaireSubTypeValidator;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.data.redis.RedisProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@ -28,14 +30,15 @@ import java.nio.charset.StandardCharsets;
@EnableConfigurationProperties(RedisProperties.class)
public class RedisLettuceConfig {
@Bean
@SuppressWarnings(value = {"unchecked", "rawtypes"})
public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory factory) {
RedisTemplate<String, Object> template = new RedisTemplate<>();
template.setConnectionFactory(factory);
Jackson2JsonRedisSerializer jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer(Object.class);
Jackson2JsonRedisSerializer<Object> jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer<>(Object.class);
ObjectMapper mapper = new ObjectMapper();
mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
mapper.registerModule(new JavaTimeModule());
mapper.activateDefaultTyping(LaissezFaireSubTypeValidator.instance, ObjectMapper.DefaultTyping.NON_FINAL);
jackson2JsonRedisSerializer.setObjectMapper(mapper);

40
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/ShardingAlgorithmAutoConfiguration.java Normal file
View File

@ -0,0 +1,40 @@
package com.chinaunicom.mall.ebtp.common.sharding;
import com.chinaunicom.mall.ebtp.common.sharding.aspect.RequestShardValueAspect;
import com.chinaunicom.mall.ebtp.common.sharding.expression.ExpressionResolver;
import com.chinaunicom.mall.ebtp.common.sharding.expression.KeyResolver;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* 分片初始化
*
* @author dino
* @date 2021/09/10 11:14
*/
@Configuration(proxyBeanMethods = false)
@AutoConfigureAfter(RedisAutoConfiguration.class)
public class ShardingAlgorithmAutoConfiguration {
/**
* 切面 拦截处理所有 @RequestShardValue
* @return Aspect
*/
@Bean
public RequestShardValueAspect requestShardValueAspect() {
return new RequestShardValueAspect();
}
/**
* key 解析器
* @return KeyResolver
*/
@Bean
@ConditionalOnMissingBean(KeyResolver.class)
public KeyResolver keyResolver() {
return new ExpressionResolver();
}
}

23
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/algorithm/handler/ShardValue.java Normal file
View File

@ -0,0 +1,23 @@
package com.chinaunicom.mall.ebtp.common.sharding.algorithm.handler;
import com.chinaunicom.mall.ebtp.common.sharding.annotation.RequestShardValue;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* redis键值存储类
*
* @author dino
* @date 2021/09/06 10:48
*/
@Data
@NoArgsConstructor
@AllArgsConstructor(staticName = "of")
public class ShardValue {
private String redisKey;
private RequestShardValue requestShardValue;
}

59
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/algorithm/handler/YearHintShardingHandler.java Normal file
View File

@ -0,0 +1,59 @@
package com.chinaunicom.mall.ebtp.common.sharding.algorithm.handler;
import io.netty.util.concurrent.FastThreadLocal;
import org.apache.shardingsphere.api.hint.HintManager;
/**
* 分片值处理类
*
* @author dino
* @date 2021/09/06 08:28
*/
public class YearHintShardingHandler {
private YearHintShardingHandler() {
}
/**割接数据表-后缀*/
public static final Integer OLD_TABLE_SUFFIX = 2020;
/**割接数据表-标识*/
public static final Integer CUTOVER_STATUS = 1;
/**数据库名称*/
private static final String DB_NAME = "ds";
/**数据库分片值*/
private static final Integer DB_SHARD_VALUE = 0;
/*表名*/
private static final String BIZ_RSMS_BID_EVAL_DETAIL = "biz_rsms_bid_eval_detail";
/**分片值*/
public static final FastThreadLocal<ShardValue> REQUEST_SHARD_VALUE = new FastThreadLocal<>();
/**分片缓存前缀*/
public static final String REQUEST_SHARD_KEY = "request_shard_key:";
public static HintManager setting() {
YearHintShardingHandler.clear();
// HintManager API 工具类实例
HintManager hintManager = HintManager.getInstance();
// 直接指定对应具体的数据库
hintManager.addDatabaseShardingValue(DB_NAME,DB_SHARD_VALUE);
return hintManager;
}
public static void clear() {
// 清除掉上一次的规则，否则会报错
HintManager.clear();
REQUEST_SHARD_VALUE.remove();
}
public static void addRsmsTableShardingValue(Integer year) {
HintManager hintManager = YearHintShardingHandler.setting();
// 设置表的分片健
hintManager.addTableShardingValue(BIZ_RSMS_BID_EVAL_DETAIL , year);
}
public static String getKey(String key) {
return REQUEST_SHARD_KEY + key;
}
}

30
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/algorithm/table/YearHintShardingAlgorithm.java Normal file
View File

@ -0,0 +1,30 @@
package com.chinaunicom.mall.ebtp.common.sharding.algorithm.table;
import com.google.common.collect.Lists;
import org.apache.shardingsphere.api.sharding.hint.HintShardingAlgorithm;
import org.apache.shardingsphere.api.sharding.hint.HintShardingValue;
import java.util.Collection;
/**
* hint 年分片算法
*
* @author dino
* @date 2021/09/01 16:02
*/
public class YearHintShardingAlgorithm implements HintShardingAlgorithm<Integer> {
@Override
public Collection<String> doSharding(Collection<String> tableNames, HintShardingValue<Integer> hintShardingValue) {
Collection<String> result = Lists.newArrayList();
for (String tableName : tableNames) {
for (Integer shardingValue : hintShardingValue.getValues()) {
if (tableName.contains(String.valueOf(shardingValue))) {
result.add(tableName);
}
}
}
return result;
}
}

46
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/annotation/RequestShardValue.java Normal file
View File

@ -0,0 +1,46 @@
package com.chinaunicom.mall.ebtp.common.sharding.annotation;
import java.lang.annotation.*;
import java.util.concurrent.TimeUnit;
/**
* 请求值
*
* @author dino
* @date 2021/09/01 10:07
*/
@Inherited
@Target({ElementType.METHOD})
@Retention(value = RetentionPolicy.RUNTIME)
public @interface RequestShardValue {
/**
* 幂等操作的唯一标识，使用spring el表达式 用#来引用方法参数
* @return Spring-EL expression
*/
String key();
/**
* 有效期 默认：1 有效期要大于程序执行时间，否则请求还是可能会进来
* @return expireTime
*/
int expireTime() default 1;
/**
* 时间单位 默认：d
* @return TimeUnit
*/
TimeUnit timeUnit() default TimeUnit.DAYS;
/**
* 提示信息，可自定义
* @return String
*/
String info() default "无法获取分片规则，请稍后重试";
/**
* 是否在业务完成后删除key true:删除 false:不删除
* @return boolean
*/
boolean delKey() default false;
}

112
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/aspect/RequestShardValueAspect.java Normal file
View File

@ -0,0 +1,112 @@
package com.chinaunicom.mall.ebtp.common.sharding.aspect;
import cn.hutool.json.JSONUtil;
import com.chinaunicom.mall.ebtp.common.base.entity.BaseResponse;
import com.chinaunicom.mall.ebtp.common.exception.common.CommonExceptionEnum;
import com.chinaunicom.mall.ebtp.common.exception.entity.BusinessException;
import com.chinaunicom.mall.ebtp.common.sharding.algorithm.handler.ShardValue;
import com.chinaunicom.mall.ebtp.common.sharding.algorithm.handler.YearHintShardingHandler;
import com.chinaunicom.mall.ebtp.common.sharding.annotation.RequestShardValue;
import com.chinaunicom.mall.ebtp.common.sharding.expression.KeyResolver;
import com.chinaunicom.mall.ebtp.common.sharding.feign.client.ProcessService;
import com.chinaunicom.mall.ebtp.common.sharding.feign.domain.ProjectVO;
import com.chinaunicom.mall.ebtp.common.util.HttpContextUtils;
import lombok.extern.slf4j.Slf4j;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.Signature;
import org.aspectj.lang.annotation.After;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.annotation.Order;
import org.springframework.data.redis.core.RedisTemplate;
import javax.servlet.http.HttpServletRequest;
import java.lang.reflect.Method;
import java.util.Objects;
import java.util.Optional;
import static com.chinaunicom.mall.ebtp.common.sharding.algorithm.handler.YearHintShardingHandler.*;
/**
* 自定义请求值实现
*
* @author dino
* @date 2021/09/01 10:10
*/
@Slf4j
@Aspect
@Order(99)
public class RequestShardValueAspect {
@Autowired
private KeyResolver keyResolver;
@Autowired
private ProcessService processService;
@Autowired
private RedisTemplate<String, Object> redisTemplate;
@Pointcut("@annotation(com.chinaunicom.mall.ebtp.common.sharding.annotation.RequestShardValue)")
public void pointCut() {
}
@Before("pointCut()")
public void beforePointCut(JoinPoint joinPoint) {
HttpServletRequest request = HttpContextUtils.getHttpServletRequest();
Method method = this.getMethod(joinPoint);
if (!method.isAnnotationPresent(RequestShardValue.class)) {
return;
}
RequestShardValue requestShardValue = method.getAnnotation(RequestShardValue.class);
String key = keyResolver.resolver(requestShardValue, joinPoint);
log.info("[RequestShardValue]:has add key={}",key);
String redisKey = YearHintShardingHandler.getKey(key);
/*redis db无数据去查询*/
ProjectVO projectVO = Optional.ofNullable((ProjectVO) this.redisTemplate.opsForValue().get(redisKey)).orElseGet(()-> Optional.ofNullable(findProject(key)).orElseThrow(()-> new BusinessException(CommonExceptionEnum.FRAME_EXCEPTION_COMMON_DATA_OTHER_ERROR,requestShardValue.info())));
Integer year = Objects.equals(projectVO.getCutoverStatus(),CUTOVER_STATUS) ? projectVO.getCreateDate().getYear() : OLD_TABLE_SUFFIX;
/*新增rsms分表*/
YearHintShardingHandler.addRsmsTableShardingValue(year);
/*新增线程*/
REQUEST_SHARD_VALUE.set(ShardValue.of(redisKey, requestShardValue));
synchronized(this) {
this.redisTemplate.opsForValue().setIfAbsent(redisKey, projectVO, requestShardValue.expireTime(), requestShardValue.timeUnit());
}
log.info("request params>>>>>>url={},args={},key={},projectId={},year={}", request.getRequestURL().toString(), JSONUtil.toJsonStr(joinPoint.getArgs()), redisKey, projectVO.getProjectId(), year);
}
@After("pointCut()")
public void after(JoinPoint joinPoint) {
Method method = this.getMethod(joinPoint);
if (!method.isAnnotationPresent(RequestShardValue.class)) {
return;
}
ShardValue shardValue = REQUEST_SHARD_VALUE.get();
RequestShardValue requestShardValue = shardValue.getRequestShardValue();
String key = shardValue.getRedisKey();
Optional<Object> o = Optional.ofNullable(this.redisTemplate.opsForValue().get(key));
if (o.isPresent() && requestShardValue.delKey()) {
this.redisTemplate.delete(key);
log.info("[RequestShardValue]:has removed key={}", key);
}
YearHintShardingHandler.clear();
}
private Method getMethod(JoinPoint joinPoint) {
Signature signature = joinPoint.getSignature();
MethodSignature methodSignature = (MethodSignature) signature;
return methodSignature.getMethod();
}
private ProjectVO findProject(String key) {
BaseResponse<ProjectVO> result = processService.getProjectInfo(key);
Optional<Object> optional = Optional.ofNullable(result);
if (!optional.isPresent()) {
throw new BusinessException(CommonExceptionEnum.FRAME_EXCEPTION_COMMON_DATA_OTHER_ERROR,"feign 通过id查询项目分片信息异常!");
}
return result.getData();
}
}

71
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/expression/ExpressionResolver.java Normal file
View File

@ -0,0 +1,71 @@
package com.chinaunicom.mall.ebtp.common.sharding.expression;
import com.chinaunicom.mall.ebtp.common.exception.common.CommonExceptionEnum;
import com.chinaunicom.mall.ebtp.common.exception.entity.BusinessException;
import com.chinaunicom.mall.ebtp.common.sharding.annotation.RequestShardValue;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.core.LocalVariableTableParameterNameDiscoverer;
import org.springframework.expression.Expression;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.expression.spel.support.StandardEvaluationContext;
import java.lang.reflect.Method;
/**
* 默认key 抽取， 优先根据 spel 处理
*
* @author dino
* @date 2021/09/01 10:23
*/
public class ExpressionResolver implements KeyResolver {
private static final SpelExpressionParser PARSER = new SpelExpressionParser();
private static final LocalVariableTableParameterNameDiscoverer DISCOVERER = new LocalVariableTableParameterNameDiscoverer();
/**
* 解析处理 key
*
* @param requestShardValue 接口注解标识
* @param point 接口切点信息
* @return java.lang.String
* @author dino
* @date 2021/9/1 10:22
*/
@Override
public String resolver(RequestShardValue requestShardValue, JoinPoint point) {
Object[] arguments = point.getArgs();
String[] params = DISCOVERER.getParameterNames(getMethod(point));
StandardEvaluationContext context = new StandardEvaluationContext();
if (params != null && params.length > 0) {
for (int len = 0; len < params.length; len++) {
context.setVariable(params[len], arguments[len]);
}
}
Expression expression = PARSER.parseExpression(requestShardValue.key());
return expression.getValue(context, String.class);
}
/**
* 根据切点解析方法信息
* @param joinPoint 切点信息
* @return Method 原信息
*/
private Method getMethod(JoinPoint joinPoint) {
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
Method method = signature.getMethod();
if (method.getDeclaringClass().isInterface()) {
try {
method = joinPoint.getTarget().getClass().getDeclaredMethod(joinPoint.getSignature().getName(),
method.getParameterTypes());
}
catch (SecurityException | NoSuchMethodException e) {
throw new BusinessException(CommonExceptionEnum.FRAME_EXCEPTION_COMMON_DATA_OTHER_ERROR,"切面获取参数类型异常!");
}
}
return method;
}
}

24
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/expression/KeyResolver.java Normal file
View File

@ -0,0 +1,24 @@
package com.chinaunicom.mall.ebtp.common.sharding.expression;
import com.chinaunicom.mall.ebtp.common.sharding.annotation.RequestShardValue;
import org.aspectj.lang.JoinPoint;
/**
* 唯一标志处理器
*
* @author dino
* @date 2021/09/01 10:21
*/
public interface KeyResolver {
/**
* 解析处理 key
*
* @param requestShardValue 接口注解标识
* @param point 接口切点信息
* @return java.lang.String
* @author dino
* @date 2021/9/1 10:22
*/
String resolver(RequestShardValue requestShardValue, JoinPoint point);
}

23
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/feign/client/ProcessService.java Normal file
View File

@ -0,0 +1,23 @@
package com.chinaunicom.mall.ebtp.common.sharding.feign.client;
import com.chinaunicom.mall.ebtp.common.base.entity.BaseResponse;
import com.chinaunicom.mall.ebtp.common.sharding.feign.domain.ProjectVO;
import com.chinaunicom.mall.ebtp.common.sharding.feign.factory.ProcessServiceFallbackFactory;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
/**
* 远程调用流程服务
*
* @author dino
* @date 2021/09/10 10:49
*/
@FeignClient(contextId = "processService",value = "${mconfig.feign.name.process}",
fallbackFactory = ProcessServiceFallbackFactory.class)
//@FeignClient(name = "processService",url = "http://10.242.31.158:18022/api/biz-service-ebtp-process/", fallbackFactory = ProcessServiceFallbackFactory.class)
public interface ProcessService {
@GetMapping("/v1/bizassessroom/project/info")
BaseResponse<ProjectVO> getProjectInfo(@RequestParam String id);
}

25
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/feign/domain/ProjectVO.java Normal file
View File

@ -0,0 +1,25 @@
package com.chinaunicom.mall.ebtp.common.sharding.feign.domain;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data;
import org.springframework.format.annotation.DateTimeFormat;
import java.time.LocalDateTime;
@Data
public class ProjectVO {
private String projectId;
private Integer cutoverStatus;
/**
* 开始评审时间
*/
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
private LocalDateTime createDate;
}

22
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/feign/factory/ProcessServiceFallbackFactory.java Normal file
View File

@ -0,0 +1,22 @@
package com.chinaunicom.mall.ebtp.common.sharding.feign.factory;
import com.chinaunicom.mall.ebtp.common.sharding.feign.client.ProcessService;
import com.chinaunicom.mall.ebtp.common.sharding.feign.fallback.ProcessServiceFallbackImpl;
import feign.hystrix.FallbackFactory;
/**
* 断路器
*
* @author dino
* @date 2021/09/10 10:53
*/
public class ProcessServiceFallbackFactory implements FallbackFactory<ProcessService> {
@Override
public ProcessService create(Throwable throwable) {
ProcessServiceFallbackImpl processServiceFallback = new ProcessServiceFallbackImpl();
processServiceFallback.setCause(throwable);
return processServiceFallback;
}
}

26
uboot-core/src/main/java/com/chinaunicom/mall/ebtp/common/sharding/feign/fallback/ProcessServiceFallbackImpl.java Normal file
View File

@ -0,0 +1,26 @@
package com.chinaunicom.mall.ebtp.common.sharding.feign.fallback;
import com.chinaunicom.mall.ebtp.common.base.entity.BaseResponse;
import com.chinaunicom.mall.ebtp.common.sharding.feign.client.ProcessService;
import com.chinaunicom.mall.ebtp.common.sharding.feign.domain.ProjectVO;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
/**
* 熔断
*
* @author dino
* @date 2021/09/10 10:53
*/
@Slf4j
public class ProcessServiceFallbackImpl implements ProcessService {
@Setter
private Throwable cause;
@Override
public BaseResponse<ProjectVO> getProjectInfo(String id) {
log.error("feign 查询项目分片信息错误:{}",id, cause);
return null;
}
}

4
uboot-core/src/main/resources/META-INF/spring.factories Normal file
View File

@ -0,0 +1,4 @@
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
com.chinaunicom.mall.ebtp.common.sharding.ShardingAlgorithmAutoConfiguration,\
com.chinaunicom.mall.ebtp.common.sharding.feign.fallback.ProcessServiceFallbackImpl,\
com.chinaunicom.mall.ebtp.common.sharding.feign.factory.ProcessServiceFallbackFactory