From c1c0590ccea495f673398ef116b0fcf865d350c8 Mon Sep 17 00:00:00 2001 From: Hanserwei <2628273921@qq.com> Date: Tue, 21 Oct 2025 19:45:43 +0800 Subject: [PATCH] =?UTF-8?q?feat(data-align):=20=E5=AE=9E=E7=8E=B0=E7=AC=94?= =?UTF-8?q?=E8=AE=B0=E7=82=B9=E8=B5=9E=E5=A2=9E=E9=87=8F=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=A4=84=E7=90=86=E4=B8=8E=E5=B8=83=E9=9A=86=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E5=99=A8=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 Redis 布隆过滤器 Lua 脚本，用于校验日增量变更数据- 创建 InsertRecordMapper 及 XML 映射文件，支持笔记和用户点赞数落库 - 定义 LikeUnlikeNoteMqDTO用于 MQ 消息传输 - 配置 RedisTemplate 支持 JSON 序列化 - 修改 TableConstants 中 buildTableNameSuffix 方法参数类型为 long - 实现 TodayNoteLikeIncrementData2DBConsumer 消费者逻辑： - 使用布隆过滤器去重判断 - 数据库写入操作使用事务保证原子性 - 写入成功后更新布隆过滤器 - 更新 IntelliJ IDEA 数据源映射与 SQL 检查配置 --- .idea/data_source_mapping.xml | 1 + .idea/inspectionProfiles/Project_Default.xml | 3 + .../align/config/RedisTemplateConfig.java | 31 +++++++ .../align/constant/RedisKeyConstants.java | 21 +++++ .../data/align/constant/TableConstants.java | 2 +- ...TodayNoteLikeIncrementData2DBConsumer.java | 88 +++++++++++++++++++ .../domain/mapper/InsertRecordMapper.java | 20 +++++ .../align/model/vo/LikeUnlikeNoteMqDTO.java | 31 +++++++ .../lua/bloom_today_note_like_check.lua | 16 ++++ .../mapperxml/InsertRecordMapper.xml | 13 +++ 10 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/config/RedisTemplateConfig.java create mode 100644 han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/constant/RedisKeyConstants.java create mode 100644 han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/domain/mapper/InsertRecordMapper.java create mode 100644 han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/model/vo/LikeUnlikeNoteMqDTO.java create mode 100644 han-note-data-align/src/main/resources/lua/bloom_today_note_like_check.lua create mode 100644 han-note-data-align/src/main/resources/mapperxml/InsertRecordMapper.xml diff --git a/.idea/data_source_mapping.xml b/.idea/data_source_mapping.xml index a734c44..34a3631 100644 --- a/.idea/data_source_mapping.xml +++ b/.idea/data_source_mapping.xml @@ -3,6 +3,7 @@ + diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index 1baedb3..863e3f2 100755 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -24,5 +24,8 @@ + + \ No newline at end of file diff --git a/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/config/RedisTemplateConfig.java b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/config/RedisTemplateConfig.java new file mode 100644 index 0000000..dbc549d --- /dev/null +++ b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/config/RedisTemplateConfig.java @@ -0,0 +1,31 @@ +package com.hanserwei.hannote.data.align.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; +import org.springframework.data.redis.serializer.StringRedisSerializer; + +@Configuration +public class RedisTemplateConfig { + + @Bean + public RedisTemplate redisTemplate(RedisConnectionFactory connectionFactory) { + RedisTemplate redisTemplate = new RedisTemplate<>(); + // 设置 RedisTemplate 的连接工厂 + redisTemplate.setConnectionFactory(connectionFactory); + + // 使用 StringRedisSerializer 来序列化和反序列化 redis 的 key 值，确保 key 是可读的字符串 + redisTemplate.setKeySerializer(new StringRedisSerializer()); + redisTemplate.setHashKeySerializer(new StringRedisSerializer()); + + // 使用 Jackson2JsonRedisSerializer 来序列化和反序列化 redis 的 value 值, 确保存储的是 JSON 格式 + Jackson2JsonRedisSerializer serializer = new Jackson2JsonRedisSerializer<>(Object.class); + redisTemplate.setValueSerializer(serializer); + redisTemplate.setHashValueSerializer(serializer); + + redisTemplate.afterPropertiesSet(); + return redisTemplate; + } +} \ No newline at end of file diff --git a/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/constant/RedisKeyConstants.java b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/constant/RedisKeyConstants.java new file mode 100644 index 0000000..6369a3f --- /dev/null +++ b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/constant/RedisKeyConstants.java @@ -0,0 +1,21 @@ +package com.hanserwei.hannote.data.align.constant; + +public class RedisKeyConstants { + + /** + * 布隆过滤器：日增量变更数据，用户笔记点赞，取消点赞 前缀 + */ + public static final String BLOOM_TODAY_NOTE_LIKE_LIST_KEY = "bloom:dataAlign:note:likes:"; + + + /** + * 构建完整的布隆过滤器：日增量变更数据，用户笔记点赞，取消点赞 KEY + * + * @param date 日期 + * @return 完整的布隆过滤器：日增量变更数据，用户笔记点赞，取消点赞 KEY + */ + public static String buildBloomUserNoteLikeListKey(String date) { + return BLOOM_TODAY_NOTE_LIKE_LIST_KEY + date; + } + +} \ No newline at end of file diff --git a/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/constant/TableConstants.java b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/constant/TableConstants.java index 41fb298..a7111cb 100644 --- a/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/constant/TableConstants.java +++ b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/constant/TableConstants.java @@ -13,7 +13,7 @@ public class TableConstants { * @param hashKey 哈希Keu * @return 表名后缀 */ - public static String buildTableNameSuffix(String date, int hashKey) { + public static String buildTableNameSuffix(String date, long hashKey) { // 拼接完整的表名 return date + TABLE_NAME_SEPARATE + hashKey; } diff --git a/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/consumer/TodayNoteLikeIncrementData2DBConsumer.java b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/consumer/TodayNoteLikeIncrementData2DBConsumer.java index f98f739..4daa005 100644 --- a/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/consumer/TodayNoteLikeIncrementData2DBConsumer.java +++ b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/consumer/TodayNoteLikeIncrementData2DBConsumer.java @@ -1,10 +1,28 @@ package com.hanserwei.hannote.data.align.consumer; +import com.hanserwei.framework.common.utils.JsonUtils; import com.hanserwei.hannote.data.align.constant.MQConstants; +import com.hanserwei.hannote.data.align.constant.RedisKeyConstants; +import com.hanserwei.hannote.data.align.constant.TableConstants; +import com.hanserwei.hannote.data.align.domain.mapper.InsertRecordMapper; +import com.hanserwei.hannote.data.align.model.vo.LikeUnlikeNoteMqDTO; +import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.rocketmq.spring.annotation.RocketMQMessageListener; import org.apache.rocketmq.spring.core.RocketMQListener; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.ClassPathResource; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.script.DefaultRedisScript; +import org.springframework.data.redis.core.script.RedisScript; +import org.springframework.scripting.support.ResourceScriptSource; import org.springframework.stereotype.Component; +import org.springframework.transaction.support.TransactionTemplate; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.Objects; @Component @Slf4j @@ -13,8 +31,78 @@ import org.springframework.stereotype.Component; topic = MQConstants.TOPIC_COUNT_NOTE_LIKE ) public class TodayNoteLikeIncrementData2DBConsumer implements RocketMQListener { + + @Resource + private RedisTemplate redisTemplate; + @Resource + private TransactionTemplate transactionTemplate; + @Resource + private InsertRecordMapper insertRecordMapper; + + /** + * 表总分片数 + */ + @Value("${table.shards}") + private int tableShards; @Override public void onMessage(String body) { log.info("## TodayNoteLikeIncrementData2DBConsumer 消费到了 MQ: {}", body); + // 1. 布隆过滤器判断该日增量数据是否已经记录 + // Json字符串转DTO + LikeUnlikeNoteMqDTO noteLikeCountMqDTO = JsonUtils.parseObject(body, LikeUnlikeNoteMqDTO.class); + if (Objects.isNull(noteLikeCountMqDTO)) { + return; + } + log.info("## TodayNoteLikeIncrementData2DBConsumer 笔记点赞数据：{}", JsonUtils.toJsonString(noteLikeCountMqDTO)); + // 获取被点赞或者取消点赞的笔记ID + Long noteId = noteLikeCountMqDTO.getNoteId(); + // 获取点赞或取消点赞的笔记的创建者ID + Long noteCreatorId = noteLikeCountMqDTO.getNoteCreatorId(); + + // 今日日期 + String date = LocalDate.now() + .format(DateTimeFormatter.ofPattern("yyyyMMdd")); // 转字符串 + + String bloomKey = RedisKeyConstants.buildBloomUserNoteLikeListKey(date); + + // 1. 布隆过滤器判断该日增量数据是否已经记录 + DefaultRedisScript script = new DefaultRedisScript<>(); + // Lua 脚本路径 + script.setScriptSource(new ResourceScriptSource(new ClassPathResource("/lua/bloom_today_note_like_check.lua"))); + // 返回值类型 + script.setResultType(Long.class); + + // 执行 Lua 脚本，拿到返回结果 + Long result = redisTemplate.execute(script, Collections.singletonList(bloomKey), noteId); + log.info("布隆过滤器判断结果：{}", result); + + // 若布隆过滤器判断不存在（绝对正确） + if (Objects.equals(result, 0L)) { + // 2. 若无，才会落库，减轻数据库压力 + // 根据分片总数，取模，分别获取对应的分片序号 + long userIdHashKey = noteCreatorId % tableShards; + long noteIdHashKey = noteId % tableShards; + log.info("根据分片总数，取模，分别获取对应的分片序号user:{},note:{}", userIdHashKey, noteIdHashKey); + + // 编程式事务，保证多语句的原子性 + transactionTemplate.execute(status -> { + try { + // 将日增量变更数据，分别写入两张表 + // - t_data_align_note_like_count_temp_日期_分片序号 + // - t_data_align_user_like_count_temp_日期_分片序号 + insertRecordMapper.insert2DataAlignNoteLikeCountTempTable(TableConstants.buildTableNameSuffix(date, noteIdHashKey), noteId); + insertRecordMapper.insert2DataAlignUserLikeCountTempTable(TableConstants.buildTableNameSuffix(date, userIdHashKey), noteCreatorId); + return true; + } catch (Exception ex) { + status.setRollbackOnly(); + log.error("## TodayNoteLikeIncrementData2DBConsumer 落库失败，回滚事务", ex); + } + return false; + }); + // 3. 数据库写入成功后，再添加布隆过滤器中 + // 4. 数据库写入成功后，再添加布隆过滤器中 + RedisScript bloomAddScript = RedisScript.of("return redis.call('BF.ADD', KEYS[1], ARGV[1])", Long.class); + redisTemplate.execute(bloomAddScript, Collections.singletonList(bloomKey), noteId); + } } } diff --git a/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/domain/mapper/InsertRecordMapper.java b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/domain/mapper/InsertRecordMapper.java new file mode 100644 index 0000000..b8152cd --- /dev/null +++ b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/domain/mapper/InsertRecordMapper.java @@ -0,0 +1,20 @@ +package com.hanserwei.hannote.data.align.domain.mapper; + +import org.apache.ibatis.annotations.Param; + +/** + * 添加记录 + */ +public interface InsertRecordMapper { + + /** + * 笔记点赞数：计数变更 + */ + void insert2DataAlignNoteLikeCountTempTable(@Param("tableNameSuffix") String tableNameSuffix, @Param("noteId") Long noteId); + + /** + * 用户获得的点赞数：计数变更 + */ + void insert2DataAlignUserLikeCountTempTable(@Param("tableNameSuffix") String tableNameSuffix, @Param("userId") Long userId); + +} \ No newline at end of file diff --git a/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/model/vo/LikeUnlikeNoteMqDTO.java b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/model/vo/LikeUnlikeNoteMqDTO.java new file mode 100644 index 0000000..03b515a --- /dev/null +++ b/han-note-data-align/src/main/java/com/hanserwei/hannote/data/align/model/vo/LikeUnlikeNoteMqDTO.java @@ -0,0 +1,31 @@ +package com.hanserwei.hannote.data.align.model.vo; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class LikeUnlikeNoteMqDTO { + + private Long userId; + + private Long noteId; + + /** + * 0: 取消点赞， 1：点赞 + */ + private Integer type; + + /** + * 笔记发布者 ID + */ + private Long noteCreatorId; + + private LocalDateTime createTime; +} \ No newline at end of file diff --git a/han-note-data-align/src/main/resources/lua/bloom_today_note_like_check.lua b/han-note-data-align/src/main/resources/lua/bloom_today_note_like_check.lua new file mode 100644 index 0000000..3633429 --- /dev/null +++ b/han-note-data-align/src/main/resources/lua/bloom_today_note_like_check.lua @@ -0,0 +1,16 @@ +-- LUA 脚本：自增量笔记点赞、取消点赞变更数据布隆过滤器 + +local key = KEYS[1] -- 操作的 Redis Key +local noteIdAndNoteCreatorId = ARGV[1] -- Redis Value + +-- 使用 EXISTS 命令检查布隆过滤器是否存在 +local exists = redis.call('EXISTS', key) +if exists == 0 then + -- 创建布隆过滤器 + redis.call('BF.ADD', key, '') + -- 设置过期时间，一天后过期 + redis.call("EXPIRE", key, 20 * 60 * 60) +end + +-- 校验该变更数据是否已经存在(1 表示已存在，0 表示不存在) +return redis.call('BF.EXISTS', key, noteIdAndNoteCreatorId) diff --git a/han-note-data-align/src/main/resources/mapperxml/InsertRecordMapper.xml b/han-note-data-align/src/main/resources/mapperxml/InsertRecordMapper.xml new file mode 100644 index 0000000..29a2fb9 --- /dev/null +++ b/han-note-data-align/src/main/resources/mapperxml/InsertRecordMapper.xml @@ -0,0 +1,13 @@ + + + + + insert into `t_data_align_note_like_count_temp_${tableNameSuffix}` (note_id) + values (#{noteId}) + + + + insert into `t_data_align_user_like_count_temp_${tableNameSuffix}` (user_id) + values (#{userId}) + + \ No newline at end of file