feat(cache): 增加批量操作支持以提升性能
- 在 HoTimeCache 中新增 SessionsGet、SessionsSet 和 SessionsDelete 方法,支持批量获取、设置和删除 Session 缓存 - 优化缓存逻辑,减少数据库写入次数,提升性能 - 更新文档,详细说明批量操作的使用方法和性能对比 - 添加调试日志记录,便于追踪批量操作的执行情况
This commit is contained in:
Vendored
+323
@@ -1,11 +1,36 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
. "code.hoteas.com/golang/hotime/common"
|
||||
)
|
||||
|
||||
const debugLogPath = `d:\work\hotimev1.5\.cursor\debug.log`
|
||||
|
||||
// debugLog 写入调试日志
|
||||
func debugLog(hypothesisId, location, message string, data map[string]interface{}) {
|
||||
// #region agent log
|
||||
logFile, _ := os.OpenFile(debugLogPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if logFile != nil {
|
||||
logEntry, _ := json.Marshal(map[string]interface{}{
|
||||
"sessionId": "cache-debug",
|
||||
"runId": "test-run",
|
||||
"hypothesisId": hypothesisId,
|
||||
"location": location,
|
||||
"message": message,
|
||||
"data": data,
|
||||
"timestamp": time.Now().UnixMilli(),
|
||||
})
|
||||
logFile.Write(append(logEntry, '\n'))
|
||||
logFile.Close()
|
||||
}
|
||||
// #endregion
|
||||
}
|
||||
|
||||
// HoTimeCache 可配置memory,db,redis,默认启用memory,默认优先级为memory>redis>db,memory与数据库缓存设置项一致,
|
||||
// 缓存数据填充会自动反方向反哺,加入memory缓存过期将自动从redis更新,但memory永远不会更新redis,如果是集群建议不要开启memory,配置即启用
|
||||
type HoTimeCache struct {
|
||||
@@ -181,6 +206,299 @@ func (that *HoTimeCache) Cache(key string, data ...interface{}) *Obj {
|
||||
return reData
|
||||
}
|
||||
|
||||
// SessionsGet 批量获取 Session 缓存
|
||||
// 返回 Map,key 为缓存键,value 为缓存值
|
||||
// 优先级:memory > redis > db,低优先级数据会反哺到高优先级缓存
|
||||
func (that *HoTimeCache) SessionsGet(keys []string) Map {
|
||||
if len(keys) == 0 {
|
||||
return Map{}
|
||||
}
|
||||
|
||||
// #region agent log
|
||||
debugLog("D", "cache.go:SessionsGet:start", "SessionsGet开始", map[string]interface{}{
|
||||
"keys_count": len(keys),
|
||||
"has_memory": that.memoryCache != nil,
|
||||
"has_redis": that.redisCache != nil,
|
||||
"has_db": that.dbCache != nil,
|
||||
})
|
||||
// #endregion
|
||||
|
||||
result := make(Map, len(keys))
|
||||
missingKeys := keys
|
||||
|
||||
// 从 memory 获取
|
||||
if that.memoryCache != nil && that.memoryCache.SessionSet {
|
||||
memResult := that.memoryCache.CachesGet(keys)
|
||||
// #region agent log
|
||||
debugLog("D", "cache.go:SessionsGet:memory", "从Memory获取", map[string]interface{}{
|
||||
"found_count": len(memResult),
|
||||
})
|
||||
// #endregion
|
||||
for k, v := range memResult {
|
||||
result[k] = v
|
||||
}
|
||||
// 计算未命中的 keys
|
||||
missingKeys = make([]string, 0)
|
||||
for _, k := range keys {
|
||||
if _, exists := result[k]; !exists {
|
||||
missingKeys = append(missingKeys, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 从 redis 获取未命中的
|
||||
if len(missingKeys) > 0 && that.redisCache != nil && that.redisCache.SessionSet {
|
||||
redisResult := that.redisCache.CachesGet(missingKeys)
|
||||
// #region agent log
|
||||
debugLog("D", "cache.go:SessionsGet:redis", "从Redis获取", map[string]interface{}{
|
||||
"missing_count": len(missingKeys),
|
||||
"found_count": len(redisResult),
|
||||
})
|
||||
// #endregion
|
||||
// 反哺到 memory
|
||||
if that.memoryCache != nil && that.memoryCache.SessionSet && len(redisResult) > 0 {
|
||||
that.memoryCache.CachesSet(redisResult)
|
||||
// #region agent log
|
||||
debugLog("D", "cache.go:SessionsGet:backfill_redis_to_mem", "Redis数据反哺到Memory", map[string]interface{}{
|
||||
"backfill_count": len(redisResult),
|
||||
})
|
||||
// #endregion
|
||||
}
|
||||
for k, v := range redisResult {
|
||||
result[k] = v
|
||||
}
|
||||
// 更新未命中的 keys
|
||||
newMissing := make([]string, 0)
|
||||
for _, k := range missingKeys {
|
||||
if _, exists := result[k]; !exists {
|
||||
newMissing = append(newMissing, k)
|
||||
}
|
||||
}
|
||||
missingKeys = newMissing
|
||||
}
|
||||
|
||||
// 从 db 获取未命中的
|
||||
if len(missingKeys) > 0 && that.dbCache != nil && that.dbCache.SessionSet {
|
||||
dbResult := that.dbCache.CachesGet(missingKeys)
|
||||
// #region agent log
|
||||
debugLog("D", "cache.go:SessionsGet:db", "从DB获取", map[string]interface{}{
|
||||
"missing_count": len(missingKeys),
|
||||
"found_count": len(dbResult),
|
||||
})
|
||||
// #endregion
|
||||
// 反哺到 memory 和 redis
|
||||
if len(dbResult) > 0 {
|
||||
if that.memoryCache != nil && that.memoryCache.SessionSet {
|
||||
that.memoryCache.CachesSet(dbResult)
|
||||
// #region agent log
|
||||
debugLog("D", "cache.go:SessionsGet:backfill_db_to_mem", "DB数据反哺到Memory", map[string]interface{}{
|
||||
"backfill_count": len(dbResult),
|
||||
})
|
||||
// #endregion
|
||||
}
|
||||
if that.redisCache != nil && that.redisCache.SessionSet {
|
||||
that.redisCache.CachesSet(dbResult)
|
||||
// #region agent log
|
||||
debugLog("D", "cache.go:SessionsGet:backfill_db_to_redis", "DB数据反哺到Redis", map[string]interface{}{
|
||||
"backfill_count": len(dbResult),
|
||||
})
|
||||
// #endregion
|
||||
}
|
||||
}
|
||||
for k, v := range dbResult {
|
||||
result[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
// #region agent log
|
||||
debugLog("D", "cache.go:SessionsGet:end", "SessionsGet完成", map[string]interface{}{
|
||||
"total_found": len(result),
|
||||
})
|
||||
// #endregion
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// SessionsSet 批量设置 Session 缓存
|
||||
// data: Map,key 为缓存键,value 为缓存值
|
||||
func (that *HoTimeCache) SessionsSet(data Map) {
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// #region agent log
|
||||
debugLog("A", "cache.go:SessionsSet:start", "SessionsSet开始", map[string]interface{}{
|
||||
"data_count": len(data),
|
||||
"has_memory": that.memoryCache != nil,
|
||||
"has_redis": that.redisCache != nil,
|
||||
"has_db": that.dbCache != nil,
|
||||
})
|
||||
// #endregion
|
||||
|
||||
if that.memoryCache != nil && that.memoryCache.SessionSet {
|
||||
that.memoryCache.CachesSet(data)
|
||||
// #region agent log
|
||||
debugLog("A", "cache.go:SessionsSet:memory", "写入Memory完成", map[string]interface{}{"count": len(data)})
|
||||
// #endregion
|
||||
}
|
||||
if that.redisCache != nil && that.redisCache.SessionSet {
|
||||
that.redisCache.CachesSet(data)
|
||||
// #region agent log
|
||||
debugLog("A", "cache.go:SessionsSet:redis", "写入Redis完成", map[string]interface{}{"count": len(data)})
|
||||
// #endregion
|
||||
}
|
||||
if that.dbCache != nil && that.dbCache.SessionSet {
|
||||
that.dbCache.CachesSet(data)
|
||||
// #region agent log
|
||||
debugLog("A", "cache.go:SessionsSet:db", "写入DB完成", map[string]interface{}{"count": len(data)})
|
||||
// #endregion
|
||||
}
|
||||
|
||||
// #region agent log
|
||||
debugLog("A", "cache.go:SessionsSet:end", "SessionsSet完成", nil)
|
||||
// #endregion
|
||||
}
|
||||
|
||||
// SessionsDelete 批量删除 Session 缓存
|
||||
func (that *HoTimeCache) SessionsDelete(keys []string) {
|
||||
if len(keys) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// #region agent log
|
||||
debugLog("C", "cache.go:SessionsDelete:start", "SessionsDelete开始", map[string]interface{}{
|
||||
"keys_count": len(keys),
|
||||
"has_memory": that.memoryCache != nil,
|
||||
"has_redis": that.redisCache != nil,
|
||||
"has_db": that.dbCache != nil,
|
||||
})
|
||||
// #endregion
|
||||
|
||||
if that.memoryCache != nil && that.memoryCache.SessionSet {
|
||||
that.memoryCache.CachesDelete(keys)
|
||||
// #region agent log
|
||||
debugLog("C", "cache.go:SessionsDelete:memory", "从Memory删除完成", map[string]interface{}{"count": len(keys)})
|
||||
// #endregion
|
||||
}
|
||||
if that.redisCache != nil && that.redisCache.SessionSet {
|
||||
that.redisCache.CachesDelete(keys)
|
||||
// #region agent log
|
||||
debugLog("C", "cache.go:SessionsDelete:redis", "从Redis删除完成", map[string]interface{}{"count": len(keys)})
|
||||
// #endregion
|
||||
}
|
||||
if that.dbCache != nil && that.dbCache.SessionSet {
|
||||
that.dbCache.CachesDelete(keys)
|
||||
// #region agent log
|
||||
debugLog("C", "cache.go:SessionsDelete:db", "从DB删除完成", map[string]interface{}{"count": len(keys)})
|
||||
// #endregion
|
||||
}
|
||||
|
||||
// #region agent log
|
||||
debugLog("C", "cache.go:SessionsDelete:end", "SessionsDelete完成", nil)
|
||||
// #endregion
|
||||
}
|
||||
|
||||
// CachesGet 批量获取普通缓存
|
||||
// 返回 Map,key 为缓存键,value 为缓存值
|
||||
// 优先级:memory > redis > db,低优先级数据会反哺到高优先级缓存
|
||||
func (that *HoTimeCache) CachesGet(keys []string) Map {
|
||||
if len(keys) == 0 {
|
||||
return Map{}
|
||||
}
|
||||
|
||||
result := make(Map, len(keys))
|
||||
missingKeys := keys
|
||||
|
||||
// 从 memory 获取
|
||||
if that.memoryCache != nil {
|
||||
memResult := that.memoryCache.CachesGet(keys)
|
||||
for k, v := range memResult {
|
||||
result[k] = v
|
||||
}
|
||||
// 计算未命中的 keys
|
||||
missingKeys = make([]string, 0)
|
||||
for _, k := range keys {
|
||||
if _, exists := result[k]; !exists {
|
||||
missingKeys = append(missingKeys, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 从 redis 获取未命中的
|
||||
if len(missingKeys) > 0 && that.redisCache != nil {
|
||||
redisResult := that.redisCache.CachesGet(missingKeys)
|
||||
// 反哺到 memory
|
||||
if that.memoryCache != nil && len(redisResult) > 0 {
|
||||
that.memoryCache.CachesSet(redisResult)
|
||||
}
|
||||
for k, v := range redisResult {
|
||||
result[k] = v
|
||||
}
|
||||
// 更新未命中的 keys
|
||||
newMissing := make([]string, 0)
|
||||
for _, k := range missingKeys {
|
||||
if _, exists := result[k]; !exists {
|
||||
newMissing = append(newMissing, k)
|
||||
}
|
||||
}
|
||||
missingKeys = newMissing
|
||||
}
|
||||
|
||||
// 从 db 获取未命中的
|
||||
if len(missingKeys) > 0 && that.dbCache != nil {
|
||||
dbResult := that.dbCache.CachesGet(missingKeys)
|
||||
// 反哺到 memory 和 redis
|
||||
if len(dbResult) > 0 {
|
||||
if that.memoryCache != nil {
|
||||
that.memoryCache.CachesSet(dbResult)
|
||||
}
|
||||
if that.redisCache != nil {
|
||||
that.redisCache.CachesSet(dbResult)
|
||||
}
|
||||
}
|
||||
for k, v := range dbResult {
|
||||
result[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// CachesSet 批量设置普通缓存
|
||||
// data: Map,key 为缓存键,value 为缓存值
|
||||
func (that *HoTimeCache) CachesSet(data Map) {
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if that.memoryCache != nil {
|
||||
that.memoryCache.CachesSet(data)
|
||||
}
|
||||
if that.redisCache != nil {
|
||||
that.redisCache.CachesSet(data)
|
||||
}
|
||||
if that.dbCache != nil {
|
||||
that.dbCache.CachesSet(data)
|
||||
}
|
||||
}
|
||||
|
||||
// CachesDelete 批量删除普通缓存
|
||||
func (that *HoTimeCache) CachesDelete(keys []string) {
|
||||
if len(keys) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if that.memoryCache != nil {
|
||||
that.memoryCache.CachesDelete(keys)
|
||||
}
|
||||
if that.redisCache != nil {
|
||||
that.redisCache.CachesDelete(keys)
|
||||
}
|
||||
if that.dbCache != nil {
|
||||
that.dbCache.CachesDelete(keys)
|
||||
}
|
||||
}
|
||||
|
||||
func (that *HoTimeCache) Init(config Map, hotimeDb HoTimeDBInterface, err ...*Error) {
|
||||
//防止空数据问题
|
||||
if config == nil {
|
||||
@@ -263,6 +581,10 @@ func (that *HoTimeCache) Init(config Map, hotimeDb HoTimeDBInterface, err ...*Er
|
||||
if db.Get("timeout") == nil {
|
||||
db["timeout"] = 60 * 60 * 24 * 30
|
||||
}
|
||||
// mode 默认为 "compatible"(兼容模式,便于老系统平滑升级)
|
||||
if db.Get("mode") == nil {
|
||||
db["mode"] = CacheModeCompatible
|
||||
}
|
||||
that.Config["db"] = db
|
||||
|
||||
that.dbCache = &CacheDb{
|
||||
@@ -270,6 +592,7 @@ func (that *HoTimeCache) Init(config Map, hotimeDb HoTimeDBInterface, err ...*Er
|
||||
DbSet: db.GetBool("db"),
|
||||
SessionSet: db.GetBool("session"),
|
||||
HistorySet: db.GetBool("history"),
|
||||
Mode: db.GetString("mode"),
|
||||
Db: hotimeDb,
|
||||
}
|
||||
|
||||
|
||||
Vendored
+289
-38
@@ -3,17 +3,41 @@ package cache
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
. "code.hoteas.com/golang/hotime/common"
|
||||
)
|
||||
|
||||
// debugLogDb 写入调试日志
|
||||
func debugLogDb(hypothesisId, location, message string, data map[string]interface{}) {
|
||||
// #region agent log
|
||||
logFile, _ := os.OpenFile(`d:\work\hotimev1.5\.cursor\debug.log`, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if logFile != nil {
|
||||
logEntry, _ := json.Marshal(map[string]interface{}{
|
||||
"sessionId": "cache-db-debug",
|
||||
"runId": "test-run",
|
||||
"hypothesisId": hypothesisId,
|
||||
"location": location,
|
||||
"message": message,
|
||||
"data": data,
|
||||
"timestamp": time.Now().UnixMilli(),
|
||||
})
|
||||
logFile.Write(append(logEntry, '\n'))
|
||||
logFile.Close()
|
||||
}
|
||||
// #endregion
|
||||
}
|
||||
|
||||
// 表名常量
|
||||
const (
|
||||
CacheTableName = "hotime_cache"
|
||||
CacheHistoryTableName = "hotime_cache_history"
|
||||
LegacyCacheTableName = "cached" // 老版本缓存表名
|
||||
DefaultCacheTimeout = 24 * 60 * 60 // 默认过期时间 24 小时
|
||||
CacheModeNew = "new" // 新模式:只使用新表
|
||||
CacheModeCompatible = "compatible" // 兼容模式:写新读老
|
||||
)
|
||||
|
||||
type HoTimeDBInterface interface {
|
||||
@@ -32,7 +56,8 @@ type CacheDb struct {
|
||||
TimeOut int64
|
||||
DbSet bool
|
||||
SessionSet bool
|
||||
HistorySet bool // 是否开启历史记录
|
||||
HistorySet bool // 是否开启历史记录
|
||||
Mode string // 缓存模式:"new"(默认,只用新表) 或 "compatible"(写新读老)
|
||||
Db HoTimeDBInterface
|
||||
*Error
|
||||
ContextBase
|
||||
@@ -57,6 +82,24 @@ func (that *CacheDb) getHistoryTableName() string {
|
||||
return that.Db.GetPrefix() + CacheHistoryTableName
|
||||
}
|
||||
|
||||
// getLegacyTableName 获取带前缀的老版本缓存表名
|
||||
func (that *CacheDb) getLegacyTableName() string {
|
||||
return that.Db.GetPrefix() + LegacyCacheTableName
|
||||
}
|
||||
|
||||
// isCompatibleMode 是否为兼容模式
|
||||
func (that *CacheDb) isCompatibleMode() bool {
|
||||
return that.Mode == CacheModeCompatible
|
||||
}
|
||||
|
||||
// getEffectiveMode 获取有效模式(默认为 new)
|
||||
func (that *CacheDb) getEffectiveMode() string {
|
||||
if that.Mode == CacheModeCompatible {
|
||||
return CacheModeCompatible
|
||||
}
|
||||
return CacheModeNew
|
||||
}
|
||||
|
||||
// initDbTable 初始化数据库表
|
||||
func (that *CacheDb) initDbTable() {
|
||||
if that.isInit {
|
||||
@@ -66,17 +109,22 @@ func (that *CacheDb) initDbTable() {
|
||||
dbType := that.Db.GetType()
|
||||
tableName := that.getTableName()
|
||||
historyTableName := that.getHistoryTableName()
|
||||
legacyTableName := that.getLegacyTableName()
|
||||
|
||||
// 检查并创建主表
|
||||
if !that.tableExists(tableName) {
|
||||
that.createMainTable(dbType, tableName)
|
||||
}
|
||||
|
||||
// 检查并迁移旧 cached 表
|
||||
oldTableName := that.Db.GetPrefix() + "cached"
|
||||
if that.tableExists(oldTableName) {
|
||||
that.migrateFromCached(dbType, oldTableName, tableName)
|
||||
// 根据模式处理老表
|
||||
// new 模式:迁移老表数据到新表(不删除老表,由人工删除)
|
||||
// compatible 模式:不迁移,老表继续使用
|
||||
if that.getEffectiveMode() == CacheModeNew {
|
||||
if that.tableExists(legacyTableName) {
|
||||
that.migrateFromCached(dbType, legacyTableName, tableName)
|
||||
}
|
||||
}
|
||||
// compatible 模式不做任何处理,老表保留供读取
|
||||
|
||||
// 检查并创建历史表(开启历史记录时)
|
||||
if that.HistorySet && !that.tableExists(historyTableName) {
|
||||
@@ -212,14 +260,15 @@ func (that *CacheDb) createHistoryTable(dbType, tableName string) {
|
||||
that.Db.Exec(createSQL)
|
||||
}
|
||||
|
||||
// migrateFromCached 从旧 cached 表迁移数据
|
||||
// migrateFromCached 从旧 cached 表迁移数据(不删除老表,由人工删除)
|
||||
func (that *CacheDb) migrateFromCached(dbType, oldTableName, newTableName string) {
|
||||
var migrateSQL string
|
||||
|
||||
switch dbType {
|
||||
case "mysql":
|
||||
// 去重迁移:取每个 key 的最后一条记录(id 最大)
|
||||
migrateSQL = "INSERT INTO `" + newTableName + "` (`key`, `value`, `end_time`, `state`, `create_time`, `modify_time`) " +
|
||||
// 使用 INSERT IGNORE 避免重复 key 冲突
|
||||
migrateSQL = "INSERT IGNORE INTO `" + newTableName + "` (`key`, `value`, `end_time`, `state`, `create_time`, `modify_time`) " +
|
||||
"SELECT c.`key`, c.`value`, FROM_UNIXTIME(c.`endtime`), 0, " +
|
||||
"FROM_UNIXTIME(c.`time` / 1000000000), FROM_UNIXTIME(c.`time` / 1000000000) " +
|
||||
"FROM `" + oldTableName + "` c " +
|
||||
@@ -227,7 +276,7 @@ func (that *CacheDb) migrateFromCached(dbType, oldTableName, newTableName string
|
||||
"ON c.id = m.max_id"
|
||||
|
||||
case "sqlite":
|
||||
migrateSQL = `INSERT INTO "` + newTableName + `" ("key", "value", "end_time", "state", "create_time", "modify_time") ` +
|
||||
migrateSQL = `INSERT OR IGNORE INTO "` + newTableName + `" ("key", "value", "end_time", "state", "create_time", "modify_time") ` +
|
||||
`SELECT c."key", c."value", datetime(c."endtime", 'unixepoch'), 0, ` +
|
||||
`datetime(c."time" / 1000000000, 'unixepoch'), datetime(c."time" / 1000000000, 'unixepoch') ` +
|
||||
`FROM "` + oldTableName + `" c ` +
|
||||
@@ -240,22 +289,12 @@ func (that *CacheDb) migrateFromCached(dbType, oldTableName, newTableName string
|
||||
`to_timestamp(c."time" / 1000000000), to_timestamp(c."time" / 1000000000) ` +
|
||||
`FROM "` + oldTableName + `" c ` +
|
||||
`INNER JOIN (SELECT "key", MAX(id) as max_id FROM "` + oldTableName + `" GROUP BY "key") m ` +
|
||||
`ON c.id = m.max_id`
|
||||
`ON c.id = m.max_id ` +
|
||||
`ON CONFLICT ("key") DO NOTHING`
|
||||
}
|
||||
|
||||
// 执行迁移
|
||||
_, err := that.Db.Exec(migrateSQL)
|
||||
if err.GetError() == nil {
|
||||
// 迁移成功,删除旧表
|
||||
var dropSQL string
|
||||
switch dbType {
|
||||
case "mysql":
|
||||
dropSQL = "DROP TABLE `" + oldTableName + "`"
|
||||
case "sqlite", "postgres":
|
||||
dropSQL = `DROP TABLE "` + oldTableName + `"`
|
||||
}
|
||||
that.Db.Exec(dropSQL)
|
||||
}
|
||||
// 执行迁移,不删除老表(由人工确认后删除,更安全)
|
||||
that.Db.Exec(migrateSQL)
|
||||
}
|
||||
|
||||
// writeHistory 写入历史记录
|
||||
@@ -288,25 +327,31 @@ func (that *CacheDb) writeHistory(key string) {
|
||||
that.Db.Insert(historyTableName, historyData)
|
||||
}
|
||||
|
||||
// get 获取缓存
|
||||
func (that *CacheDb) get(key string) interface{} {
|
||||
tableName := that.getTableName()
|
||||
cached := that.Db.Get(tableName, "*", Map{"key": key})
|
||||
// getLegacy 从老表获取缓存(兼容模式使用)
|
||||
// 老表结构:key, value, endtime(unix秒), time(纳秒时间戳)
|
||||
func (that *CacheDb) getLegacy(key string) interface{} {
|
||||
legacyTableName := that.getLegacyTableName()
|
||||
|
||||
// 检查老表是否存在
|
||||
if !that.tableExists(legacyTableName) {
|
||||
return nil
|
||||
}
|
||||
|
||||
cached := that.Db.Get(legacyTableName, "*", Map{"key": key})
|
||||
if cached == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 使用字符串比较判断过期(ISO 格式天然支持)
|
||||
endTime := cached.GetString("end_time")
|
||||
nowTime := Time2Str(time.Now())
|
||||
if endTime != "" && endTime <= nowTime {
|
||||
// 惰性删除:过期只返回 nil,不立即删除
|
||||
// 依赖随机清理批量删除过期数据
|
||||
// 检查过期时间(老表使用 unix 时间戳)
|
||||
endTime := cached.GetInt64("endtime")
|
||||
nowUnix := time.Now().Unix()
|
||||
if endTime > 0 && endTime <= nowUnix {
|
||||
// 已过期,删除该条记录
|
||||
that.deleteLegacy(key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 直接解析 value,不再需要 {"data": value} 包装
|
||||
// 解析 value(老表 value 格式可能是 {"data": xxx} 或直接值)
|
||||
valueStr := cached.GetString("value")
|
||||
if valueStr == "" {
|
||||
return nil
|
||||
@@ -318,9 +363,69 @@ func (that *CacheDb) get(key string) interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 兼容老版本 {"data": xxx} 包装格式
|
||||
if dataMap, ok := data.(map[string]interface{}); ok {
|
||||
if innerData, exists := dataMap["data"]; exists {
|
||||
return innerData
|
||||
}
|
||||
}
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
// deleteLegacy 从老表删除缓存(兼容模式使用)
|
||||
func (that *CacheDb) deleteLegacy(key string) {
|
||||
legacyTableName := that.getLegacyTableName()
|
||||
|
||||
// 检查老表是否存在
|
||||
if !that.tableExists(legacyTableName) {
|
||||
return
|
||||
}
|
||||
|
||||
del := strings.Index(key, "*")
|
||||
// 如果通配删除
|
||||
if del != -1 {
|
||||
keyPrefix := Substr(key, 0, del)
|
||||
that.Db.Delete(legacyTableName, Map{"key[~]": keyPrefix + "%"})
|
||||
} else {
|
||||
that.Db.Delete(legacyTableName, Map{"key": key})
|
||||
}
|
||||
}
|
||||
|
||||
// get 获取缓存
|
||||
func (that *CacheDb) get(key string) interface{} {
|
||||
tableName := that.getTableName()
|
||||
cached := that.Db.Get(tableName, "*", Map{"key": key})
|
||||
|
||||
if cached != nil {
|
||||
// 使用字符串比较判断过期(ISO 格式天然支持)
|
||||
endTime := cached.GetString("end_time")
|
||||
nowTime := Time2Str(time.Now())
|
||||
if endTime != "" && endTime <= nowTime {
|
||||
// 惰性删除:过期只返回 nil,不立即删除
|
||||
// 依赖随机清理批量删除过期数据
|
||||
// 继续检查老表(如果是兼容模式)
|
||||
} else {
|
||||
// 直接解析 value,不再需要 {"data": value} 包装
|
||||
valueStr := cached.GetString("value")
|
||||
if valueStr != "" {
|
||||
var data interface{}
|
||||
err := json.Unmarshal([]byte(valueStr), &data)
|
||||
if err == nil {
|
||||
return data
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 兼容模式:新表没有数据时,回退读取老表
|
||||
if that.isCompatibleMode() {
|
||||
return that.getLegacy(key)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// set 设置缓存
|
||||
func (that *CacheDb) set(key string, value interface{}, endTime time.Time) {
|
||||
// 直接序列化 value,不再包装
|
||||
@@ -374,6 +479,11 @@ func (that *CacheDb) set(key string, value interface{}, endTime time.Time) {
|
||||
// 写入历史记录
|
||||
that.writeHistory(key)
|
||||
|
||||
// 兼容模式:写新表后删除老表同 key 记录(加速老数据消亡)
|
||||
if that.isCompatibleMode() {
|
||||
that.deleteLegacy(key)
|
||||
}
|
||||
|
||||
// 随机执行删除过期数据命令(5% 概率)
|
||||
if Rand(1000) > 950 {
|
||||
nowTimeStr := Time2Str(time.Now())
|
||||
@@ -387,11 +497,16 @@ func (that *CacheDb) delete(key string) {
|
||||
del := strings.Index(key, "*")
|
||||
// 如果通配删除
|
||||
if del != -1 {
|
||||
key = Substr(key, 0, del)
|
||||
that.Db.Delete(tableName, Map{"key[~]": key + "%"})
|
||||
keyPrefix := Substr(key, 0, del)
|
||||
that.Db.Delete(tableName, Map{"key[~]": keyPrefix + "%"})
|
||||
} else {
|
||||
that.Db.Delete(tableName, Map{"key": key})
|
||||
}
|
||||
|
||||
// 兼容模式:同时删除老表中的 key(避免回退读到老数据)
|
||||
if that.isCompatibleMode() {
|
||||
that.deleteLegacy(key)
|
||||
}
|
||||
}
|
||||
|
||||
// Cache 缓存操作入口
|
||||
@@ -424,9 +539,9 @@ func (that *CacheDb) Cache(key string, data ...interface{}) *Obj {
|
||||
}
|
||||
} else if len(data) >= 2 {
|
||||
// 使用指定的超时时间
|
||||
that.SetError(nil)
|
||||
tempTimeout := ObjToInt64(data[1], that.Error)
|
||||
if that.GetError() == nil && tempTimeout > 0 {
|
||||
var err Error
|
||||
tempTimeout := ObjToInt64(data[1], &err)
|
||||
if err.GetError() == nil && tempTimeout > 0 {
|
||||
timeout = tempTimeout
|
||||
} else {
|
||||
timeout = that.TimeOut
|
||||
@@ -440,3 +555,139 @@ func (that *CacheDb) Cache(key string, data ...interface{}) *Obj {
|
||||
that.set(key, data[0], endTime)
|
||||
return &Obj{Data: nil}
|
||||
}
|
||||
|
||||
// CachesGet 批量获取缓存(使用 IN 查询优化)
|
||||
// 返回 Map,key 为缓存键,value 为缓存值(不存在或过期的 key 不包含在结果中)
|
||||
func (that *CacheDb) CachesGet(keys []string) Map {
|
||||
that.initDbTable()
|
||||
result := make(Map, len(keys))
|
||||
if len(keys) == 0 {
|
||||
return result
|
||||
}
|
||||
|
||||
// #region agent log
|
||||
debugLogDb("E", "cache_db.go:CachesGet:start", "CacheDb.CachesGet开始", map[string]interface{}{
|
||||
"keys_count": len(keys),
|
||||
"keys": keys,
|
||||
})
|
||||
// #endregion
|
||||
|
||||
tableName := that.getTableName()
|
||||
nowTime := Time2Str(time.Now())
|
||||
|
||||
// 使用 IN 查询批量获取
|
||||
cachedList := that.Db.Select(tableName, "*", Map{
|
||||
"key": keys,
|
||||
"end_time[>]": nowTime,
|
||||
})
|
||||
|
||||
// #region agent log
|
||||
debugLogDb("E", "cache_db.go:CachesGet:afterSelect", "DB Select完成", map[string]interface{}{
|
||||
"table": tableName,
|
||||
"now_time": nowTime,
|
||||
"found_rows": len(cachedList),
|
||||
})
|
||||
// #endregion
|
||||
|
||||
for _, cached := range cachedList {
|
||||
valueStr := cached.GetString("value")
|
||||
if valueStr != "" {
|
||||
var data interface{}
|
||||
err := json.Unmarshal([]byte(valueStr), &data)
|
||||
if err == nil {
|
||||
result[cached.GetString("key")] = data
|
||||
} else {
|
||||
// #region agent log
|
||||
debugLogDb("E", "cache_db.go:CachesGet:unmarshalError", "JSON解析失败", map[string]interface{}{
|
||||
"key": cached.GetString("key"),
|
||||
"error": err.Error(),
|
||||
})
|
||||
// #endregion
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 兼容模式:新表没有的 key,回退读取老表
|
||||
if that.isCompatibleMode() {
|
||||
// #region agent log
|
||||
debugLogDb("E", "cache_db.go:CachesGet:compatMode", "兼容模式检查老表", nil)
|
||||
// #endregion
|
||||
for _, key := range keys {
|
||||
if _, exists := result[key]; !exists {
|
||||
legacyData := that.getLegacy(key)
|
||||
if legacyData != nil {
|
||||
result[key] = legacyData
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// #region agent log
|
||||
debugLogDb("E", "cache_db.go:CachesGet:end", "CacheDb.CachesGet完成", map[string]interface{}{
|
||||
"result_count": len(result),
|
||||
})
|
||||
// #endregion
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// CachesSet 批量设置缓存
|
||||
// data: Map,key 为缓存键,value 为缓存值
|
||||
// timeout: 可选过期时间(秒),不传则使用默认超时时间
|
||||
func (that *CacheDb) CachesSet(data Map, timeout ...int64) {
|
||||
that.initDbTable()
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// #region agent log
|
||||
debugLogDb("A", "cache_db.go:CachesSet:start", "CacheDb.CachesSet开始", map[string]interface{}{
|
||||
"data_count": len(data),
|
||||
})
|
||||
// #endregion
|
||||
|
||||
// 计算过期时间
|
||||
var tim int64
|
||||
if len(timeout) > 0 && timeout[0] > 0 {
|
||||
tim = timeout[0]
|
||||
} else {
|
||||
tim = that.TimeOut
|
||||
if tim == 0 {
|
||||
tim = DefaultCacheTimeout
|
||||
}
|
||||
}
|
||||
endTime := time.Now().Add(time.Duration(tim) * time.Second)
|
||||
|
||||
// 逐个设置(保持事务一致性和历史记录)
|
||||
for key, value := range data {
|
||||
that.set(key, value, endTime)
|
||||
}
|
||||
|
||||
// #region agent log
|
||||
debugLogDb("A", "cache_db.go:CachesSet:end", "CacheDb.CachesSet完成", map[string]interface{}{
|
||||
"data_count": len(data),
|
||||
"end_time": Time2Str(endTime),
|
||||
})
|
||||
// #endregion
|
||||
}
|
||||
|
||||
// CachesDelete 批量删除缓存
|
||||
func (that *CacheDb) CachesDelete(keys []string) {
|
||||
that.initDbTable()
|
||||
if len(keys) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
tableName := that.getTableName()
|
||||
|
||||
// 使用 IN 条件批量删除
|
||||
that.Db.Delete(tableName, Map{"key": keys})
|
||||
|
||||
// 兼容模式:同时删除老表中的 keys
|
||||
if that.isCompatibleMode() {
|
||||
legacyTableName := that.getLegacyTableName()
|
||||
if that.tableExists(legacyTableName) {
|
||||
that.Db.Delete(legacyTableName, Map{"key": keys})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+39
@@ -113,3 +113,42 @@ func (c *CacheMemory) Cache(key string, data ...interface{}) *Obj {
|
||||
c.set(key, data[0], expireAt)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CachesGet 批量获取缓存
|
||||
// 返回 Map,key 为缓存键,value 为缓存值(不存在或过期的 key 不包含在结果中)
|
||||
func (c *CacheMemory) CachesGet(keys []string) Map {
|
||||
result := make(Map, len(keys))
|
||||
for _, key := range keys {
|
||||
obj := c.get(key)
|
||||
if obj != nil && obj.Data != nil {
|
||||
result[key] = obj.Data
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// CachesSet 批量设置缓存
|
||||
// data: Map,key 为缓存键,value 为缓存值
|
||||
// timeout: 可选过期时间(秒),不传则使用默认超时时间
|
||||
func (c *CacheMemory) CachesSet(data Map, timeout ...int64) {
|
||||
now := time.Now().Unix()
|
||||
expireAt := now + c.TimeOut
|
||||
if len(timeout) > 0 && timeout[0] > 0 {
|
||||
if timeout[0] > now {
|
||||
expireAt = timeout[0]
|
||||
} else {
|
||||
expireAt = now + timeout[0]
|
||||
}
|
||||
}
|
||||
|
||||
for key, value := range data {
|
||||
c.set(key, value, expireAt)
|
||||
}
|
||||
}
|
||||
|
||||
// CachesDelete 批量删除缓存
|
||||
func (c *CacheMemory) CachesDelete(keys []string) {
|
||||
for _, key := range keys {
|
||||
c.delete(key)
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+96
@@ -188,3 +188,99 @@ func (that *CacheRedis) Cache(key string, data ...interface{}) *Obj {
|
||||
|
||||
return reData
|
||||
}
|
||||
|
||||
// CachesGet 批量获取缓存(使用 Redis MGET 命令优化)
|
||||
// 返回 Map,key 为缓存键,value 为缓存值(不存在的 key 不包含在结果中)
|
||||
func (that *CacheRedis) CachesGet(keys []string) Map {
|
||||
result := make(Map, len(keys))
|
||||
if len(keys) == 0 {
|
||||
return result
|
||||
}
|
||||
|
||||
conn := that.getConn()
|
||||
if conn == nil {
|
||||
return result
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
// 构建 MGET 参数
|
||||
args := make([]interface{}, len(keys))
|
||||
for i, key := range keys {
|
||||
args[i] = key
|
||||
}
|
||||
|
||||
values, err := redis.Strings(conn.Do("MGET", args...))
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "nil returned") {
|
||||
that.Error.SetError(err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// 将结果映射回 Map
|
||||
for i, value := range values {
|
||||
if value != "" {
|
||||
result[keys[i]] = value
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// CachesSet 批量设置缓存(使用 Redis pipeline 优化)
|
||||
// data: Map,key 为缓存键,value 为缓存值
|
||||
// timeout: 可选过期时间(秒),不传则使用默认超时时间
|
||||
func (that *CacheRedis) CachesSet(data Map, timeout ...int64) {
|
||||
if len(data) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
conn := that.getConn()
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
tim := that.TimeOut
|
||||
if len(timeout) > 0 && timeout[0] > 0 {
|
||||
if timeout[0] > tim {
|
||||
tim = timeout[0]
|
||||
} else {
|
||||
tim = tim + timeout[0]
|
||||
}
|
||||
}
|
||||
|
||||
// 使用 pipeline 批量设置
|
||||
conn.Send("MULTI")
|
||||
for key, value := range data {
|
||||
conn.Send("SET", key, ObjToStr(value), "EX", ObjToStr(tim))
|
||||
}
|
||||
_, err := conn.Do("EXEC")
|
||||
if err != nil {
|
||||
that.Error.SetError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// CachesDelete 批量删除缓存(使用 Redis DEL 命令批量删除)
|
||||
func (that *CacheRedis) CachesDelete(keys []string) {
|
||||
if len(keys) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
conn := that.getConn()
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
// 构建 DEL 参数
|
||||
args := make([]interface{}, len(keys))
|
||||
for i, key := range keys {
|
||||
args[i] = key
|
||||
}
|
||||
|
||||
_, err := conn.Do("DEL", args...)
|
||||
if err != nil {
|
||||
that.Error.SetError(err)
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+4
@@ -11,6 +11,10 @@ type CacheIns interface {
|
||||
GetError() *Error
|
||||
SetError(err *Error)
|
||||
Cache(key string, data ...interface{}) *Obj
|
||||
// 批量操作
|
||||
CachesGet(keys []string) Map // 批量获取
|
||||
CachesSet(data Map, timeout ...int64) // 批量设置
|
||||
CachesDelete(keys []string) // 批量删除
|
||||
}
|
||||
|
||||
// 单条缓存数据
|
||||
|
||||
Reference in New Issue
Block a user