前言
记录使用FLASHDB的过程，同步解析源码。
使用分区模式，开启cache。
需要关注：
1结构体sector_hdr_data、kv_hdr_data。
2.迭代器_iterator的使用。
3.理解异常情况GC进程。
fdb_kvdb_init（）
_fdb_init_ex();外部存储介质初始化。_fdb_kv_load();检测扇区、存储的KV，更新到对应的扇区cache和kv cache。
fdb_err_t fdb_kvdb_init(fdb_kvdb_t db, const char *name, const char *path, struct fdb_default_kv *default_kv,
void *user_data)
{
fdb_err_t result = FDB_NO_ERR;
struct kvdb_sec_info sector;
#ifdef FDB_KV_USING_CACHE
size_t i;
#endif
/* must be aligned with write granularity */
FDB_ASSERT((FDB_STR_KV_VALUE_MAX_SIZE * 8) % FDB_WRITE_GRAN == 0);
/*包含初始化存储介质、更新分区信息到当前数据库db */
result = _fdb_init_ex((fdb_db_t) db, name, path, FDB_DB_TYPE_KV, user_data);
if (result != FDB_NO_ERR) {
goto __exit;
}
/* lock the KVDB 互斥锁*/
db_lock(db);
db->gc_request = false;
db->in_recovery_check = false;
/*检查是否存在默认kv要写入*/
if (default_kv) {
db->default_kvs = *default_kv;
} else {
db->default_kvs.num = 0;
db->default_kvs.kvs = NULL;
}
{ /* find the oldest sector address */
uint32_t sector_oldest_addr = 0;
fdb_sector_store_status_t last_sector_status = FDB_SECTOR_STORE_UNUSED;
db_oldest_addr(db) = 0;
FDB_DEBUG("size:%d\n",db_max_size(db));
/*遍历当前分区。默认db_oldest_addr(db)为0 这个地址为分区起始地址的扇区偏移， */
sector_iterator(db, §or, FDB_SECTOR_STORE_UNUSED, §or_oldest_addr, &last_sector_status,
check_oldest_addr_cb, false);
db_oldest_addr(db) = sector_oldest_addr;
FDB_DEBUG("The oldest addr is @0x%08" PRIX32 "\n", db_oldest_addr(db));
}
/* there is at least one empty sector for GC. */
FDB_ASSERT((FDB_GC_EMPTY_SEC_THRESHOLD > 0 && FDB_GC_EMPTY_SEC_THRESHOLD < SECTOR_NUM))
/*扇区cache进行初始配置*/
#ifdef FDB_KV_USING_CACHE
for (i = 0; i < FDB_SECTOR_CACHE_TABLE_SIZE; i++) {
db->sector_cache_table[i].check_ok = false;
db->sector_cache_table[i].empty_kv = FAILED_ADDR;
db->sector_cache_table[i].addr = FDB_DATA_UNUSED;
}
/*KV cache 地址初始配置*/
for (i = 0; i < FDB_KV_CACHE_TABLE_SIZE; i++) {
db->kv_cache_table[i].addr = FDB_DATA_UNUSED;
}
#endif /* FDB_KV_USING_CACHE */
FDB_DEBUG("KVDB size is %" PRIu32 " bytes.\n", db_max_size(db));
db_unlock(db);
/*这个函数是核心：会对扇区和存储的KV检查，并对GC失败或者KV写入异常情况进行处理。*/
result = _fdb_kv_load(db);
db_lock(db);
#ifdef FDB_KV_AUTO_UPDATE
if (result == FDB_NO_ERR) {
kv_auto_update(db);
}
#endif
/* unlock the KVDB */
db_unlock(db);
__exit:
_fdb_init_finish((fdb_db_t)db, result);
return result;
}
_fdb_init_ex((fdb_db_t) db, name, path, FDB_DB_TYPE_KV, user_data);
初始化存储介质、绑定分区所对应的存储介质 及参数检测。
fdb_err_t _fdb_init_ex(fdb_db_t db, const char *name, const char *path, fdb_db_type type, void *user_data)
{
FDB_ASSERT(db);
FDB_ASSERT(name);
FDB_ASSERT(path);
if (db->init_ok) {
return FDB_NO_ERR;
}
db->name = name;/* 数据库命名*/
db->type = type;/* 数据库类型：KV 或者TS*/
db->user_data = user_data; /*用户数据*/
/* 文件模式设置在数据库初始化之前 通过API:fdb_kvdb_control */
if (db->file_mode) {
#ifdef FDB_USING_FILE_MODE
memset(db->cur_file_sec, FDB_FAILED_ADDR, FDB_FILE_CACHE_TABLE_SIZE * sizeof(db->cur_file_sec[0]));
/* must set when using file mode */
FDB_ASSERT(db->sec_size != 0);
FDB_ASSERT(db->max_size != 0);
#ifdef FDB_USING_FILE_POSIX_MODE
memset(db->cur_file, -1, FDB_FILE_CACHE_TABLE_SIZE * sizeof(db->cur_file[0]));
#else
memset(db->cur_file, 0, FDB_FILE_CACHE_TABLE_SIZE * sizeof(db->cur_file[0]));
#endif
db->storage.dir = path;
FDB_ASSERT(strlen(path) != 0)
#endif
} else {
#ifdef FDB_USING_FAL_MODE/*分区模式*/
size_t block_size;
/* FAL (Flash Abstraction Layer) initialization */
fal_init();/*1. 存储介质初始化 2.检查每一个分区所处于的存储介质是否存在（通过name匹配）,绑定设备到对应的分区cache 中*/
/* 检查当前要初始化的分区是否存在，存在则更新到数据库下*/
if ((db->storage.part = fal_partition_find(path)) == NULL) {
FDB_INFO("Error: Partition (%s) not found.\n", path);
return FDB_PART_NOT_FOUND;
}
block_size = fal_flash_device_find(db->storage.part->flash_name)->blk_size;/*查找擦除数据块大小*/
if (db->sec_size == 0) {
db->sec_size = block_size;/* 块大小 更新到数据库*/
} else {
/* must be aligned with block size */
if (db->sec_size % block_size != 0) {
FDB_INFO("Error: db sector size (%" PRIu32 ") MUST align with block size (%zu).\n", db->sec_size, block_size);
return FDB_INIT_FAILED;
}
}
db->max_size = db->storage.part->len;//当前分区设置的容量
#endif /* FDB_USING_FAL_MODE */
}
/*扇区大小为2^n次幂 */
FDB_ASSERT((db->sec_size & (db->sec_size - 1)) == 0);
/* must align with sector size */
if (db->max_size % db->sec_size != 0) {
FDB_INFO("Error: db total size (%" PRIu32 ") MUST align with sector size (%" PRIu32 ").\n", db->max_size, db->sec_size);
return FDB_INIT_FAILED;
}
/* must has more than or equal 2 sectors 使用的扇区数目最小不能低于2 ：因为最少必须预留一个GC扇区 */
if (db->max_size / db->sec_size < 2) {
FDB_INFO("Error: db MUST has more than or equal 2 sectors, current has %" PRIu32 " sector(s)\n", db->max_size / db->sec_size);
return FDB_INIT_FAILED;
}
return FDB_NO_ERR;
}
通过函数check_and_update_part_cache(const struct fal_partition *table, size_t len) 绑定分区所对应的存储介质。
static int check_and_update_part_cache(const struct fal_partition *table, size_t len)
{
const struct fal_flash_dev *flash_dev = NULL;
size_t i;
for (i = 0; i < len; i++) {
flash_dev = fal_flash_device_find(table[i].flash_name);//分区名称和设备名称匹配，返回设备结构体地址
if (flash_dev == NULL) {
log_d("Warning: Do NOT found the flash device(%s).", table[i].flash_name);
continue;
}
if (table[i].offset >= (long)flash_dev->len) { //检查分区起始地址是否大于设备容量
log_e("Initialize failed! Partition(%s) offset address(%ld) out of flash bound(<%d).",
table[i].name, table[i].offset, flash_dev->len);
partition_table_len = 0;//超过设备容量，失败，清空分区数目。
return -1;
}
part_flash_cache[i].flash_dev = flash_dev;// 把匹配的flash设备结构 更新到part_flash_cache列表中
}
return 0;
}
_fdb_kv_load(db)
_fdb_kv_load(db)是初始化的关键函数。
首先需要关注几个结构体：
struct sector_hdr_data{}；格式化后，分区下每个扇区起始位置存储固定格式数据，用于扇区检测。
`struct sector_hdr_data {
struct {
uint8_t store[FDB_STORE_STATUS_TABLE_SIZE]; /*扇区存储状态*/
uint8_t dirty[FDB_DIRTY_STATUS_TABLE_SIZE]; /*删除已有KV置为TRUE */
} status_table;
uint32_t magic; /* 扇区格式化标志magic word(`E`, `F`, `4`, `0`) */
uint32_t combined; /**< the combined next sector number, default: not combined 可不用，默认即可*/
uint32_t reserved;
#if (FDB_WRITE_GRAN == 64) || (FDB_WRITE_GRAN == 128)
uint8_t padding[4]; /**< align padding for 64bit and 128bit write granularity */
#endif
};
sector cache 扇区信息缓存。
/* KVDB section information */
struct kvdb_sec_info {
bool check_ok; /**< sector header check is OK */
struct {
fdb_sector_store_status_t store; /**< sector store status @see fdb_sector_store_status_t */
fdb_sector_dirty_status_t dirty; /**< sector dirty status @see sector_dirty_status_t */
} status;
uint32_t addr; /**< sector start address */
uint32_t magic; /**< magic word(`E`, `F`, `4`, `0`) */
uint32_t combined; /**< the combined next sector number, 0xFFFFFFFF: not combined */
size_t remain; /**< remain size */
uint32_t empty_kv; /**< the next empty KV node start address */
};
struct kv_hdr_data {}；每个KV都以固定格式存储，用于KV数据检测。
struct kv_hdr_data {
uint8_t status_table[KV_STATUS_TABLE_SIZE]; /**< KV node status, @see fdb_kv_status_t */
uint32_t magic; /**< magic word(`K`, `V`, `0`, `0`) */
uint32_t len; /**< KV node total length (header + name + value), must align by FDB_WRITE_GRAN */
uint32_t crc32; /**< KV node crc32(name_len + data_len + name + value) */
uint8_t name_len; /**< name length */
uint32_t value_len; /**< value length */
#if (FDB_WRITE_GRAN == 64)
uint8_t padding[4]; /**< align padding for 64bit write granularity */
#endif
#if (FDB_WRITE_GRAN == 128)
uint8_t padding[12]; /**< align padding for 128bit write granularity */
#endif
};
kv cache
struct kv_cache_node {
uint16_t name_crc; /**< KV name's CRC32 low 16bit value */
uint16_t active; /**< KV node access active degree */
uint32_t addr; /**< KV node address */
};
_fdb_kv_load（）
static fdb_err_t _fdb_kv_load(fdb_kvdb_t db)
{
fdb_err_t result = FDB_NO_ERR;
struct fdb_kv kv;
struct kvdb_sec_info sector;
size_t check_failed_count = 0;
db->in_recovery_check = true;
/* 通过检测分区下的每个扇区状态，统计损坏或者未格式化的扇区个数，并格式化损坏扇区，更新到sector cache。*/
sector_iterator(db, §or, FDB_SECTOR_STORE_UNUSED, &check_failed_count, db, check_sec_hdr_cb, false);
if (db->parent.not_formatable && check_failed_count > 0) {
return FDB_READ_ERR;
}
/* 假设分区下所有扇区未被格式化过，会再次格式化一遍;并检测是否有默认的KV要写入，如有，则写入*/
if (check_failed_count == SECTOR_NUM) {
FDB_INFO("All sector header is incorrect. Set it to default.\n");
fdb_kv_set_default(db);
}
/* 如果上次GC过程异常，扇区状态：FDB_SECTOR_DIRTY_GC，重新进行GC过程。 */
sector_iterator(db, §or, FDB_SECTOR_STORE_UNUSED, db, NULL, check_and_recovery_gc_cb, false);
__retry:
/* 1.检测分区下KV是否完整写入或者是否删除成功。写入不完整，则更新KV标志为错误状态(FDB_KV_ERR_HDR); 若删除失败，则迁移数据到可用扇区，并重新删除旧KV。2.更新KV到KV_cache中。*/
kv_iterator(db, &kv, db, NULL, check_and_recovery_kv_cb);
/*迁移数据到预留GC区域*/
if (db->gc_request) {
gc_collect(db);
goto __retry;
}
db->in_recovery_check = false;
return result;
}
对内部的函数逐个分析
1. sector_iterator(db, §or, FDB_SECTOR_STORE_UNUSED, &check_failed_count, db, check_sec_hdr_cb, false);
首先使用sector_iterator()遍历当前分区下的所有扇区，read_sector_info(db, sec_addr, sector, false)读取扇区信息，通过回调函数check_sec_hdr_cb()统计被破坏或者未格式化的扇区个数,，并对检测失败的扇区进行格式化。
static void sector_iterator(fdb_kvdb_t db, kv_sec_info_t sector, fdb_sector_store_status_t status, void *arg1, void *arg2, bool (*callback)(kv_sec_info_t sector, void *arg1, void *arg2), bool traversal_kv)
{
uint32_t sec_addr, traversed_len = 0;
/* search all sectors */
sec_addr = db_oldest_addr(db);/*GC后会改变db_oldest_addr(db)，默认扇区偏移0开始*/
do {
traversed_len += db_sec_size(db);/*偏移一个扇区大小*/
read_sector_info(db, sec_addr, sector, false);
if (status == FDB_SECTOR_STORE_UNUSED || status == sector->status.store) {
if (traversal_kv) {
read_sector_info(db, sec_addr, sector, true);
}
/* iterator is interrupted when callback return true */
if (callback && callback(sector, arg1, arg2)) {
return;
}
}
} while ((sec_addr = get_next_sector_addr(db, sector, traversed_len)) != FAILED_ADDR);/*扇区偏移地址*/
}
回调函数：check_sec_hdr_cb（）
统计检测失败的扇区个数，并格式化失败扇区。
static bool check_sec_hdr_cb(kv_sec_info_t sector, void *arg1, void *arg2)
{
/*扇区检测失败*/
if (!sector->check_ok) {
size_t *failed_count = arg1;
fdb_kvdb_t db = arg2;
(*failed_count) ++;/*统计失败的扇区*/
if (db->parent.not_formatable) {
return true;
} else {/*对扇区进行格式化，同时写入默认的扇区头数据，更新到sector cache*/
FDB_INFO("Sector header info is incorrect. Auto format this sector (0x%08" PRIX32 ").\n", sector->addr);
format_sector(db, sector->addr, SECTOR_NOT_COMBINED);
//DEBUG_PRINTF ("sector->addr:%d \n",sector->addr);
}
}
return false;
}
格式化函数： format_sector(db, sector->addr, SECTOR_NOT_COMBINED);
格式化失败扇区更新 sector cache 。
static fdb_err_t format_sector(fdb_kvdb_t db, uint32_t addr, uint32_t combined_value)
{
fdb_err_t result = FDB_NO_ERR;
struct sector_hdr_data sec_hdr = { 0 };
FDB_ASSERT(addr % db_sec_size(db) == 0);
/*addr：分区扇区偏移地址 擦除一个扇区*/
result = _fdb_flash_erase((fdb_db_t)db, addr, db_sec_size(db));
if (result == FDB_NO_ERR) {
/* initialize the header data */
memset(&sec_hdr, FDB_BYTE_ERASED, sizeof(struct sector_hdr_data));
#if (FDB_WRITE_GRAN == 1)
_fdb_set_status(sec_hdr.status_table.store, FDB_SECTOR_STORE_STATUS_NUM, FDB_SECTOR_STORE_EMPTY);/*扇区store状态unuse->empty*/
_fdb_set_status(sec_hdr.status_table.dirty, FDB_SECTOR_DIRTY_STATUS_NUM, FDB_SECTOR_DIRTY_FALSE);/*扇区dirty状态unuse->dirty-false*/
sec_hdr.magic = SECTOR_MAGIC_WORD;
sec_hdr.combined = combined_value;
sec_hdr.reserved = FDB_DATA_UNUSED;
/* save the header 写入sector header数据*/
result = _fdb_flash_write((fdb_db_t)db, addr, (uint32_t *)&sec_hdr, SECTOR_HDR_DATA_SIZE, true);
#else // seperate the whole "sec_hdr" program to serval sinle program operation to prevent re-program issue on STM32L4xx or
// other MCU internal flash
/* write the sector store status */
_fdb_write_status((fdb_db_t)db,
addr + SECTOR_STORE_OFFSET,
sec_hdr.status_table.store,
FDB_SECTOR_STORE_STATUS_NUM,
FDB_SECTOR_STORE_EMPTY,
true);
/* write the sector dirty status */
_fdb_write_status((fdb_db_t)db,
addr + SECTOR_DIRTY_OFFSET,
sec_hdr.status_table.dirty,
FDB_SECTOR_DIRTY_STATUS_NUM,
FDB_SECTOR_DIRTY_FALSE,
true);
/* write the magic word and combined next sector number */
sec_hdr.magic = SECTOR_MAGIC_WORD;
sec_hdr.combined = combined_value;
sec_hdr.reserved = FDB_DATA_UNUSED;
result = _fdb_flash_write((fdb_db_t)db,
addr + SECTOR_MAGIC_OFFSET,
(void *)(&(sec_hdr.magic)),
(sizeof(struct sector_hdr_data) - SECTOR_MAGIC_OFFSET),
true);
#endif
#ifdef FDB_KV_USING_CACHE
{/* 更新扇区信息到sector cache*/
struct kvdb_sec_info sector = {.addr = addr, .check_ok = false, .empty_kv = FAILED_ADDR };
/* delete the sector cache */
update_sector_cache(db, §or);
}
#endif /* FDB_KV_USING_CACHE */
}
return result;
}
read_sector_info()
参数：traversal=flase时,只从cache中读取扇区信息；raversal=true时，还需要对扇区的空闲区域进行统计，主要更新sector->remain(空闲空间大小) 、sector->empty_kv(空闲区域起始地址)。
static fdb_err_t read_sector_info(fdb_kvdb_t db, uint32_t addr, kv_sec_info_t sector, bool traversal)
{
fdb_err_t result = FDB_NO_ERR;
struct sector_hdr_data sec_hdr = { 0 };
FDB_ASSERT(addr % db_sec_size(db) == 0);
FDB_ASSERT(sector);
#ifdef FDB_KV_USING_CACHE
/*先从扇区cache 中查找扇区信息*/
kv_sec_info_t sector_cache = get_sector_from_cache(db, addr);
if (sector_cache && ((!traversal) || (traversal && sector_cache->empty_kv != FAILED_ADDR))) {
memcpy(sector, sector_cache, sizeof(struct kvdb_sec_info));
return result;
}
#endif /* FDB_KV_USING_CACHE */
/* read sector header raw data 直接从FLASH中读取扇区头数据 */
_fdb_flash_read((fdb_db_t)db, addr, (uint32_t *)&sec_hdr, sizeof(struct sector_hdr_data));
sector->status.store = FDB_SECTOR_STORE_UNUSED;
sector->status.dirty = FDB_SECTOR_DIRTY_UNUSED;
sector->addr = addr;
sector->magic = sec_hdr.magic;
/* check magic word and combined value 检查扇区是否有效*/
if (sector->magic != SECTOR_MAGIC_WORD ||
(sec_hdr.combined != SECTOR_NOT_COMBINED && sec_hdr.combined != SECTOR_COMBINED)) {
sector->check_ok = false;
sector->combined = SECTOR_NOT_COMBINED;
return FDB_INIT_FAILED;
}
sector->check_ok = true;/*扇区有效*/
/* get other sector info */
sector->combined = sec_hdr.combined;
/* 获取当前扇区的store状态及dirty状态*/
sector->status.store = (fdb_sector_store_status_t) _fdb_get_status(sec_hdr.status_table.store, FDB_SECTOR_STORE_STATUS_NUM);
sector->status.dirty = (fdb_sector_dirty_status_t) _fdb_get_status(sec_hdr.status_table.dirty, FDB_SECTOR_DIRTY_STATUS_NUM);
/* traversal all KV and calculate the remain space size 计算扇区可用空闲地址和剩余空间大小 */
if (traversal) {
sector->remain = 0;
sector->empty_kv = sector->addr + SECTOR_HDR_DATA_SIZE;/*扇区空闲空间：起始地址*/
if (sector->status.store == FDB_SECTOR_STORE_EMPTY) {
sector->remain = db_sec_size(db) - SECTOR_HDR_DATA_SIZE;/* 当前扇区未被使用 空闲空间大小为：扇区大小减去扇区头数据*/
}
else if (sector->status.store == FDB_SECTOR_STORE_USING) /*当前扇区已被使用 计算剩余部分空闲空间*/
{
struct fdb_kv kv_obj;
sector->remain = db_sec_size(db) - SECTOR_HDR_DATA_SIZE;
kv_obj.addr.start = sector->addr + SECTOR_HDR_DATA_SIZE;
/*遍历扇区 找寻空闲区域的起始地址*/
do {
read_kv(db, &kv_obj);/* 读取KV */
if (!kv_obj.crc_is_ok) {
if (kv_obj.status != FDB_KV_PRE_WRITE && kv_obj.status != FDB_KV_ERR_HDR) {
sector->remain = 0;
result = FDB_READ_ERR;
break;
}
}
Feed_DOG();
sector->empty_kv += kv_obj.len;/* 下一个KV */
sector->remain -= kv_obj.len;
} while ((kv_obj.addr.start = get_next_kv_addr(db, sector, &kv_obj)) != FAILED_ADDR);
/* check the empty KV address by read continue 0xFF on flash */
{
uint32_t ff_addr;
/*遍历sector->remain起始的区域 返回连续区域是0xff 的首地址*/
ff_addr = _fdb_continue_ff_addr((fdb_db_t)db, sector->empty_kv, sector->addr + db_sec_size(db));
/* 空闲区域地址不匹配：可能前面存在损坏的KV*/
if (sector->empty_kv != ff_addr) {
/* update the sector information 更新为连续检测查找的空闲地址*/
sector->empty_kv = ff_addr;
sector->remain = db_sec_size(db) - (ff_addr - sector->addr);
//FDB_INFO("status:using,empty_kv[0x%x],remain[x%0x]\r\n",sector->empty_kv,sector->remain);
}
}
}
#ifdef FDB_KV_USING_CACHE
update_sector_cache(db, sector);/*重新更新到sector cache */
} else {
kv_sec_info_t sec_cache = get_sector_from_cache(db, sector->addr);
if (!sec_cache) {
sector->empty_kv = FAILED_ADDR;
sector->remain = 0;
update_sector_cache(db, sector);
}
#endif
}
return result;
}
2.fdb_kv_set_default(db)
1.格式化当前分区下的所有扇区。
2.写入默认KV数据。
fdb_err_t fdb_kv_set_default(fdb_kvdb_t db)
{
fdb_err_t result = FDB_NO_ERR;
uint32_t addr, i, value_len;
struct kvdb_sec_info sector;
/* lock the KV cache */
db_lock(db);
#ifdef FDB_KV_USING_CACHE
for (i = 0; i < FDB_KV_CACHE_TABLE_SIZE; i++) {
db->kv_cache_table[i].addr = FDB_DATA_UNUSED;
}
#endif /* FDB_KV_USING_CACHE */
/* format all sectors */
for (addr = 0; addr < db_max_size(db); addr += db_sec_size(db)) {
result = format_sector(db, addr, SECTOR_NOT_COMBINED);
if (result != FDB_NO_ERR) {
goto __exit;
}
}
/* create default KV */
for (i = 0; i < db->default_kvs.num; i++) {
/* It seems to be a string when value length is 0.
* This mechanism is for compatibility with older versions (less then V4.0). */
if (db->default_kvs.kvs[i].value_len == 0) {
value_len = strlen(db->default_kvs.kvs[i].value);
} else {
value_len = db->default_kvs.kvs[i].value_len;
}
sector.empty_kv = FAILED_ADDR;
create_kv_blob(db, §or, db->default_kvs.kvs[i].key, db->default_kvs.kvs[i].value, value_len);
if (result != FDB_NO_ERR) {
goto __exit;
}
}
__exit:
db_oldest_addr(db) = 0;
/* unlock the KV cache */
db_unlock(db);
return result;
}
3.sector_iterator(db, §or, FDB_SECTOR_STORE_UNUSED, db, NULL, check_and_recovery_gc_cb, false);
使用sector_iterator()遍历当前分区下的扇区，read_sector_info(db, sec_addr, sector, false)读取扇区信息，通过回调函数check_and_recovery_gc_cb()检测是否需要重新恢复GC进程。
主要看下回调函数：
检测有效扇区dirty状态，如果是dirty=FDB_SECTOR_DIRTY_GC，说明上次的GC过程异常中止，未执行完毕，因为执行完毕，需要擦除GC扇区，扇区dirty状态应该是扇区擦除后的状态。
tatic bool check_and_recovery_gc_cb(kv_sec_info_t sector, void *arg1, void *arg2)
{
fdb_kvdb_t db = arg1;
if (sector->check_ok && sector->status.dirty == FDB_SECTOR_DIRTY_GC) {
/* make sure the GC request flag to true */
db->gc_request = true;/*需重新执行GC*/
/* resume the GC operate */
gc_collect(db);
}
return false;
}
假设执行GC，是如何处理的呢？
通过gc_collect()->gc_collect_by_free_size();
1.需要明确什么时候会执行GC？除FDB_GC_EMPTY_SEC_THRESHOLD定义的预留扇区之外，分区下其他扇区空间不足以写入当前数据量的情况下，触发GC进程。
static void gc_collect_by_free_size(fdb_kvdb_t db, size_t free_size)
{
struct kvdb_sec_info sector;
size_t empty_sec = 0;
struct gc_cb_args arg = { db, 0, free_size, 0 };
/* GC check the empty sector number 先检查GC空间大小：几个 sectors*/
sector_iterator(db, §or, FDB_SECTOR_STORE_EMPTY, &empty_sec, NULL, gc_check_cb, false);
/* do GC collect */
FDB_DEBUG("The remain empty sector is %" PRIu32 ", GC threshold is %" PRIdLEAST16 ".\n", (uint32_t)empty_sec, FDB_GC_EMPTY_SEC_THRESHOLD);
/* 注意此处<= 说明分区扇区空闲空间不足 要使用GC区域了 */
if (empty_sec <= FDB_GC_EMPTY_SEC_THRESHOLD) {
sector_iterator(db, §or, FDB_SECTOR_STORE_UNUSED, &arg, NULL, do_gc, false);
}
db->gc_request = false;
}
上面函数有两个迭代器，第一个迭代器函数sector_iterator(db, §or, FDB_SECTOR_STORE_EMPTY, &empty_sec, NULL, gc_check_cb, false);参数：FDB_SECTOR_STORE_EMPTY查找对应扇区的存储状态，说明需要找寻未使用的扇区信息。
回调函数gc_check_cb()统计空闲扇区的个数：
static bool gc_check_cb(kv_sec_info_t sector, void *arg1, void *arg2)
{
size_t *empty_sec = arg1;
/*统计空闲扇区的个数*/
if (sector->check_ok) {
*empty_sec = *empty_sec + 1;
}
return false;
}
第二个迭代器函数开始执行GC进程：
如果empty_sec <= FDB_GC_EMPTY_SEC_THRESHOLD，执行GC进程。
回调函数do_gc() 执行GC进程
1.sector->status.dirty == FDB_SECTOR_DIRTY_GC 就是我们假设GC进程异常的情况。
2.数据迁移完毕后，格式化扇区。
static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2)
{
struct fdb_kv kv;
struct gc_cb_args *gc = (struct gc_cb_args *)arg1;
fdb_kvdb_t db = gc->db;
// do gc 扇区状态FDB_SECTOR_DIRTY_TRUE:删除KV;FDB_SECTOR_DIRTY_GC:GC失败的。
if (sector->check_ok && (sector->status.dirty == FDB_SECTOR_DIRTY_TRUE || sector->status.dirty == FDB_SECTOR_DIRTY_GC)) {
uint8_t status_table[FDB_DIRTY_STATUS_TABLE_SIZE];
/* change the sector status to GC */
_fdb_write_status((fdb_db_t)db, sector->addr + SECTOR_DIRTY_OFFSET, status_table, FDB_SECTOR_DIRTY_STATUS_NUM, FDB_SECTOR_DIRTY_GC, true);
/* search all KV */
kv.addr.start = sector->addr + SECTOR_HDR_DATA_SIZE;
do {
read_kv(db, &kv);
//KV数据正确 才会move_kv：检查分区每个扇区的情况 能插入数据就插入 并更新扇区的状态 kv的状态
if (kv.crc_is_ok && (kv.status == FDB_KV_WRITE || kv.status == FDB_KV_PRE_DELETE)) {
/* move the KV to new space */
if (move_kv(db, &kv) != FDB_NO_ERR) {
FDB_INFO("Error: Moved the KV (%.*s) for GC failed.\n", kv.name_len, kv.name);
}
}
} while ((kv.addr.start = get_next_kv_addr(db, sector, &kv)) != FAILED_ADDR);
format_sector(db, sector->addr, SECTOR_NOT_COMBINED);
gc->cur_free_size += db_sec_size(db) - SECTOR_HDR_DATA_SIZE;
FDB_DEBUG("Collect a sector @0x%08" PRIX32 "\n", sector->addr);
/* update oldest_addr for next GC sector format */
db_oldest_addr(db) = get_next_sector_addr(db, sector, 0);
FDB_DEBUG("gc_old_adrr:%d,cur_free_size:%d,setting_free_size:%d\r\n",db_oldest_addr(db),gc->cur_free_size,gc->setting_free_size);
if (gc->cur_free_size >= gc->setting_free_size)
return true;
}
return false;
}
GC时数据是怎么样的逻辑迁移的呢？
while()里面有两个重要函数
1.read_kv() 读取当前扇区的一个KV，检测是否有效。
每个KV都包含固定的kv_hdr_data ，检测kv_hdr_data 即可判断KV的状态。
static fdb_err_t read_kv(fdb_kvdb_t db, fdb_kv_t kv)
{
struct kv_hdr_data kv_hdr;
uint8_t buf[32];
uint32_t calc_crc32 = 0, crc_data_len, kv_name_addr;
fdb_err_t result = FDB_NO_ERR;
size_t len, size;
/* read KV header raw data */
_fdb_flash_read((fdb_db_t)db, kv->addr.start, (uint32_t *)&kv_hdr, sizeof(struct kv_hdr_data));
kv->status = (fdb_kv_status_t) _fdb_get_status(kv_hdr.status_table, FDB_KV_STATUS_NUM);
kv->len = kv_hdr.len;
/*KV 总长度检测 */
if (kv->len == UINT32_MAX || kv->len > db_max_size(db) || kv->len < KV_HDR_DATA_SIZE) {
/* the KV length was not write, so reserved the info for current KV */
kv->len = KV_HDR_DATA_SIZE;
if (kv->status != FDB_KV_ERR_HDR) {
kv->status = FDB_KV_ERR_HDR;
FDB_INFO("Error: The KV @0x%08" PRIX32 " length has an error.\n", kv->addr.start);
_fdb_write_status((fdb_db_t)db, kv->addr.start, kv_hdr.status_table, FDB_KV_STATUS_NUM, FDB_KV_ERR_HDR, true);/* kv 数据异常:错误状态 */
}
kv->crc_is_ok = false;
return FDB_READ_ERR;
} else if (kv->len > db_sec_size(db) - SECTOR_HDR_DATA_SIZE && kv->len < db_max_size(db)) {
//TODO Sector continuous mode, or the write length is not written completely
}
/* CRC32 data len(header.name_len + header.value_len + name + value), using sizeof(uint32_t) for compatible V1.x */
calc_crc32 = fdb_calc_crc32(calc_crc32, &kv_hdr.name_len, sizeof(uint32_t));/*KV_NAME CRC校验 */
calc_crc32 = fdb_calc_crc32(calc_crc32, &kv_hdr.value_len, sizeof(uint32_t));/*KV 真实数据长度 */
crc_data_len = kv->len - KV_HDR_DATA_SIZE;//KV_HDR_DATA_SIZE:24
/* calculate the CRC32 value name value 再计算name 和数据 */
for (len = 0, size = 0; len < crc_data_len; len += size) {
if (len + sizeof(buf) < crc_data_len) {
size = sizeof(buf);
} else {
size = crc_data_len - len;
}
_fdb_flash_read((fdb_db_t)db, kv->addr.start + KV_HDR_DATA_SIZE + len, (uint32_t *) buf, FDB_WG_ALIGN(size));
calc_crc32 = fdb_calc_crc32(calc_crc32, buf, size);
}
/* check CRC32 判断数据是否有效*/
if (calc_crc32 != kv_hdr.crc32) {/*数据无效 返回错误*/
size_t name_len = kv_hdr.name_len > FDB_KV_NAME_MAX ? FDB_KV_NAME_MAX : kv_hdr.name_len;
kv->crc_is_ok = false;
result = FDB_READ_ERR;
/* try read the KV name, maybe read name has error */
kv_name_addr = kv->addr.start + KV_HDR_DATA_SIZE;
_fdb_flash_read((fdb_db_t)db, kv_name_addr, (uint32_t *)kv->name, FDB_WG_ALIGN(name_len));
FDB_INFO("Error: Read the KV (%.*s@0x%08" PRIX32 ") CRC32 check failed!\n", name_len, kv->name, kv->addr.start);
} else {
kv->crc_is_ok = true;/*数据有效 */
/* the name is behind aligned KV header */
kv_name_addr = kv->addr.start + KV_HDR_DATA_SIZE;/* KV name 地址*/
_fdb_flash_read((fdb_db_t)db, kv_name_addr, (uint32_t *) kv->name, FDB_WG_ALIGN(kv_hdr.name_len));/* 读取KV name*/
/* the value is behind aligned name */
kv->addr.value = kv_name_addr + FDB_WG_ALIGN(kv_hdr.name_len);/* 实际数据起始地址*/
kv->value_len = kv_hdr.value_len;/* 有效数据长度*/
kv->name_len = kv_hdr.name_len; /* kv NAME 长度*/
if (kv_hdr.name_len >= sizeof(kv->name) / sizeof(kv->name[0])) {/*添加结束符*/
kv_hdr.name_len = sizeof(kv->name) / sizeof(kv->name[0]) - 1;
}
kv->name[kv_hdr.name_len] = '\0';
}
return result;
}
2.move_kv()数据迁移到其他扇区可存放的区域。
static fdb_err_t move_kv(fdb_kvdb_t db, fdb_kv_t kv)
{
fdb_err_t result = FDB_NO_ERR;
uint8_t status_table[KV_STATUS_TABLE_SIZE];
uint32_t kv_addr;
struct kvdb_sec_info sector;
/* prepare to delete the current KV */
if (kv->status == FDB_KV_WRITE) {
del_kv(db, NULL, kv, false);
}
//kv_addr 要迁移到的扇区空闲区域首地址
if ((kv_addr = alloc_kv(db, §or, kv->len)) != FAILED_ADDR) {
if (db->in_recovery_check && kv->status == FDB_KV_PRE_DELETE) {
struct fdb_kv kv_bak;
char name[FDB_KV_NAME_MAX + 1] = { 0 };
strncpy(name, kv->name, kv->name_len);
/* check the KV in flash is already create success */
if (find_kv_no_cache(db, name, &kv_bak)) {
/* already create success, don't need to duplicate */
result = FDB_NO_ERR;
goto __exit;
}
}
} else {
return FDB_SAVED_FULL;
}
/* start move the KV */
{
uint8_t buf[32];
size_t len, size, kv_len = kv->len;
/* update the new KV sector status first 改变扇区状态using or full */
update_sec_status(db, §or, kv->len, NULL);
_fdb_write_status((fdb_db_t)db, kv_addr, status_table, FDB_KV_STATUS_NUM, FDB_KV_PRE_WRITE, false);/* 改变KV状态 准备写入 */
kv_len -= KV_MAGIC_OFFSET;//去掉KV状态 迁移后面数据
//分几次读取 最多32字节
for (len = 0, size = 0; len < kv_len; len += size) {
if (len + sizeof(buf) < kv_len) {
size = sizeof(buf);
} else {
size = kv_len - len;
}
_fdb_flash_read((fdb_db_t)db, kv->addr.start + KV_MAGIC_OFFSET + len, (uint32_t *) buf, FDB_WG_ALIGN(size));
result = _fdb_flash_write((fdb_db_t)db, kv_addr + KV_MAGIC_OFFSET + len, (uint32_t *) buf, size, true);
}
_fdb_write_status((fdb_db_t)db, kv_addr, status_table, FDB_KV_STATUS_NUM, FDB_KV_WRITE, true);/* 改变KV状态 写入完成 */
#ifdef FDB_KV_USING_CACHE
update_sector_empty_addr_cache(db, FDB_ALIGN_DOWN(kv_addr, db_sec_size(db)),
kv_addr + KV_HDR_DATA_SIZE + FDB_WG_ALIGN(kv->name_len) + FDB_WG_ALIGN(kv->value_len));
update_kv_cache(db, kv->name, kv->name_len, kv_addr);
#endif /* FDB_KV_USING_CACHE */
}
FDB_DEBUG("Moved the KV (%.*s) from 0x%08" PRIX32 " to 0x%08" PRIX32 ".\n", kv->name_len, kv->name, kv->addr.start, kv_addr);
__exit:
del_kv(db, NULL, kv, true);//删除旧的KV
return result;
}
其中alloc_kv(db, §or, kv->len))查找可用扇区区域。
static uint32_t alloc_kv(fdb_kvdb_t db, kv_sec_info_t sector, size_t kv_size)
{
uint32_t empty_kv = FAILED_ADDR;
size_t empty_sector = 0, using_sector = 0;
struct alloc_kv_cb_args arg = {db, kv_size, &empty_kv};
/* sector status statistics 查找使用的扇区 和未使用的扇区个数 <遍历整个分区 >*/
sector_iterator(db, sector, FDB_SECTOR_STORE_UNUSED, &empty_sector, &using_sector, sector_statistics_cb, false);
if (using_sector > 0) {/* 先检测使用的扇区是否 有空间写入当前KV*/
/* alloc the KV from the using status sector first */
sector_iterator(db, sector, FDB_SECTOR_STORE_USING, &arg, NULL, alloc_kv_cb, true);//alloc_kv_cb 更新新的空闲地址
}
//如已使用的扇区空间不足写下当前KV，但还有其他扇区未写入过数据(包含GC区域)
if (empty_sector > 0 && empty_kv == FAILED_ADDR) {
/* 判断是否使用GC区域 */
if (empty_sector > FDB_GC_EMPTY_SEC_THRESHOLD || db->gc_request) {
sector_iterator(db, sector, FDB_SECTOR_STORE_EMPTY, &arg, NULL, alloc_kv_cb, true);
} else {
/* no space for new KV now will GC and retry */
FDB_DEBUG("Trigger a GC check after alloc KV failed.\n");
db->gc_request = true;
}
}
return empty_kv;
}
由于前面假设直接进入GC进程 ， 所以条件满足empty_sector<=FDB_GC_EMPTY_SEC_THRESHOLD ;db->gc_request=true。当 if (using_sector > 0) {
sector_iterator(db, sector, FDB_SECTOR_STORE_USING, &arg, NULL, alloc_kv_cb, true);//alloc_kv_cb 更新新的空闲地址
}发现空间不足时，执行sector_iterator(db, sector, FDB_SECTOR_STORE_EMPTY, &arg, NULL, alloc_kv_cb, true)，直接使用预留的GC区域(已格式化后的扇区)。
alloc_kv_cb()
更新扇区空闲地址
static bool alloc_kv_cb(kv_sec_info_t sector, void *arg1, void *arg2)
{
struct alloc_kv_cb_args *arg = arg1;
/* 1. sector has space
* 2. the NO dirty sector
* 3. the dirty sector only when the gc_request is false */
if (sector->check_ok && sector->remain > arg->kv_size + FDB_SEC_REMAIN_THRESHOLD
&& ((sector->status.dirty == FDB_SECTOR_DIRTY_FALSE)
|| (sector->status.dirty == FDB_SECTOR_DIRTY_TRUE && !arg->db->gc_request))) {
*(arg->empty_kv) = sector->empty_kv;
return true;
}
return false;
}
3.kv_iterator(db, &kv, db, NULL, check_and_recovery_kv_cb);
KV迭代器就是检测使用过的扇区，把扇区有效KV 更新到kv cache.
static void kv_iterator(fdb_kvdb_t db, fdb_kv_t kv, void *arg1, void *arg2,
bool (*callback)(fdb_kv_t kv, void *arg1, void *arg2))
{
struct kvdb_sec_info sector;
uint32_t sec_addr, traversed_len = 0;
sec_addr = db_oldest_addr(db);
/* search all sectors */
do {
traversed_len += db_sec_size(db);
if (read_sector_info(db, sec_addr, §or, false) != FDB_NO_ERR) {
continue;
}
if (callback == NULL) {
continue;
}
/* sector has KV 未使用的扇区不做遍历 */
if (sector.status.store == FDB_SECTOR_STORE_USING || sector.status.store == FDB_SECTOR_STORE_FULL) {
kv->addr.start = sector.addr + SECTOR_HDR_DATA_SIZE;
/* search all KV */
do {
read_kv(db, kv);/*读取KV信息*/
/* iterator is interrupted when callback return true */
if (callback(kv, arg1, arg2)) {
return;
}
} while ((kv->addr.start = get_next_kv_addr(db, §or, kv)) != FAILED_ADDR);/*下一个KV起始地址*/
}
} while ((sec_addr = get_next_sector_addr(db, §or, traversed_len)) != FAILED_ADDR);/*下一个扇区起始偏移地址*/
}
check_and_recovery_kv_cb()
处理异常KV，同时更新有效KV到KV cache。
static bool check_and_recovery_kv_cb(fdb_kv_t kv, void *arg1, void *arg2)
{
fdb_kvdb_t db = arg1;
/* FDB_KV_PRE_DELETE说明 当前KV未被正确删除 重新删除，并迁移到新位置*/
if (kv->crc_is_ok && kv->status == FDB_KV_PRE_DELETE) {
FDB_INFO("Found an KV (%.*s) which has changed value failed. Now will recovery it.\n", kv->name_len, kv->name);
/* 先查找新的空间，迁移到新的区域，并删除FDB_KV_PRE_DELETE状态的KV*/
if (move_kv(db, kv) == FDB_NO_ERR) {
FDB_DEBUG("Recovery the KV successful.\n");
} else {
FDB_DEBUG("Warning: Moved an KV (size %" PRIu32 ") failed when recovery. Now will GC then retry.\n", kv->len);
return true;
}
} else if (kv->status == FDB_KV_PRE_WRITE) {/*上次KV写入异常 */
uint8_t status_table[KV_STATUS_TABLE_SIZE];
/* the KV has not write finish, change the status to error */
//TODO Draw the state replacement diagram of exception handling
_fdb_write_status((fdb_db_t)db, kv->addr.start, status_table, FDB_KV_STATUS_NUM, FDB_KV_ERR_HDR, true);
return true;
} else if (kv->crc_is_ok && kv->status == FDB_KV_WRITE) {/*kv 有效 更新到KV cache*/
#ifdef FDB_KV_USING_CACHE
/* update the cache when first load. If caching is disabled, this step is not performed */
update_kv_cache(db, kv->name, kv->name_len, kv->addr.start);
#endif
}
return false;
}
由于前面3.kv_iterator()可能发现存在KV未完全删除的情况，需要执行move_kv()。而move_kv()需要进行数据迁移，就会执行alloc_kv(）,判断是否使用GC预留区域，如果使用，db->gc_request=true，开启GC过程。
if (db->gc_request) {
gc_collect(db);
goto __retry;
}