构建一个简单的数据库系列（五）持久化到硬盘
英文链接：https://cstack.github.io/db_tutorial/parts/part5.html
“Nothing in the world can take the place of persistence.” – Calvin Coolidge
像sqlite一样，我们通过保存整个数据库到一个文件到方式进行记录的持久化。
我们已经实现了把记录反序列化到页大小到内存块中。持久化的实现，我们可以简单地把内存块写入到文件，当下次启动时，我们再把数据读回到内存当中。
为了简化，我们可以抽象一层：pager，通过pager请求得到我们想要的数据。
它的基本逻辑：先在缓冲查找，如果找不到，就会从磁盘把数据读回到内存当中。
Pager可以访问page缓冲和文件，Table通过向Pager请求页信息。
+struct Pager_t {
+ int file_descriptor;
+ uint32_t file_length;
+ void* pages[TABLE_MAX_PAGES];
+};
+typedef struct Pager_t Pager;
+
struct Table_t {
- void* pages[TABLE_MAX_PAGES];
+ Pager* pager;
uint32_t num_rows;
};将函数new_table重命名为db_open，因为它有新建一个数据库连接到含义。新建一个连接有几层含义：
1、打开一个数据库文件
2、初始化pager结构体
3、初始化data结构体
-Table* new_table() {
+Table* db_open(const char* filename) {
+ Pager* pager = pager_open(filename);
+ uint32_t num_rows = pager->file_length / ROW_SIZE;
+
Table* table = malloc(sizeof(Table));
- table->num_rows = 0;
+ table->pager = pager;
+ table->num_rows = num_rows;
return table;
}
新增函数说明：
pager_open 打开数据库文件，并且一直track数据库文件大小。初始化时，会把page cache全部置为NULL
get_page 处理缓冲未命中。我们假设pager按顺序存放在文件中，Page 0 从offset 0 开始，page 1从offset 4096开始，page 3从offset 2开始，以此类推。如果请求到页在文件边界外，那么我们需要重新分配内存。page会被追加到数据库文件中，当我们把cache刷到文件后。
db_close 将cache刷新到文件，关闭数据库文件，释放Pager，Table使用的内存。
pager_flush 同时维护了pager number和size，这是因为最后一页可能是部分页。这并不是最好的设计，当我们使用B-tree时，这个问题可以得到相对完美的解决。
执行过程：
sh-3.2# ./mydb mydb.db
db > insert 1 cstack [email protected]
Executed.
db > insert 2 hello [email protected]
Executed.
db > select
(1, cstack, [email protected])
(2, hello, [email protected])
Executed.
# 退出系统再登陆，验证下是否持久化并且可以正常读入数据。
db > .exit
sh-3.2# ./mydb mydb.db
db > select
(1, cstack, [email protected])
(2, hello, [email protected])
Executed.
截止目前完整的代码：
#include <errno.h>
#include <fcntl.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
/* 定义 元数据操作结果*/
enum MetaCommandResult_t {
META_COMMAND_SUCCESS,
META_COMMAND_UNRECOGNIZED_COMMAND
};
typedef enum MetaCommandResult_t MetaCommandResult;
/* 执行结果*/
enum ExecuteResult_t { EXECUTE_SUCCESS, EXECUTE_TABLE_FULL };
typedef enum ExecuteResult_t ExecuteResult;
/* sql解析结果 */
enum PrepareResult_t {
PREPARE_SUCCESS,
PREPARE_NEGATIVE_ID,
PREPARE_STRING_TOO_LONG,
PREPARE_SYNTAX_ERROR,
PREPARE_UNRECOGNIZED_STATEMENT
};
typedef enum PrepareResult_t PrepareResult;
/* 行定义，对应具体的业务 */
const uint32_t COLUMN_USERNAME_SIZE = 32;
const uint32_t COLUMN_EMAIL_SIZE = 255;
struct Row_t {
uint32_t id;
char username[COLUMN_USERNAME_SIZE + 1];
char email[COLUMN_EMAIL_SIZE + 1];
};
typedef struct Row_t Row;
/* sql type*/
enum StatementType_t{
STATEMENT_INSERT,
STATEMENT_SELECT
};
typedef enum StatementType_t StatementType;
struct Statement_t {
StatementType type;
Row row_to_insert; /* only used by insert statement */
};
typedef struct Statement_t Statement;
/**/
#define size_of_attribute(Struct, Attribute) sizeof(((Struct*)0)->Attribute)
const uint32_t ID_SIZE = size_of_attribute(Row, id);
const uint32_t USERNAME_SIZE = size_of_attribute(Row, username);
const uint32_t EMAIL_SIZE = size_of_attribute(Row, email);
const uint32_t ID_OFFSET = 0;
const uint32_t USERNAME_OFFSET = ID_OFFSET + ID_SIZE;
const uint32_t EMAIL_OFFSET = USERNAME_OFFSET + USERNAME_SIZE;
const uint32_t ROW_SIZE = ID_SIZE + USERNAME_SIZE + EMAIL_SIZE;
/* 大部分系统结构的页大小都是4K ，所以这里也定义为4K 这样就不用做转换 */
const uint32_t PAGE_SIZE = 4096;
const uint32_t TABLE_MAX_PAGES = 100;
const uint32_t ROWS_PER_PAGE = PAGE_SIZE / ROW_SIZE;
const uint32_t TABLE_MAX_ROWS = ROWS_PER_PAGE * TABLE_MAX_PAGES;
/*
* 页和表定义
*/
struct Pager_t {
int file_descriptor;
uint32_t file_length;
void *pages[TABLE_MAX_PAGES];
};
typedef struct Pager_t Pager;
struct Table_t {
Pager *pager;
uint32_t num_rows;
};
typedef struct Table_t Table;
void print_row(Row* row) {
printf("(%d, %s, %s)\n", row->id, row->username, row->email);
}
/* 序列化： 将row写到内存中 */
void serialize_row(Row* source, void* destination) {
memcpy(destination + ID_OFFSET, &(source->id), ID_SIZE);
memcpy(destination + USERNAME_OFFSET, &(source->username), USERNAME_SIZE);
memcpy(destination + EMAIL_OFFSET, &(source->email), EMAIL_SIZE);
}
/* 反序列化 */
void deserialize_row(void* source, Row* destination) {
memcpy(&(destination->id), source + ID_OFFSET, ID_SIZE);
memcpy(&(destination->username), source + USERNAME_OFFSET, USERNAME_SIZE);
memcpy(&(destination->email), source + EMAIL_OFFSET, EMAIL_SIZE);
}
/**
* 获取页num对应的页
* @param table
* @param row_num
* @return
*/
void *get_page(Pager *pager, uint32_t page_num)
{
if (page_num > TABLE_MAX_PAGES) {
printf("Tried to fetch page number out of bounds. %d > %d\n", page_num,
TABLE_MAX_PAGES);
exit(EXIT_FAILURE);
}
if (pager->pages[page_num] == NULL) {
// Cache miss. Allocate memory and load from file.
void *page = malloc(PAGE_SIZE);
uint32_t num_pages = pager->file_length / PAGE_SIZE;
// We might save a partial page at the end of the file
if (pager->file_length % PAGE_SIZE) {
num_pages += 1;
}
if (page_num <= num_pages) {
lseek(pager->file_descriptor, page_num * PAGE_SIZE, SEEK_SET);
ssize_t bytes_read = read(pager->file_descriptor, page, PAGE_SIZE);
if (bytes_read == -1) {
printf("Error reading file: %d\n", errno);
exit(EXIT_FAILURE);
}
}
pager->pages[page_num] = page;
}
return pager->pages[page_num];
}
/**
* 计算插入位置， 行插入槽
*/
void* row_slot(Table* table, uint32_t row_num) {
uint32_t page_num = row_num / ROWS_PER_PAGE;
void* page = get_page(table->pager, page_num);
uint32_t row_offset = row_num % ROWS_PER_PAGE;
uint32_t byte_offset = row_offset * ROW_SIZE;
return page + byte_offset;
}
/**
* 打开pager
* @param filename
* @return
*/
Pager *pager_open(const char *filename)
{
int fd = open(filename,
O_RDWR | // Read/Write mode
O_CREAT, // Create file if it does not exist
S_IWUSR | // User write permission
S_IRUSR // User read permission
);
if (fd == -1) {
printf("Unable to open file\n");
exit(EXIT_FAILURE);
}
off_t file_length = lseek(fd, 0, SEEK_END);
Pager *pager = malloc(sizeof(Pager));
pager->file_descriptor = fd;
pager->file_length = file_length;
for (uint32_t i = 0; i < TABLE_MAX_PAGES; i++) {
pager->pages[i] = NULL;
}
return pager;
}
/**
* 打开数据库文件并建立连接
* @param filename
* @return
*/
Table *db_open(const char *filename)
{
Pager *pager = pager_open(filename);
uint32_t num_rows = pager->file_length / ROW_SIZE;
Table *table = malloc(sizeof(Table));
table->num_rows = 0;
table->pager = pager;
table->num_rows = num_rows;
return table;
}
/* 接收输入*/
struct InputBuffer_t {
char* buffer;
size_t buffer_length;
ssize_t input_length;
};
typedef struct InputBuffer_t InputBuffer;
/* 初始化buffer */
InputBuffer* new_input_buffer()
{
InputBuffer* input_buffer = malloc(sizeof(InputBuffer));
input_buffer->buffer = NULL;
input_buffer->buffer_length = 0;
input_buffer->input_length = 0;
return input_buffer;
}
void print_prompt() { printf("db > "); }
/* 按行从标准输入读取 */
void read_input(InputBuffer* input_buffer)
{
ssize_t bytes_read =
getline(&(input_buffer->buffer), &(input_buffer->buffer_length), stdin);
if (bytes_read <= 0) {
printf("Error reading input\n");
exit(EXIT_FAILURE);
}
// Ignore trailing newline
input_buffer->input_length = bytes_read - 1;
input_buffer->buffer[bytes_read - 1] = 0;
}
/**
* 刷新页到文件（指定页号）
* @param pager
* @param page_num
* @param size
*/
void pager_flush(Pager *pager, uint32_t page_num, uint32_t size)
{
if (pager->pages[page_num] == NULL) {
printf("Tried to flush null page\n");
exit(EXIT_FAILURE);
}
off_t offset = lseek(pager->file_descriptor, page_num * PAGE_SIZE, SEEK_SET);
if (offset == -1) {
printf("Error seeking: %d\n", errno);
exit(EXIT_FAILURE);
}
ssize_t bytes_written =
write(pager->file_descriptor, pager->pages[page_num], size);
if (bytes_written == -1) {
printf("Error writing: %d\n", errno);
exit(EXIT_FAILURE);
}
}
/**
* 关闭数据连接
*/
void db_close(Table *table) {
Pager *pager = table->pager;
uint32_t num_full_pages = table->num_rows / ROWS_PER_PAGE;
for (uint32_t i = 0; i < num_full_pages; i++) {
if (pager->pages[i] == NULL) {
continue;
}
pager_flush(pager, i, PAGE_SIZE);
free(pager->pages[i]);
pager->pages[i] = NULL;
}
// There may be a partial page to write to the end of the file
// This should not be needed after we switch to a B-tree
uint32_t num_additional_rows = table->num_rows % ROWS_PER_PAGE;
if (num_additional_rows > 0) {
uint32_t page_num = num_full_pages;
if (pager->pages[page_num] != NULL) {
pager_flush(pager, page_num, num_additional_rows * ROW_SIZE);
free(pager->pages[page_num]);
pager->pages[page_num] = NULL;
}
}
int result = close(pager->file_descriptor);
if (result == -1) {
printf("Error closing db file.\n");
exit(EXIT_FAILURE);
}
for (uint32_t i = 0; i < TABLE_MAX_PAGES; i++) {
void *page = pager->pages[i];
if (page) {
free(page);
pager->pages[i] = NULL;
}
}
free(pager);
}
/* 元数据命令处理 */
MetaCommandResult do_meta_command(InputBuffer* input_buffer, Table* table)
{
if (strcmp(input_buffer->buffer, ".exit") == 0) {
db_close(table);
exit(EXIT_SUCCESS);
} else {
return META_COMMAND_UNRECOGNIZED_COMMAND;
}
}
/* insert解析和校验*/
PrepareResult prepare_insert(InputBuffer *input_buffer, Statement *statement) {
statement->type = STATEMENT_INSERT;
char *keyword = strtok(input_buffer->buffer, " ");
char *id_string = strtok(NULL, " ");
char *username = strtok(NULL, " ");
char *email = strtok(NULL, " ");
if (id_string == NULL || username == NULL || email == NULL) {
return PREPARE_SYNTAX_ERROR;
}
int id = atoi(id_string);
if (id < 0) {
return PREPARE_NEGATIVE_ID;
}
if (strlen(username) > COLUMN_USERNAME_SIZE) {
return PREPARE_STRING_TOO_LONG;
}
if (strlen(email) > COLUMN_EMAIL_SIZE) {
return PREPARE_STRING_TOO_LONG;
}
statement->row_to_insert.id = id;
strcpy(statement->row_to_insert.username, username);
strcpy(statement->row_to_insert.email, email);
return PREPARE_SUCCESS;
}
/* sql解析 */
PrepareResult prepare_statement(InputBuffer* input_buffer,Statement* statement)
{
if (strncasecmp(input_buffer->buffer, "insert", 6) == 0) {
return prepare_insert(input_buffer, statement);
}
if (strncasecmp(input_buffer->buffer, "select", 6) == 0) {
statement->type = STATEMENT_SELECT;
return PREPARE_SUCCESS;
}
return PREPARE_UNRECOGNIZED_STATEMENT;
}
/* 执行insert*/
ExecuteResult execute_insert(Statement *statement, Table *table)
{
if (table->num_rows >= TABLE_MAX_ROWS) {
return EXECUTE_TABLE_FULL;
}
Row *row_to_insert = &(statement->row_to_insert);
serialize_row(row_to_insert, row_slot(table, table->num_rows));
table->num_rows += 1;
return EXECUTE_SUCCESS;
}
/* 执行查询*/
ExecuteResult execute_select(Statement *statement, Table *table)
{
Row row;
for (uint32_t i = 0; i < table->num_rows; i++) {
deserialize_row(row_slot(table, i), &row);
print_row(&row);
}
return EXECUTE_SUCCESS;
}
/* sql执行*/
ExecuteResult execute_statement(Statement* statement , Table* table)
{
switch (statement->type)
{
case (STATEMENT_INSERT):
return execute_insert(statement, table);
case (STATEMENT_SELECT):
return execute_select(statement, table);
}
}
/* 主函数*/
int main(int argc, char* argv[])
{
if (argc < 2) {
printf("Must supply a database filename.\n");
exit(EXIT_FAILURE);
}
char *filename = argv[1];
Table *table = db_open(filename);
InputBuffer *input_buffer = new_input_buffer();
while (true)
{
print_prompt();
read_input(input_buffer);
if (input_buffer->buffer[0] == '.')
{
switch (do_meta_command(input_buffer,table))
{
case (META_COMMAND_SUCCESS):
continue;
case (META_COMMAND_UNRECOGNIZED_COMMAND):
printf("Unrecognized command '%s'\n", input_buffer->buffer);
continue;
}
}
Statement statement;
switch (prepare_statement(input_buffer, &statement))
{
case (PREPARE_SUCCESS):
break;
case (PREPARE_NEGATIVE_ID):
printf("ID must be positive.\n");
continue;
case (PREPARE_STRING_TOO_LONG):
printf("String is too long.\n");
continue;
case (PREPARE_SYNTAX_ERROR):
printf("Syntax error. Could not parse statement.\n");
continue;
case (PREPARE_UNRECOGNIZED_STATEMENT):
printf("Unrecognized keyword at start of '%s'.\n",
input_buffer->buffer);
continue;
}
switch (execute_statement(&statement, table))
{
case (EXECUTE_SUCCESS):
printf("Executed.\n");
break;
case (EXECUTE_TABLE_FULL):
printf("Error: Table full.\n");
break;
}
}
}