1. 首页
  2. 文章中心
  3. # 后端应用设计

异步日志系统设计demo简单版本1优化版本1优化版本2

180 阅读 0 评论
我是靠谱客的博主 腼腆宝贝，这篇文章主要介绍异步日志系统设计demo简单版本1优化版本1优化版本2，现在分享给大家，希望可以做个参考。

目录

  • 简单版本1
  • 优化版本1
  • 优化版本2

对于QPS要求很高或者对性能有一定要求的服务器程序，同步写日志会对服务的关键性逻辑的快速执行和及时响应带来一定的性能损失，因为写日志时等待磁盘IO完成工作也需要一定时间。为了减少这种损失，一般采用异步写日志。
本质上仍然是一个生产者与消费者模型，产生日志的线程是生产者，将日志写入文件的线程是消费者。
如果有多个消费者线程，可能存在写日志的时间顺序错位，所以一般将日志消费者线程数量设置为1.

简单版本1

下面给出一个简单的版本：

复制代码
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
#include <iostream>
#include <thread>
#include <mutex>
#include <list>
#include <string>
#include <sstream>
#include <vector>
// 保护队列的mutex
std::mutex log_mutex;
std::list<std::string> cached_logs;
FILE* log_file = nullptr;

bool init_log_file()
{
    // 以追加的模式写入文件，如果文件不存在，则创建
    log_file = fopen("my.log","a+");
    return log_file != nullptr;
}

void uninit_log_file()
{
    if (log_file != nullptr)
        fclose(log_file);
}

bool write_log_tofile(const std::string& line)
{
    if (log_file == nullptr)
        return false;
    if (fwrite((void *)line.c_str(), 1, line.length(), log_file) != line.length())
        return false;
    // 将日志flush到文件中
    fflush(log_file);
    return true;
}

void log_producer()
{
    int index = 0;
    while (true) {
        ++index;
        std::ostringstream os;
        os << "This is log, index :" << index << ", producer threadID:" << std::this_thread::get_id() << "n";

        {
            std::lock_guard<std::mutex> lock(log_mutex);
            cached_logs.emplace_back(os.str());
        }
        // 生产出一个log之后，休眠100ms再生产
        std::chrono::milliseconds duration(100);
        std::this_thread::sleep_for(duration);
    }
}

void log_consumer()
{
    std::string line;
    while (true) {
        {
            std::lock_guard<std::mutex> lock(log_mutex);
            if (!cached_logs.empty()) {
                line = cached_logs.front();
                cached_logs.pop_front();
            }
        }
        // 如果取出来的行为空，说明队列里面是空的，消费者休眠一会儿再去消费
        if (line.empty()) {
            std::chrono::milliseconds duration(1000);
            std::this_thread::sleep_for(duration);

            continue;
        }
        // 否则将line写入到log_file中
        write_log_tofile(line);
        line.clear();
    }
}

int main(int argc, char* argv[])
{
    if (!init_log_file()) {
        std::cout << "init log file error." << std::endl;
        return -1;
    }
    std::thread log_producer1(log_producer);
    std::thread log_producer2(log_producer);
    std::thread log_producer3(log_producer);

    std::thread log_consumer1(log_consumer);
    std::thread log_consumer2(log_consumer);
    std::thread log_consumer3(log_consumer);

    log_producer1.join();
    log_producer2.join();
    log_producer3.join();

    log_consumer1.join();
    log_consumer2.join();
    log_consumer3.join();

    uninit_log_file();
    return 0;
}

效果：

复制代码
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
This is log, index :1, producer threadID:139910877185792
This is log, index :1, producer threadID:139910868793088
This is log, index :1, producer threadID:139910860400384
This is log, index :2, producer threadID:139910877185792
This is log, index :2, producer threadID:139910860400384
This is log, index :3, producer threadID:139910877185792
This is log, index :3, producer threadID:139910860400384
This is log, index :2, producer threadID:139910868793088
This is log, index :3, producer threadID:139910868793088
This is log, index :4, producer threadID:139910877185792
This is log, index :4, producer threadID:139910860400384
This is log, index :4, producer threadID:139910868793088
This is log, index :5, producer threadID:139910877185792
This is log, index :5, producer threadID:139910860400384
This is log, index :5, producer threadID:139910868793088
This is log, index :6, producer threadID:139910860400384
This is log, index :6, producer threadID:139910868793088
This is log, index :7, producer threadID:139910877185792
This is log, index :7, producer threadID:139910860400384
This is log, index :7, producer threadID:139910868793088
This is log, index :8, producer threadID:139910877185792
This is log, index :8, producer threadID:139910860400384
This is log, index :8, producer threadID:139910868793088
This is log, index :9, producer threadID:139910877185792
This is log, index :9, producer threadID:139910860400384
This is log, index :9, producer threadID:139910868793088
This is log, index :6, producer threadID:139910877185792
This is log, index :10, producer threadID:139910860400384
This is log, index :10, producer threadID:139910868793088
This is log, index :10, producer threadID:139910877185792
This is log, index :11, producer threadID:139910860400384
This is log, index :11, producer threadID:139910868793088
This is log, index :12, producer threadID:139910860400384
This is log, index :12, producer threadID:139910868793088
This is log, index :11, producer threadID:139910877185792
This is log, index :12, producer threadID:139910877185792
This is log, index :13, producer threadID:139910860400384
This is log, index :13, producer threadID:139910868793088
This is log, index :13, producer threadID:139910877185792
This is log, index :14, producer threadID:139910868793088
This is log, index :14, producer threadID:139910877185792
This is log, index :15, producer threadID:139910868793088
This is log, index :15, producer threadID:139910877185792
This is log, index :14, producer threadID:139910860400384
This is log, index :15, producer threadID:139910860400384
This is log, index :16, producer threadID:139910877185792
This is log, index :16, producer threadID:139910860400384
This is log, index :17, producer threadID:139910877185792
This is log, index :17, producer threadID:139910860400384
This is log, index :17, producer threadID:139910868793088
This is log, index :16, producer threadID:139910868793088
This is log, index :18, producer threadID:139910877185792
This is log, index :19, producer threadID:139910860400384
This is log, index :19, producer threadID:139910877185792
This is log, index :19, producer threadID:139910868793088
This is log, index :20, producer threadID:139910860400384
This is log, index :18, producer threadID:139910860400384
This is log, index :20, producer threadID:139910877185792
This is log, index :18, producer threadID:139910868793088
This is log, index :20, producer threadID:139910868793088
This is log, index :21, producer threadID:139910868793088
This is log, index :21, producer threadID:139910877185792
This is log, index :21, producer threadID:139910860400384
This is log, index :22, producer threadID:139910860400384
This is log, index :22, producer threadID:139910877185792
This is log, index :22, producer threadID:139910868793088
This is log, index :23, producer threadID:139910860400384
This is log, index :23, producer threadID:139910877185792
This is log, index :23, producer threadID:139910868793088
This is log, index :24, producer threadID:139910860400384
This is log, index :24, producer threadID:139910868793088
This is log, index :24, producer threadID:139910877185792
This is log, index :25, producer threadID:139910860400384
This is log, index :25, producer threadID:139910868793088
This is log, index :25, producer threadID:139910877185792
This is log, index :26, producer threadID:139910868793088
This is log, index :26, producer threadID:139910860400384
This is log, index :26, producer threadID:139910877185792
This is log, index :27, producer threadID:139910868793088
This is log, index :27, producer threadID:139910860400384
This is log, index :27, producer threadID:139910877185792
This is log, index :28, producer threadID:139910868793088
This is log, index :28, producer threadID:139910877185792
This is log, index :28, producer threadID:139910860400384
This is log, index :29, producer threadID:139910877185792
This is log, index :29, producer threadID:139910868793088
This is log, index :29, producer threadID:139910860400384
This is log, index :30, producer threadID:139910877185792
This is log, index :30, producer threadID:139910868793088
This is log, index :30, producer threadID:139910860400384
This is log, index :31, producer threadID:139910868793088
This is log, index :31, producer threadID:139910877185792
This is log, index :31, producer threadID:139910860400384
This is log, index :32, producer threadID:139910877185792
This is log, index :32, producer threadID:139910868793088
This is log, index :32, producer threadID:139910860400384
This is log, index :33, producer threadID:139910860400384
This is log, index :33, producer threadID:139910868793088
This is log, index :33, producer threadID:139910877185792
This is log, index :34, producer threadID:139910860400384
This is log, index :34, producer threadID:139910877185792
This is log, index :34, producer threadID:139910868793088
This is log, index :35, producer threadID:139910860400384
This is log, index :35, producer threadID:139910868793088
This is log, index :35, producer threadID:139910877185792
This is log, index :36, producer threadID:139910877185792
This is log, index :36, producer threadID:139910868793088
This is log, index :37, producer threadID:139910860400384
This is log, index :37, producer threadID:139910877185792
This is log, index :37, producer threadID:139910868793088
This is log, index :38, producer threadID:139910860400384
This is log, index :38, producer threadID:139910877185792
This is log, index :38, producer threadID:139910868793088
This is log, index :39, producer threadID:139910860400384
This is log, index :39, producer threadID:139910877185792
This is log, index :39, producer threadID:139910868793088
This is log, index :40, producer threadID:139910860400384
This is log, index :40, producer threadID:139910877185792
This is log, index :40, producer threadID:139910868793088
This is log, index :36, producer threadID:139910860400384
This is log, index :41, producer threadID:139910860400384
This is log, index :41, producer threadID:139910877185792
This is log, index :42, producer threadID:139910860400384
This is log, index :42, producer threadID:139910877185792
This is log, index :42, producer threadID:139910868793088
This is log, index :43, producer threadID:139910860400384
This is log, index :43, producer threadID:139910868793088
This is log, index :43, producer threadID:139910877185792
This is log, index :44, producer threadID:139910860400384
This is log, index :44, producer threadID:139910868793088
This is log, index :44, producer threadID:139910877185792
This is log, index :45, producer threadID:139910860400384
This is log, index :45, producer threadID:139910868793088
This is log, index :45, producer threadID:139910877185792
This is log, index :46, producer threadID:139910860400384
This is log, index :46, producer threadID:139910868793088
This is log, index :46, producer threadID:139910877185792
This is log, index :47, producer threadID:139910860400384
This is log, index :47, producer threadID:139910868793088
This is log, index :47, producer threadID:139910877185792
This is log, index :48, producer threadID:139910860400384
This is log, index :48, producer threadID:139910868793088
This is log, index :48, producer threadID:139910877185792
This is log, index :49, producer threadID:139910860400384
This is log, index :49, producer threadID:139910877185792
This is log, index :49, producer threadID:139910868793088
This is log, index :50, producer threadID:139910877185792
This is log, index :50, producer threadID:139910860400384
This is log, index :50, producer threadID:139910868793088
This is log, index :41, producer threadID:139910868793088

优化版本1

上面的代码，在当前缓存队列没有日志记录的时候，消费日志线程会做无用功。
这里可以使用条件变量，如果当前队列中没有日志记录，就将日志消费者线程挂起；
当产生了新的日志后，signal条件变量，唤醒消费线程，将被日志从队列中取出，并写入文件。
下面是主要修改

复制代码
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
#include <condition_variable>
std::condition_variable log_cv;
void log_producer()
{
    int index = 0;
    while (true) {
        ++index;
        std::ostringstream os;
        os << "This is log, index :" << index << ", producer threadID:" << std::this_thread::get_id() << "n";

        {
            std::lock_guard<std::mutex> lock(log_mutex);
            cached_logs.emplace_back(os.str());
            log_cv.notify_one();
        }
        // 生产出一个log之后，休眠100ms再生产
        std::chrono::milliseconds duration(100);
        std::this_thread::sleep_for(duration);
    }
}

void log_consumer()
{
    std::string line;
    while (true) {
        {
            std::unique_lock<std::mutex> lock(log_mutex);
            while (cached_logs.empty()) {
                // 无限等待
                log_cv.wait(lock);
            }
            line = cached_logs.front();
            cached_logs.pop_front();
        }
        // 如果取出来的行为空，说明队列里面是空的，消费者休眠一会儿再去消费
        if (line.empty()) {
            std::chrono::milliseconds duration(1000);
            std::this_thread::sleep_for(duration);

            continue;
        }
        // 否则将line写入到log_file中
        write_log_tofile(line);
        line.clear();
    }
}

优化版本2

还可以使用信号量来设计异步日志系统。
信号量是带有资源计数的线程同步对象，每产生一条日志，就将信号量资源计数+1，日志消费线程默认等待这个信号量是否signal，如果signal，就唤醒一个日志消费线程，信号量计数自动-1。如果当前资源计数为0，则将消费者自动挂起。
C++现在还没有提供不同平台的信号量对象封装，这里以Linux系统为例：
明显能感觉运行相同时间，写入的日志更多了。。

复制代码
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
#include <unistd.h>
#include <iostream>
#include <thread>
#include <mutex>
#include <list>
#include <string>
#include <sstream>
#include <semaphore.h>

// 保护队列的mutex
pthread_mutex_t log_mutex = PTHREAD_MUTEX_INITIALIZER;
sem_t log_semphore;
std::list<std::string> cached_logs;
FILE* log_file = nullptr;

bool init()
{
    pthread_mutex_init(&log_mutex, nullptr);
    // 初始信号量资源数量为0
    sem_init(&log_semphore, 0, 0);
    // 以追加的模式写入文件，如果文件不存在，则创建
    log_file = fopen("my.log","a++");
    return log_file != nullptr;
}

void uninit()
{
    pthread_mutex_destroy(&log_mutex);
    sem_destroy(&log_semphore);
    if (log_file != nullptr)
        fclose(log_file);
}

bool write_log_tofile(const std::string& line)
{
    if (log_file == nullptr)
        return false;
    // 对于比较长的日志应该分段写入，因为单次写入可能只写入部分内容
    // 这里逻辑从简
    if (fwrite((void *)line.c_str(), 1, line.length(), log_file) != line.length())
        return false;
    // 将日志flush到文件中
    fflush(log_file);
    return true;
}

void* log_producer(void* arg)
{
    int index = 0;
    while (true) {
        ++index;
        std::ostringstream os;
        os << "This is log, index :" << index << ", producer threadID:" << std::this_thread::get_id() << "n";
        pthread_mutex_lock(&log_mutex);
        cached_logs.emplace_back(os.str());
        pthread_mutex_unlock(&log_mutex);
        sem_post(&log_semphore);
        usleep(1000);
    }
}

void* log_consumer(void* arg)
{
    std::string line;
    while (true) {
        // 无限等待
        sem_wait(&log_semphore);
        pthread_mutex_lock(&log_mutex);
        if (!cached_logs.empty()) {
            line = cached_logs.front();
            cached_logs.pop_front();
        }
        pthread_mutex_unlock(&log_mutex);
        // 如果取出来的行为空，说明队列里面是空的，消费者休眠一会儿再去消费
        if (line.empty()) {
            sleep(1);
            continue;
        }
        // 否则将line写入到log_file中
        write_log_tofile(line);
        line.clear();
    }
}

int main(int argc, char* argv[])
{
    if (!init()) {
        std::cout << "init log file error." << std::endl;
        return -1;
    }
    // 创建三个生产日志线程
    pthread_t producer_thread_id[3];
    for (size_t i = 0; i < sizeof(producer_thread_id) / sizeof(producer_thread_id[0]); ++i) {
        pthread_create(&producer_thread_id[i], NULL, log_producer, NULL);
    }
    // 创建三个消费日志线程
    pthread_t consumer_thread_id[3];
    for (size_t i = 0; i < sizeof(consumer_thread_id) / sizeof(consumer_thread_id[0]); ++i) {
        pthread_create(&consumer_thread_id[i], NULL, log_consumer, NULL);
    }

    // 等待生产者线程退出
    for (size_t i = 0; i < sizeof(producer_thread_id) / sizeof(producer_thread_id[0]); ++i) {
        pthread_join(producer_thread_id[i], NULL);
    }
    // 等待消费者线程退出
    for (size_t i = 0; i < sizeof(consumer_thread_id) / sizeof(consumer_thread_id[0]); ++i) {
        pthread_join(consumer_thread_id[i], NULL);
    }
    uninit();
    return 0;
}

最后

以上就是腼腆宝贝最近收集整理的关于异步日志系统设计demo简单版本1优化版本1优化版本2的全部内容，更多相关异步日志系统设计demo简单版本1优化版本1优化版本2内容请搜索靠谱客的其他文章。

本图文内容来源于网友提供，作为学习参考使用，或来自网络收集整理，版权属于原作者所有。
点赞(119)

相关文章

cms启动流程
cms启动流程
新主板装系统bios设置
新主板装系统bios设置
del服务器能装win7系统吗,500系列主板能不能装win7?500系列主板装win7教程(支持11代)...
del服务器能装win7系统吗,500系列主板能不能装win7?500系列主板装win7教程(支持11代)...
RAID配置全程
RAID配置全程
异步日志系统设计demo简单版本1优化版本1优化版本2
异步日志系统设计demo简单版本1优化版本1优化版本2
filebeat收集json格式的nginx程序日志（二）filebeat收集json格式的nginx日志
filebeat收集json格式的nginx程序日志（二）filebeat收集json格式的nginx日志
解决Win10 Windows10关键错误 开始菜单和Cortana无法工作问题
解决Win10 Windows10关键错误 开始菜单和Cortana无法工作问题
win10开始菜单和cortana无法工作怎么办 win10关键错误解决方法
win10开始菜单和cortana无法工作怎么办 win10关键错误解决方法

评论列表共有 0 条评论

立即
投稿
返回
顶部