VPP vpp_main线程时间轮用法

vpp_main线程通过在文件vppsrcvlibmain.c中包含#include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>指定了使用1t_3w_1024sl_ov类型时间轮构建定时器。

数据结构

• [ ] vlib_node_main_t

/* 线程与定时器相关成员 */
typedef struct
{
    ......
    
    /* Timing wheel for scheduling time-based node dispatch. */
    /* 基于时间的调度策略，依赖时间轮构建，使用的是1t_3w_1024sl_ov类型时间轮 */
    void *timing_wheel;
    /* vpp_main线程注册的定时器的私有数据保存向量，索引来自data_from_advancing_timing_wheel */
    vlib_signal_timed_event_data_t *signal_timed_event_data_pool;

    /* Opaque data vector added via timing_wheel_advance. */
    /* 已经发生的定时事件的用户句柄保存内存 */
    u32 *data_from_advancing_timing_wheel;

    /* CPU time of next process to be ready on timing wheel. */
    f64 time_next_process_ready;/* 该值目前没有使用，根据意思是最近一个可能超时的时间 */
    ......

} vlib_node_main_t;

• [ ] vlib_signal_timed_event_data_t

/* 定时器私有数据 */
typedef struct
{
    u16 n_data_elts;/* 数据元素个数 */

    u16 n_data_elt_bytes;/* 每个元素字节数 */

    /* n_data_elts * n_data_elt_bytes 即下面union存放的数据大小*/
    u32 n_data_bytes;

    /* Process node & event type to be used to signal event. */
    /* 该事件所属协程 */
    u32 process_node_index;

    u32 event_type_index;/* 事件类型索引，用于通知协程的事件通知机制什么事情发生了 */
    /* 私有数据存放位置，数据较少时，放在静态数组inline_event_data中，
    ** 否则放在动态内存event_data_as_vector中 */
    union
    {
        u8 inline_event_data[64 - 3 * sizeof (u32) - 2 * sizeof (u16)];

        /* Vector of event data used only when data does not fit inline. */
        u8 *event_data_as_vector;
    };
}vlib_signal_timed_event_data_t;

代码流程

初始化时间轮

/* Main function. */
int
vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
{
    ......
    /* 分配时间轮描述控制块 */
    nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
                       CLIB_CACHE_LINE_BYTES);
    /* 确保可以存放10个事件 */
    vec_validate (nm->data_from_advancing_timing_wheel, 10);
    _vec_len (nm->data_from_advancing_timing_wheel) = 0;

    /* Create the process timing wheel */
    /* 创建时间轮 */
    TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
                              0 /* no callback */ ,
                              10e-6 /* timer period 10us */ ,
                              ~0 /* max expirations per call */ );

    ......

    return 0;
}

定时器用户ID构成原则

定时器的用户ID由两部分构成，私有数据索引和类型标志位，最低位为类型标志位：

• 1表示时间类定时器，
• 0表示协程挂起类定时器(这类定时器给协程使用，协程使用该类型定时器挂起一段时间)。

/* 判断是否为时间类定时器，直接获取最低位值 */
always_inline uword
vlib_timing_wheel_data_is_timed_event (u32 d)
{
    return d & 1;
}
/* 构建协程挂起事件定时器用户id */
always_inline u32
vlib_timing_wheel_data_set_suspended_process (u32 i)
{
    return 0 + 2 * i;
}
/* 构建时间类定时器用户id */
always_inline u32
vlib_timing_wheel_data_set_timed_event (u32 i)
{
    return 1 + 2 * i;
}
/* 获取私有数据索引 */
always_inline uword
vlib_timing_wheel_data_get_index (u32 d)
{
    return d / 2;
}

VPP_MAIN线程驱动时间轮

/* 参数is_main决定是主线程还是worker线程 */
static_always_inline void
vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
{
    ......
    
    while (1)
    {
        ......
        
        if (is_main) /* 只有主协程才会调度该时间轮 */
        {
            /* *INDENT-OFF* */
            ELOG_TYPE_DECLARE (es) =
            {
                .format = "process tw start",
                .format_args = "",
            };
            ELOG_TYPE_DECLARE (ee) =
            {
                .format = "process tw end: %d",
                .format_args = "i4",
            };
            /* *INDENT-ON* */

            struct
            {
                int nready_procs;
            }*ed;

            /* Check if process nodes have expired from timing wheel. */
            ASSERT (nm->data_from_advancing_timing_wheel != 0);

            if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
                ed = ELOG_DATA (&vlib_global_main.elog_main, es);
            /* 执行定时器超时操作，查看是否有定时器超时，处理超时事件，超时句柄保存在                                            data_from_advancing_timing_wheel中 */
            nm->data_from_advancing_timing_wheel =
                TW (tw_timer_expire_timers_vec)
                ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm),
                 nm->data_from_advancing_timing_wheel);

            ASSERT (nm->data_from_advancing_timing_wheel != 0);

            if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
            {
                ed = ELOG_DATA (&vlib_global_main.elog_main, ee);
                ed->nready_procs =
                    _vec_len (nm->data_from_advancing_timing_wheel);
            }
            /* 存在超时事件 */
            if (PREDICT_FALSE(_vec_len (nm->data_from_advancing_timing_wheel) > 0))
            {
                uword i;
                /* 遍历超时事件 */
                for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel); i++)
                {
                    u32 d = nm->data_from_advancing_timing_wheel[i];
                    /* 句柄转换为索引，因为最低为为类型位，所以直接除以二 */
                    u32 di = vlib_timing_wheel_data_get_index (d);

                    if (vlib_timing_wheel_data_is_timed_event (d))/* 最低位为1说明是一般定时事件 */
                    {
                        vlib_signal_timed_event_data_t *te =
                            pool_elt_at_index (nm->signal_timed_event_data_pool, di);
                        vlib_node_t *n = vlib_get_node (vm, te->process_node_index);
                        vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
                        void *data;
                        /* 通知事件处理,通知指定协程，定时事件发生了 */
                        data = vlib_process_signal_event_helper (nm, n, p,
                                te->event_type_index,
                                te->n_data_elts,
                                te->n_data_elt_bytes);
                        /* 透明数据拷贝 */
                        if (te->n_data_bytes < sizeof (te->inline_event_data))
                            clib_memcpy_fast (data, te->inline_event_data, te->n_data_bytes);
                        else
                        {
                            clib_memcpy_fast (data, te->event_data_as_vector,
                                              te->n_data_bytes);
                            vec_free (te->event_data_as_vector);
                        }
                        /* 将te从signal_timed_event_data_pool移除 */
                        pool_put (nm->signal_timed_event_data_pool, te);
                    }
                    else /* 协程类定时事件 */
                    {
                        /* 处理协程定时事件，即唤醒协程 */
                        cpu_time_now = clib_cpu_time_now ();
                        cpu_time_now = dispatch_suspended_process (vm, di, cpu_time_now);
                    }
                }
                _vec_len (nm->data_from_advancing_timing_wheel) = 0;
            }
        }
        vlib_increment_main_loop_counter (vm);

        /* Record time stamp in case there are no enabled nodes and above
            calls do not update time stamp. */
        cpu_time_now = clib_cpu_time_now ();
    }
}

协程类定时器注册

static u64
dispatch_process (vlib_main_t * vm,
                  vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp)
{
    ......
    
    if (is_suspend)
    {
        vlib_pending_frame_t *pf;

        n_vectors = 0;
        pool_get (nm->suspended_process_frames, pf);
        pf->node_runtime_index = node->runtime_index;
        pf->frame_index = f ? vlib_frame_index (vm, f) : ~0;
        pf->next_frame_index = ~0;

        p->n_suspends += 1;/* 统计挂起数量 */
        p->suspended_process_frame_index = pf - nm->suspended_process_frames;
        /* 设置了使用定时器挂起标志，则通过定时器挂起该线程 */
        if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
        {
            TWT (tw_timer_wheel) * tw =
                (TWT (tw_timer_wheel) *) nm->timing_wheel;
            p->stop_timer_handle =
                TW (tw_timer_start) (tw,
                                     vlib_timing_wheel_data_set_suspended_process
                                     (node->runtime_index) /* [sic] pool idex */ ,
                                     0 /* timer_id */ ,
                                     p->resume_clock_interval);
        }
    }
    else
        p->flags &= ~VLIB_PROCESS_IS_RUNNING;

    t = clib_cpu_time_now ();

    vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend,
                               /* is_after */ 1);

    vlib_process_update_stats (vm, p,
                               /* n_calls */ !is_suspend,
                               /* n_vectors */ n_vectors,
                               /* n_clocks */ t - last_time_stamp);

    return t;
}

普通定时器注册

以quic.c模块使用了vpp_main线程的定时器为例

static void
quic_update_timer (quic_ctx_t * ctx)
{
    tw_timer_wheel_1t_3w_1024sl_ov_t *tw;
    int64_t next_timeout;

    // This timeout is in ms which is the unit of our timer
    next_timeout = quicly_get_first_timeout (ctx->c_quic_ctx_id.conn);
    tw = &quic_main.wrk_ctx[vlib_get_thread_index ()].timer_wheel;
    f64 next_timeout_f = ((f64) next_timeout) / 1000.f;

    clib_warning ("Timer set to %ld (%lf)", next_timeout, next_timeout_f);

    if (ctx->c_quic_ctx_id.timer_handle == QUIC_TIMER_HANDLE_INVALID)
    {
        if (next_timeout == INT64_MAX)
            return;
        ctx->c_quic_ctx_id.timer_handle =
            tw_timer_start_1t_3w_1024sl_ov (tw, ctx->c_c_index, 0,
                                            next_timeout_f);
    }
    else
    {
        if (next_timeout == INT64_MAX)
        {
            tw_timer_stop_1t_3w_1024sl_ov (tw, ctx->c_quic_ctx_id.timer_handle);
            ctx->c_quic_ctx_id.timer_handle = QUIC_TIMER_HANDLE_INVALID;
        }
        else
            tw_timer_update_1t_3w_1024sl_ov (tw, ctx->c_quic_ctx_id.timer_handle,
                                             next_timeout_f);
    }
}