[Linux] Wireless Stack
개요
Linux Kernel을 공부하기 위해 첫번째로 무선 통신 모듈인 중 Wireless를 타겟으로 잡고 분석을 시작했습니다.
우선, 해당 내용을 보기 전에 아래의 링크에서 보고 오시면 좋을 것 같습니다.
- Softirq : https://hogbal.github.io/posts/Linux-Softirq/
- socket buffer descriptors : https://hogbal.github.io/posts/Linux-Socket-buffer-descriptors/
💡 Bluetooth?
Bluetooth도 타겟으로 잡긴했지만, 현재 qemu에서 지원하지 않는 가상화 장치이기 때문에 위의 모듈만 먼저 공부했습니다.
해당 포스터는 Wireless Stack에 대해서 설명하고 중요한 모듈인 cfg80211, mac80211에 대해서 설명드리겠습니다!
Wireless Stack
우선, 패킷을 수신하는 과정은 다음 그림에 잘 표현되어 있습니다. 그림을 보면 패킷이 수신된다면 Driver interrupt handler를 통해 처리가 된 이후, softirq handler가 작동해서 패킷을 수신하는 것을 확인할 수 있습니다.
패킷 수신 과정
다음 챕터부터 자세하게 설명드리겠지만, 결국 사용하고 있는 Wireless Driver에서 구현한 함수까지 흘러가는 로직은 동일하고 이후부터 각각의 드라이버에서 처리하게 됩니다.
TX, RX Call Stack
우선 데이터가 담긴 패킷을 송수신하기 위한 호출 스택은 아래와 같습니다.
💡 아래에서부터 위쪽으로 함수가 호출되는 것입니다!
- 송신
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
#0 __netdev_start_xmit (more=0x0, dev=0xffff88800416a000, skb=0xffff88800684ed00, ops=0xffffffff82356a80 <ieee80211_dataif_ops>) at ./include/linux/netdevice.h:4988
#1 netdev_start_xmit (more=0x0, txq=0xffff888003b9b800, dev=0xffff88800416a000, skb=0xffff88800684ed00) at ./include/linux/netdevice.h:5003
#2 xmit_one (more=0x0, txq=0xffff888003b9b800, dev=0xffff88800416a000, skb=0xffff88800684ed00) at net/core/dev.c:3547
#3 dev_hard_start_xmit (first=first@entry=0xffff88800684ed00, dev=dev@entry=0xffff88800416a000, txq=txq@entry=0xffff888003b9b800, ret=ret@entry=0xffffc9000031f7dc) at net/core/dev.c:3563
#4 0xffffffff81b7e7b1 in __dev_queue_xmit (skb=0xffff88800684ed00, sb_dev=0x0 <fixed_percpu_data>) at net/core/dev.c:4351
#5 0xffffffff81c74003 in arp_solicit (neigh=0xffff8880067a4600, skb=<optimized out>) at net/ipv4/arp.c:392
#6 0xffffffff81b886ec in neigh_probe (neigh=0xffff8880067a4600) at net/core/neighbour.c:1075
#7 0xffffffff81b8e7a2 in __neigh_event_send (neigh=neigh@entry=0xffff8880067a4600, skb=skb@entry=0xffff8880069196e0, immediate_ok=immediate_ok@entry=0x1) at net/core/neighbour.c:1242
#8 0xffffffff81b8eb1a in neigh_event_send_probe (immediate_ok=<optimized out>, skb=<optimized out>, neigh=<optimized out>) at ./include/net/neighbour.h:466
#9 neigh_event_send (skb=0xffff8880069196e0, neigh=0xffff8880067a4600) at ./include/net/neighbour.h:472
#10 neigh_event_send (skb=0xffff8880069196e0, neigh=0xffff8880067a4600) at ./include/net/neighbour.h:470
#11 neigh_resolve_output (neigh=0xffff8880067a4600, skb=0xffff8880069196e0) at net/core/neighbour.c:1547
#12 0xffffffff81c3145b in neigh_output (skip_cache=<optimized out>, skb=0xffff8880069196e0, n=0xffff8880067a4600) at ./include/net/neighbour.h:542
#13 ip_finish_output2 (net=<optimized out>, sk=<optimized out>, skb=0xffff8880069196e0) at net/ipv4/ip_output.c:235
#14 0xffffffff81c33a8b in __ip_queue_xmit (sk=0xffff888005470000, skb=0xffff8880069196e0, fl=0xffff888005470368, tos=<optimized out>) at net/ipv4/ip_output.c:535
#15 0xffffffff81c33d70 in ip_queue_xmit (sk=<optimized out>, skb=<optimized out>, fl=<optimized out>) at net/ipv4/ip_output.c:549
#16 0xffffffff81c566a7 in __tcp_transmit_skb (sk=sk@entry=0xffff888005470000, skb=0xffff8880069196e0, skb@entry=0xffff888006919600, clone_it=clone_it@entry=0x1, gfp_mask=<optimized out>, rcv_nxt=<optimized out>) at net/ipv4/tcp_output.c:1462
#17 0xffffffff81c56ee3 in tcp_transmit_skb (gfp_mask=<optimized out>, clone_it=0x1, skb=0xffff888006919600, sk=0xffff888005470000) at net/ipv4/tcp_output.c:1480
#18 tcp_send_syn_data (syn=0xffff888006919600, sk=0xffff888005470000) at net/ipv4/tcp_output.c:4013
#19 tcp_connect (sk=sk@entry=0xffff888005470000) at net/ipv4/tcp_output.c:4099
#20 0xffffffff81c5db22 in tcp_v4_connect (sk=0xffff888005470000, uaddr=0xffffc9000031fde8, addr_len=<optimized out>) at net/ipv4/tcp_ipv4.c:324
#21 0xffffffff81c7dfdc in __inet_stream_connect (sock=0xffff8880148e5a00, uaddr=uaddr@entry=0xffffc9000031fde8, addr_len=<optimized out>, flags=0x800, is_sendmsg=is_sendmsg@entry=0x1) at net/ipv4/af_inet.c:678
#22 0xffffffff81c42357 in tcp_sendmsg_fastopen (sk=sk@entry=0xffff888005470000, msg=msg@entry=0xffffc9000031feb0, copied=copied@entry=0xffffc9000031fc60, size=size@entry=0x26, uarg=uarg@entry=0x0 <fixed_percpu_data>) at net/ipv4/tcp.c:1025
#23 0xffffffff81c42d2b in tcp_sendmsg_locked (sk=sk@entry=0xffff888005470000, msg=msg@entry=0xffffc9000031feb0, size=size@entry=0x26) at net/ipv4/tcp.c:1077
#24 0xffffffff81c43137 in tcp_sendmsg (sk=0xffff888005470000, msg=0xffffc9000031feb0, size=0x26) at net/ipv4/tcp.c:1341
#25 0xffffffff81b4e9dd in sock_sendmsg_nosec (msg=0xffffc9000031feb0, sock=0xffff8880148e5a00) at ./include/linux/uio.h:294
#26 __sock_sendmsg (msg=0xffffc9000031feb0, sock=0xffff8880148e5a00) at net/socket.c:745
#27 ____sys_sendmsg (sock=sock@entry=0xffff8880148e5a00, msg_sys=msg_sys@entry=0xffffc9000031feb0, flags=flags@entry=0x20000000, used_address=used_address@entry=0x0 <fixed_percpu_data>, allowed_msghdr_flags=allowed_msghdr_flags@entry=0x0) at net/socket.c:2584
#28 0xffffffff81b50fa3 in ___sys_sendmsg (sock=sock@entry=0xffff8880148e5a00, msg=msg@entry=0x7fff16761540, msg_sys=msg_sys@entry=0xffffc9000031feb0, flags=flags@entry=0x20000000, used_address=used_address@entry=0x0 <fixed_percpu_data>, allowed_msghdr_flags=allowed_msghdr_flags@entry=0x0) at net/socket.c:2638
#29 0xffffffff81b510c8 in __sys_sendmsg (fd=<optimized out>, msg=0x7fff16761540, flags=0x20000000, forbid_cmsg_compat=<optimized out>) at net/socket.c:2667
#30 0xffffffff81ec2353 in do_syscall_x64 (nr=<optimized out>, regs=0xffffc9000031ff58) at arch/x86/entry/common.c:52
#31 do_syscall_64 (regs=0xffffc9000031ff58, nr=<optimized out>) at arch/x86/entry/common.c:83
- 수신
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#0 __netdev_start_xmit (more=0x0, dev=0xffff888004168000, skb=0xffff88800684e000, ops=0xffffffff82356a80 <ieee80211_dataif_ops>) at ./include/linux/netdevice.h:4988
#1 netdev_start_xmit (more=0x0, txq=0xffff888003b9ae00, dev=0xffff888004168000, skb=0xffff88800684e000) at ./include/linux/netdevice.h:5003
#2 xmit_one (more=0x0, txq=0xffff888003b9ae00, dev=0xffff888004168000, skb=0xffff88800684e000) at net/core/dev.c:3547
#3 dev_hard_start_xmit (first=first@entry=0xffff88800684e000, dev=dev@entry=0xffff888004168000, txq=txq@entry=0xffff888003b9ae00, ret=ret@entry=0xffffc90000003c24) at net/core/dev.c:3563
#4 0xffffffff81b7e7b1 in __dev_queue_xmit (skb=0xffff88800684e000, sb_dev=sb_dev@entry=0x0 <fixed_percpu_data>) at net/core/dev.c:4351
#5 0xffffffff81e26f1d in dev_queue_xmit (skb=<optimized out>) at ./include/linux/netdevice.h:3171
#6 0xffffffff81e2a0dd in ieee80211_rx_h_data (rx=0xffffc90000003e58) at net/mac80211/rx.c:3214
#7 ieee80211_rx_handlers (rx=rx@entry=0xffffc90000003e58, frames=frames@entry=0xffffc90000003db0) at net/mac80211/rx.c:4142
#8 0xffffffff81e2c4bb in ieee80211_invoke_rx_handlers (rx=0xffffc90000003e58) at net/mac80211/rx.c:4185
#9 ieee80211_prepare_and_rx_handle (rx=rx@entry=0xffffc90000003e58, skb=skb@entry=0xffff88800684e900, consume=consume@entry=0x1) at net/mac80211/rx.c:5033
#10 0xffffffff81e2da7c in __ieee80211_rx_handle_packet (list=<optimized out>, skb=0xffff88800684e900, pubsta=<optimized out>, hw=<optimized out>) at net/mac80211/rx.c:5239
#11 ieee80211_rx_list (hw=<optimized out>, pubsta=<optimized out>, skb=0xffff88800684e900, list=<optimized out>) at net/mac80211/rx.c:5410
#12 0xffffffff81e2dec0 in ieee80211_rx_napi (hw=hw@entry=0xffff8880064688e0, pubsta=pubsta@entry=0x0 <fixed_percpu_data>, skb=skb@entry=0xffff88800684e900, napi=napi@entry=0x0 <fixed_percpu_data>) at net/mac80211/rx.c:5433
#13 0xffffffff81df8941 in ieee80211_rx (skb=0xffff88800684e900, hw=<optimized out>) at ./include/net/mac80211.h:4983
#14 ieee80211_tasklet_handler (t=<optimized out>) at net/mac80211/main.c:318
#15 0xffffffff8108ee5b in tasklet_action_common (tl_head=0xffff88807d01c6b0, softirq_nr=0x6, a=<optimized out>) at kernel/softirq.c:780
#16 0xffffffff81ed2410 in __do_softirq () at kernel/softirq.c:553
#17 0xffffffff8108f5a5 in invoke_softirq () at kernel/softirq.c:427
#18 __irq_exit_rcu () at kernel/softirq.c:632
#19 irq_exit_rcu () at kernel/softirq.c:644
#20 0xffffffff81ec5fc0 in sysvec_apic_timer_interrupt (regs=0xffffffff82a03dd8) at arch/x86/kernel/apic/apic.c:1076
호출 스택을 보면 알겠지만 udp 수신을 제외한 공통적으로 다음의 함수들이 순서대로 호출이 되는 것을 확인할 수 있습니다.
💡 그 외에 앞에서 호출되는 함수는 패킷 헤더를 생성하거나 tcp 연결을 위해 호출되는 함수입니다.
dev_queue_xmit__dev_queue_xmitdev_hard_start_xmitxmit_onenetdev_start_xmit__netdev_start_xmit
dev_queue_xmit
- include/linux/netdevice.h
전달받은 skb를 __dev_queue_xmit 함수에 전달하고 2번째 인자인 struct net_device *sb_dev 을 NULL로 전달합니다.
1
2
3
4
static inline int dev_queue_xmit(structsk_buff *skb)
{
return __dev_queue_xmit(skb, NULL);
}
__dev_queue_xmit
- net/core/dev.c
해당 함수에서 soft irq를 비활성화 하고 queue가 존재하면 __dev_xmit_skb 함수를 호출하고 아니라면 바로 dev_hard_start_xmit 함수를 호출합니다.
💡 여기서 tcp 요청이면
Qdisc구조체를 통해 queue에 저장해야 되기 때문에 해당 구조체의enqueue멤버 변수가 존재한다면__dev_xmit_skb함수를 호출합니다.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
{
struct net_device *dev = skb->dev;
struct netdev_queue *txq = NULL;
struct Qdisc *q;
int rc = -ENOMEM;
bool again = false;
skb_reset_mac_header(skb);
skb_assert_len(skb);
if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
__skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
/* Disable soft irqs for various locks below. Also
* stops preemption for RCU.
*/
rcu_read_lock_bh();
skb_update_prio(skb);
qdisc_pkt_len_init(skb);
tcx_set_ingress(skb, false);
#ifdef CONFIG_NET_EGRESS
if (static_branch_unlikely(&egress_needed_key)) {
if (nf_hook_egress_active()) {
skb = nf_hook_egress(skb, &rc, dev);
if (!skb)
goto out;
}
netdev_xmit_skip_txqueue(false);
nf_skip_egress(skb, true);
skb = sch_handle_egress(skb, &rc, dev);
if (!skb)
goto out;
nf_skip_egress(skb, false);
if (netdev_xmit_txqueue_skipped())
txq = netdev_tx_queue_mapping(dev, skb);
}
#endif
/* If device/qdisc don't need skb->dst, release it right now while
* its hot in this cpu cache.
*/
if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
skb_dst_drop(skb);
else
skb_dst_force(skb);
if (!txq)
txq = netdev_core_pick_tx(dev, skb, sb_dev);
q = rcu_dereference_bh(txq->qdisc);
trace_net_dev_queue(skb);
if (q->enqueue) {
rc = __dev_xmit_skb(skb, q, dev, txq);
goto out;
}
/* The device has no queue. Common case for software devices:
* loopback, all the sorts of tunnels...
* Really, it is unlikely that netif_tx_lock protection is necessary
* here. (f.e. loopback and IP tunnels are clean ignoring statistics
* counters.)
* However, it is possible, that they rely on protection
* made by us here.
* Check this and shot the lock. It is not prone from deadlocks.
*Either shot noqueue qdisc, it is even simpler 8)
*/
if (dev->flags & IFF_UP) {
int cpu = smp_processor_id(); /* ok because BHs are off */
/* Other cpus might concurrently change txq->xmit_lock_owner
* to -1 or to their cpu id, but not to our id.
*/
if (READ_ONCE(txq->xmit_lock_owner) != cpu) {
if (dev_xmit_recursion())
goto recursion_alert;
skb = validate_xmit_skb(skb, dev, &again);
if (!skb)
goto out;
HARD_TX_LOCK(dev, txq, cpu);
if (!netif_xmit_stopped(txq)) {
dev_xmit_recursion_inc();
skb = dev_hard_start_xmit(skb, dev, txq, &rc);
dev_xmit_recursion_dec();
if (dev_xmit_complete(rc)) {
HARD_TX_UNLOCK(dev, txq);
goto out;
}
}
HARD_TX_UNLOCK(dev, txq);
net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
dev->name);
} else {
/* Recursion is detected! It is possible,
* unfortunately
*/
recursion_alert:
net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
dev->name);
}
}
rc = -ENETDOWN;
rcu_read_unlock_bh();
dev_core_stats_tx_dropped_inc(dev);
kfree_skb_list(skb);
return rc;
out:
rcu_read_unlock_bh();
return rc;
}
EXPORT_SYMBOL(__dev_queue_xmit);
dev_hard_start_xmit
- net/core/dev.c
해당 함수에서 반복문을 돌면서 sk_buff의 리스트를 하나씩 인자로 넣어줘서 xmit_one 함수를 호출하게 됩니다.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
struct netdev_queue *txq, int *ret)
{
struct sk_buff *skb = first;
int rc = NETDEV_TX_OK;
while (skb) {
struct sk_buff *next = skb->next;
skb_mark_not_on_list(skb);
rc = xmit_one(skb, dev, txq, next != NULL);
if (unlikely(!dev_xmit_complete(rc))) {
skb->next = next;
goto out;
}
skb = next;
if (netif_tx_queue_stopped(txq) && skb) {
rc = NETDEV_TX_BUSY;
break;
}
}
out:
*ret = rc;
return skb;
}
xmit_one
- net/core/dev.c
dev_nit_active 함수를 통해서 검증을 하고 netdev_start_xmit 함수를 호출합니다.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
static int xmit_one(struct sk_buff *skb, struct net_device *dev,
struct netdev_queue *txq, bool more)
{
unsigned int len;
int rc;
if (dev_nit_active(dev))
dev_queue_xmit_nit(skb, dev);
len = skb->len;
trace_net_dev_start_xmit(skb, dev);
rc = netdev_start_xmit(skb, dev, txq, more);
trace_net_dev_xmit(skb, rc, dev, len);
return rc;
}
netdev_start_xmit
- include/linux/netdevice.h
해당 함수는 간단합니다. __netdev_start_xmit 함수를 호출하기 위해 인자를 설정한 뒤 호출하게 됩니다.
1
2
3
4
5
6
7
8
9
10
11
12
static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
struct netdev_queue *txq, bool more)
{
const struct net_device_ops *ops = dev->netdev_ops;
netdev_tx_t rc;
rc = __netdev_start_xmit(ops, skb, dev, more);
if (rc == NETDEV_TX_OK)
txq_trans_update(txq);
return rc;
}
__netdev_start_xmit
- include/linux/netdevice.h
해당 함수는 인자로 전달된 ops 변수의 ndo_start_xmit함수를 호출하게 됩니다.
1
2
3
4
5
6
7
static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
struct sk_buff *skb, struct net_device *dev,
bool more)
{
__this_cpu_write(softnet_data.xmit.more, more);
return ops->ndo_start_xmit(skb, dev);
}
여기서 ndo_start_xmit 함수는 Wireless Driver마다 다르며 이후, 제 환경에서는 ieee80211_subif_start_xmit 함수가 실행되게 됩니다.
💡 Wireless Driver?
lspci -k | grep -i network -A 2해당 명령어를 통해 어떤 드라이버를 사용하는지 확인할 수 있습니다.
Wireless Driver
위에서 언급한 Wireless Driver는 cfg80211 또는 mac80211을 대상으로 구현해야 됩니다. 해당 모듈은 아래와 같은 차이점이 있습니다.
- cfg80211: Full MAC
- mac80211: Soft MAC
💡 MAC?
여기서 MAC은 Media Access Control의 약자입니다.
이렇게 구현되어 있기 때문에 __netdev_start_xmit 함수에서 호출할 때 자신의 환경에 맞는 함수가 호출되게 되는 것입니다.
참고 사이트
This post is licensed under CC BY 4.0 by the author.