diff options
Diffstat (limited to 'src/core/bpf/clat.bpf.c')
| -rw-r--r-- | src/core/bpf/clat.bpf.c | 1203 |
1 files changed, 1203 insertions, 0 deletions
diff --git a/src/core/bpf/clat.bpf.c b/src/core/bpf/clat.bpf.c new file mode 100644 index 00000000..97d74a61 --- /dev/null +++ b/src/core/bpf/clat.bpf.c @@ -0,0 +1,1203 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright 2021 Toke Høiland-Jørgensen <toke@toke.dk> */ +/* Copyright 2025 Mary Strodl <mstrodl@csh.rit.edu> */ +/* Copyright 2026 Beniamino Galvani <bgalvani@redhat.com> */ + +/** + * This is an implementation of a CLAT in eBPF. BPF is a different environment + * than the rest of NetworkManager, and we don't have access to most of the + * C standard library, so some things might look a little different from what + * you're used to. + * + * Check out src/core/bpf/meson.build to see how this gets built. + **/ + +#include <linux/bpf.h> +#include <linux/icmp.h> +#include <linux/icmpv6.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/in.h> +#include <linux/in6.h> +#include <linux/pkt_cls.h> +#include <linux/tcp.h> +#include <linux/udp.h> +#include <linux/if_ether.h> +#include <stdbool.h> + +#include <bpf/bpf_endian.h> +#include <bpf/bpf_helpers.h> + +#include "clat.h" + +char _license[] SEC("license") = "GPL"; + +struct clat_config config; +struct clat_stats stats; + +#ifdef DEBUG +/* Note: when enabling debugging, you also need to add CAP_PERFMON + * to the CapabilityBoundingSet of the NM systemd unit. The messages + * will be printed to /sys/kernel/debug/tracing/trace_pipe */ +#define DBG(fmt, ...) \ + ({ \ + char ____fmt[] = "clat: " fmt; \ + bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \ + }) +#else +#define DBG(fmt, ...) +#endif + +/* Macros to read the sk_buff data* pointers, preventing the compiler + * from generating a 32-bit register spill. */ +#define SKB_ACCESS_MEMBER_32(_skb, member) \ + ({ \ + void *ptr; \ + \ + asm volatile("%0 = *(u32 *)(%1 + %2)" \ + : "=r"(ptr) \ + : "r"(_skb), "i"(offsetof(struct __sk_buff, member))); \ + \ + ptr; \ + }) + +#define SKB_DATA(_skb) SKB_ACCESS_MEMBER_32(_skb, data) +#define SKB_DATA_END(_skb) SKB_ACCESS_MEMBER_32(_skb, data_end) + +struct icmpv6_pseudo { + struct in6_addr saddr; + struct in6_addr daddr; + __u32 len; + __u8 padding[3]; + __u8 nh; +} __attribute__((packed)); + +struct ip6_frag { + __u8 nexthdr; + __u8 reserved; + __u16 offset; + __u32 identification; +} __attribute__((packed)); + +#define L2_H_LEN(has_eth) (has_eth ? sizeof(struct ethhdr) : 0) +#define IP_H_LEN (sizeof(struct iphdr)) +#define IP6_H_LEN (sizeof(struct ipv6hdr)) +#define IP6_FRAG_H_LEN (sizeof(struct ip6_frag)) +#define ICMP_H_LEN (sizeof(struct icmphdr)) +#define ICMP6_H_LEN (sizeof(struct icmp6hdr)) + +#define ensure_header(header, skb, data, data_end, offset) \ + _ensure_header((void **) header, (skb), (data), (data_end), sizeof(**(header)), (offset)) + +/* + * Verifies that the header at offset @offset and with size @size can + * be accessed, and assigns the pointer to @header. In case the data + * is not available, the function tries to pull it. Note that all packet + * pointers must be refreshed after calling this function. + */ +static __always_inline bool +_ensure_header(void **header, + struct __sk_buff *skb, + void **data, + void **data_end, + unsigned size, + unsigned offset) +{ + if (*data + offset + size > *data_end) { + bpf_skb_pull_data(skb, offset + size); + *data = SKB_DATA(skb); + *data_end = SKB_DATA_END(skb); + } + + if (*data + offset + size > *data_end) + return false; + + *header = *data + offset; + return true; +} + +/* This function must be declared as inline because the BPF calling + * convention only supports up to 5 function arguments. */ +static __always_inline void +update_l4_checksum(struct __sk_buff *skb, + struct ipv6hdr *ip6h, + struct iphdr *iph, + bool has_eth, + bool v4to6, + bool is_inner, + bool is_v6_fragment, + __u32 *csum_diff) +{ + int flags = BPF_F_PSEUDO_HDR; + __u16 offset; + __u32 csum; + int ip_type; + + if (v4to6) { + void *from_ptr = &iph->saddr; + void *to_ptr = &ip6h->saddr; + + csum = bpf_csum_diff(from_ptr, 2 * sizeof(__u32), to_ptr, 2 * sizeof(struct in6_addr), 0); + offset = L2_H_LEN(has_eth) + IP_H_LEN; + ip_type = ip6h->nexthdr; + } else { + void *from_ptr = &ip6h->saddr; + void *to_ptr = &iph->saddr; + + csum = bpf_csum_diff(from_ptr, 2 * sizeof(struct in6_addr), to_ptr, 2 * sizeof(__u32), 0); + offset = L2_H_LEN(has_eth) + IP6_H_LEN; + ip_type = iph->protocol; + + if (is_inner) { + offset = offset + ICMP6_H_LEN + IP6_H_LEN; + } + } + + if (is_v6_fragment) { + offset += IP6_FRAG_H_LEN; + } + + switch (ip_type) { + case IPPROTO_TCP: + offset += offsetof(struct tcphdr, check); + break; + case IPPROTO_UDP: + offset += offsetof(struct udphdr, check); + flags |= BPF_F_MARK_MANGLED_0; + break; + default: + return; + } + + bpf_l4_csum_replace(skb, offset, 0, csum, flags); + + if (csum_diff) { + *csum_diff = bpf_csum_diff((__be32 *) &csum, sizeof(csum), 0, 0, *csum_diff); + } +} + +static __always_inline void +update_icmp_checksum(struct __sk_buff *skb, + const struct ipv6hdr *ip6h, + void *icmp_before, + void *icmp_after, + bool has_eth, + bool v4to6, + bool is_inner, + __u32 seed) +{ + struct icmpv6_pseudo ph = {.nh = IPPROTO_ICMPV6, .len = ip6h->payload_len}; + __u16 h_before; + __u16 h_after; + __u16 offset; + __u32 csum; + __u32 u_before; + __u32 u_after; + + __builtin_memcpy(&ph.saddr, &ip6h->saddr, sizeof(struct in6_addr)); + __builtin_memcpy(&ph.daddr, &ip6h->daddr, sizeof(struct in6_addr)); + + /* Do checksum update in two passes: first compute the incremental + * checksum update of the ICMPv6 pseudo header, update the checksum + * using bpf_l4_csum_replace(), and then do a separate update for the + * ICMP type and code (which is two consecutive bytes, so cast them to + * u16). The bpf_csum_diff() helper can be used to compute the + * incremental update of the full block, whereas the + * bpf_l4_csum_replace() helper can do the two-byte diff and update by + * itself. + */ + csum = bpf_csum_diff((__be32 *) &ph, + v4to6 ? 0 : sizeof(ph), + (__be32 *) &ph, + v4to6 ? sizeof(ph) : 0, + seed); + + if (v4to6) { + offset = L2_H_LEN(has_eth) + IP_H_LEN + 2; + } else { + offset = L2_H_LEN(has_eth) + IP6_H_LEN + 2; + if (is_inner) + offset += ICMP6_H_LEN + IP6_H_LEN; + } + + /* first two bytes of ICMP header, type and code */ + h_before = *(__u16 *) icmp_before; + h_after = *(__u16 *) icmp_after; + + /* last four bytes of ICMP header, the data union */ + u_before = *(__u32 *) (icmp_before + 4); + u_after = *(__u32 *) (icmp_after + 4); + + bpf_l4_csum_replace(skb, offset, 0, csum, BPF_F_PSEUDO_HDR); + bpf_l4_csum_replace(skb, offset, h_before, h_after, 2); + + if (u_before != u_after) + bpf_l4_csum_replace(skb, offset, u_before, u_after, 4); +} + +static __always_inline int +rewrite_icmp(struct __sk_buff *skb, const struct ipv6hdr *ip6h, bool has_eth) +{ + void *data_end = SKB_DATA_END(skb); + void *data = SKB_DATA(skb); + struct icmphdr icmp_buf; /* copy of the old ICMPv4 header */ + struct icmp6hdr icmp6_buf; /* buffer for the new ICMPv6 header */ + struct icmphdr *icmp; + struct icmp6hdr *icmp6; + __u32 mtu; + + if (!ensure_header(&icmp, skb, &data, &data_end, L2_H_LEN(has_eth) + IP_H_LEN)) + return -1; + + icmp_buf = *icmp; + icmp6 = (void *) icmp; + icmp6_buf = *icmp6; + + /* These translations are defined in RFC6145 section 4.2 */ + switch (icmp->type) { + case ICMP_ECHO: + icmp6_buf.icmp6_type = ICMPV6_ECHO_REQUEST; + break; + case ICMP_ECHOREPLY: + icmp6_buf.icmp6_type = ICMPV6_ECHO_REPLY; + break; + case ICMP_DEST_UNREACH: + icmp6_buf.icmp6_type = ICMPV6_DEST_UNREACH; + switch (icmp->code) { + case ICMP_NET_UNREACH: + case ICMP_HOST_UNREACH: + case ICMP_SR_FAILED: + case ICMP_NET_UNKNOWN: + case ICMP_HOST_UNKNOWN: + case ICMP_HOST_ISOLATED: + case ICMP_NET_UNR_TOS: + case ICMP_HOST_UNR_TOS: + icmp6_buf.icmp6_code = ICMPV6_NOROUTE; + break; + case ICMP_PROT_UNREACH: + icmp6_buf.icmp6_type = ICMPV6_PARAMPROB; + icmp6_buf.icmp6_code = ICMPV6_UNK_NEXTHDR; + icmp6_buf.icmp6_pointer = bpf_htonl(offsetof(struct ipv6hdr, nexthdr)); + break; + case ICMP_PORT_UNREACH: + icmp6_buf.icmp6_code = ICMPV6_PORT_UNREACH; + break; + case ICMP_FRAG_NEEDED: + icmp6_buf.icmp6_type = ICMPV6_PKT_TOOBIG; + icmp6_buf.icmp6_code = 0; + mtu = bpf_ntohs(icmp->un.frag.mtu) + 20; + /* RFC6145 section 6, "second approach" - should not be + * necessary, but might as well do this + */ + if (mtu < 1280) + mtu = 1280; + icmp6_buf.icmp6_mtu = bpf_htonl(mtu); + break; + case ICMP_NET_ANO: + case ICMP_HOST_ANO: + case ICMP_PKT_FILTERED: + case ICMP_PREC_CUTOFF: + icmp6_buf.icmp6_code = ICMPV6_ADM_PROHIBITED; + break; + default: + return -1; + } + break; + case ICMP_PARAMETERPROB: + if (icmp->code == 1) + return -1; + icmp6_buf.icmp6_type = ICMPV6_PARAMPROB; + icmp6_buf.icmp6_code = ICMPV6_HDR_FIELD; + /* The pointer field not defined in the Linux header. This + * translation is from Figure 3 of RFC6145. + */ + switch (icmp->un.reserved[0]) { + case 0: /* version/IHL */ + icmp6_buf.icmp6_pointer = 0; + break; + case 1: /* Type of Service */ + icmp6_buf.icmp6_pointer = bpf_htonl(1); + break; + case 2: /* Total length */ + case 3: + icmp6_buf.icmp6_pointer = bpf_htonl(4); + break; + case 8: /* Time to Live */ + icmp6_buf.icmp6_pointer = bpf_htonl(7); + break; + case 9: /* Protocol */ + icmp6_buf.icmp6_pointer = bpf_htonl(6); + break; + case 12: /* Source address */ + case 13: + case 14: + case 15: + icmp6_buf.icmp6_pointer = bpf_htonl(8); + break; + case 16: /* Destination address */ + case 17: + case 18: + case 19: + icmp6_buf.icmp6_pointer = bpf_htonl(24); + break; + default: + return -1; + } + break; + default: + return -1; + } + + *icmp6 = icmp6_buf; + update_icmp_checksum(skb, ip6h, &icmp_buf, icmp6, has_eth, true, false, 0); + + /* FIXME: also need to rewrite IP header embedded in ICMP error */ + + return 0; +} + +/* + * Convert an IPv4 address to the corresponding "IPv4-Embedded IPv6 Address" + * according to RFC 6052 2.2. + * + * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + * |PL| 0-------------32--40--48--56--64--72--80--88--96--104---------| + * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + * |32| prefix |v4(32) | u | suffix | + * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + * |40| prefix |v4(24) | u |(8)| suffix | + * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + * |48| prefix |v4(16) | u | (16) | suffix | + * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + * |56| prefix |(8)| u | v4(24) | suffix | + * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + * |64| prefix | u | v4(32) | suffix | + * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + * |96| prefix | v4(32) | + * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + * + */ +static __always_inline bool +v4addr_to_v6(__be32 addr4, struct in6_addr *addr6, const struct in6_addr *pref64, int pref64_len) +{ + union { + __be32 a32; + __u8 a8[4]; + } u; + + u.a32 = addr4; + + addr6->s6_addr32[0] = 0; + addr6->s6_addr32[1] = 0; + addr6->s6_addr32[2] = 0; + addr6->s6_addr32[3] = 0; + + switch (pref64_len) { + case 96: + addr6->s6_addr32[0] = pref64->s6_addr32[0]; + addr6->s6_addr32[1] = pref64->s6_addr32[1]; + addr6->s6_addr32[2] = pref64->s6_addr32[2]; + addr6->s6_addr32[3] = addr4; + break; + case 64: + addr6->s6_addr32[0] = pref64->s6_addr32[0]; + addr6->s6_addr32[1] = pref64->s6_addr32[1]; + addr6->s6_addr[9] = u.a8[0]; + addr6->s6_addr[10] = u.a8[1]; + addr6->s6_addr[11] = u.a8[2]; + addr6->s6_addr[12] = u.a8[3]; + break; + case 56: + addr6->s6_addr32[0] = pref64->s6_addr32[0]; + addr6->s6_addr32[1] = pref64->s6_addr32[1]; + addr6->s6_addr[7] = u.a8[0]; + addr6->s6_addr[9] = u.a8[1]; + addr6->s6_addr[10] = u.a8[2]; + addr6->s6_addr[11] = u.a8[3]; + break; + case 48: + addr6->s6_addr32[0] = pref64->s6_addr32[0]; + addr6->s6_addr16[2] = pref64->s6_addr16[2]; + addr6->s6_addr[6] = u.a8[0]; + addr6->s6_addr[7] = u.a8[1]; + addr6->s6_addr[9] = u.a8[2]; + addr6->s6_addr[10] = u.a8[3]; + break; + case 40: + addr6->s6_addr32[0] = pref64->s6_addr32[0]; + addr6->s6_addr[4] = pref64->s6_addr[4]; + addr6->s6_addr[5] = u.a8[0]; + addr6->s6_addr[6] = u.a8[1]; + addr6->s6_addr[7] = u.a8[2]; + addr6->s6_addr[9] = u.a8[3]; + break; + case 32: + addr6->s6_addr32[0] = pref64->s6_addr32[0]; + addr6->s6_addr32[1] = addr4; + break; + default: + return false; + } + return true; +} + +/* + * Extract the IPv4 address @addr4 and the NAT64 prefix @pref64 from an IPv6 address, + * given the known prefix length @pref64_len. See the table above. + */ +static __always_inline bool +v6addr_to_v4(const struct in6_addr *addr6, int pref64_len, __be32 *addr4, struct in6_addr *pref64) +{ + union { + __be32 a32; + __u8 a8[4]; + } u; + + pref64->s6_addr32[0] = 0; + pref64->s6_addr32[1] = 0; + pref64->s6_addr32[2] = 0; + pref64->s6_addr32[3] = 0; + + switch (pref64_len) { + case 96: + u.a32 = addr6->s6_addr32[3]; + + pref64->s6_addr32[0] = addr6->s6_addr32[0]; + pref64->s6_addr32[1] = addr6->s6_addr32[1]; + pref64->s6_addr32[2] = addr6->s6_addr32[2]; + break; + case 64: + u.a8[0] = addr6->s6_addr[9]; + u.a8[1] = addr6->s6_addr[10]; + u.a8[2] = addr6->s6_addr[11]; + u.a8[3] = addr6->s6_addr[12]; + + pref64->s6_addr32[0] = addr6->s6_addr32[0]; + pref64->s6_addr32[1] = addr6->s6_addr32[1]; + break; + case 56: + u.a8[0] = addr6->s6_addr[7]; + u.a8[1] = addr6->s6_addr[9]; + u.a8[2] = addr6->s6_addr[10]; + u.a8[3] = addr6->s6_addr[11]; + + pref64->s6_addr32[0] = addr6->s6_addr32[0]; + pref64->s6_addr32[1] = addr6->s6_addr32[1]; + pref64->s6_addr[7] = 0; + break; + case 48: + u.a8[0] = addr6->s6_addr[6]; + u.a8[1] = addr6->s6_addr[7]; + u.a8[2] = addr6->s6_addr[9]; + u.a8[3] = addr6->s6_addr[10]; + + pref64->s6_addr32[0] = addr6->s6_addr32[0]; + pref64->s6_addr32[1] = addr6->s6_addr32[1]; + pref64->s6_addr16[3] = 0; + break; + case 40: + u.a8[0] = addr6->s6_addr[5]; + u.a8[1] = addr6->s6_addr[6]; + u.a8[2] = addr6->s6_addr[7]; + u.a8[3] = addr6->s6_addr[9]; + + pref64->s6_addr32[0] = addr6->s6_addr32[0]; + pref64->s6_addr32[1] = addr6->s6_addr32[1]; + pref64->s6_addr16[3] = 0; + pref64->s6_addr[5] = 0; + + break; + case 32: + u.a32 = addr6->s6_addr32[1]; + + pref64->s6_addr32[0] = addr6->s6_addr32[0]; + break; + default: + return false; + } + + *addr4 = u.a32; + return true; +} + +/* ipv4 traffic in from application on this device, needs to be translated to v6 and sent to PLAT */ +static __always_inline int +clat_handle_v4(struct __sk_buff *skb, bool has_eth) +{ + int ret = TC_ACT_OK; + void *data_end = SKB_DATA_END(skb); + void *data = SKB_DATA(skb); + struct ipv6hdr *ip6h; + struct ipv6hdr dst_hdr = { + .version = 6, + }; + struct iphdr *iph; + struct ethhdr *eth; + + if (!ensure_header(&iph, skb, &data, &data_end, L2_H_LEN(has_eth))) + goto out; + + if (has_eth) { + eth = data; + if (eth->h_proto != bpf_htons(ETH_P_IP)) + goto out; + } + + if (iph->version != 4) + goto out; + + if (iph->saddr != config.local_v4.s_addr) + goto out; + + /* At this point we know the packet needs translation. If we can't + * rewrite it, it should be dropped. + */ + ret = TC_ACT_SHOT; + + /* we don't bother dealing with IP options or fragmented packets. The + * latter are identified by the 'frag_off' field having a value (either + * the MF bit, or the fragment offset, or both). However, this field also + * contains the "don't fragment" (DF) bit, which we ignore, so mask that + * out. The DF is the second-most-significant bit (as bit 0 is + * reserved). + */ + + if (iph->ihl != 5 || (iph->frag_off & ~bpf_htons(1 << 14))) { + DBG("v4: pkt src/dst %pI4/%pI4 has IP options or is fragmented, dropping\n", + &iph->saddr, + &iph->daddr); + goto out; + } + + if (!v4addr_to_v6(iph->daddr, &dst_hdr.daddr, &config.pref64, config.pref64_len)) + goto out; + + dst_hdr.saddr = config.local_v6; + dst_hdr.nexthdr = iph->protocol; + dst_hdr.hop_limit = iph->ttl; + /* weird definition in ipv6hdr */ + dst_hdr.priority = (iph->tos & 0x70) >> 4; + dst_hdr.flow_lbl[0] = iph->tos << 4; + dst_hdr.payload_len = bpf_htons(bpf_ntohs(iph->tot_len) - IP_H_LEN); + + DBG("v4: outgoing pkt to dst %pI4 (%pI6c)\n", &iph->daddr, &dst_hdr.daddr); + + switch (dst_hdr.nexthdr) { + case IPPROTO_ICMP: + if (rewrite_icmp(skb, &dst_hdr, has_eth)) + goto out; + dst_hdr.nexthdr = IPPROTO_ICMPV6; + break; + case IPPROTO_TCP: + case IPPROTO_UDP: + update_l4_checksum(skb, &dst_hdr, iph, has_eth, true, false, false, NULL); + break; + default: + break; + } + + if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IPV6), 0)) + goto out; + + data = SKB_DATA(skb); + data_end = SKB_DATA_END(skb); + + if (!ensure_header(&ip6h, skb, &data, &data_end, L2_H_LEN(has_eth))) + goto out; + + if (has_eth) { + eth = data; + eth->h_proto = bpf_htons(ETH_P_IPV6); + } + + *ip6h = dst_hdr; + + switch (dst_hdr.nexthdr) { + case IPPROTO_ICMPV6: + __sync_fetch_and_add(&stats.egress_icmp, 1); + break; + case IPPROTO_TCP: + __sync_fetch_and_add(&stats.egress_tcp, 1); + break; + case IPPROTO_UDP: + __sync_fetch_and_add(&stats.egress_udp, 1); + break; + default: + __sync_fetch_and_add(&stats.egress_other, 1); + break; + } + + ret = bpf_redirect(skb->ifindex, 0); +out: + if (ret == TC_ACT_SHOT) + __sync_fetch_and_add(&stats.egress_dropped, 1); + return ret; +} + +static __always_inline __u16 +csum_fold_helper(__u32 csum) +{ + __u32 sum; + sum = (csum >> 16) + (csum & 0xffff); + sum += (sum >> 16); + return ~sum; +} + +static __always_inline bool +v6addr_equal(const struct in6_addr *a, const struct in6_addr *b) +{ + int i; + + for (i = 0; i < 4; i++) { + if (a->s6_addr32[i] != b->s6_addr32[i]) + return false; + } + return true; +} + +static __always_inline void +translate_ipv6_header(const struct ipv6hdr *ip6, struct iphdr *ip, __be32 saddr, __be32 daddr) +{ + *ip = (struct iphdr) { + .version = 4, + .ihl = 5, + .tos = ip6->priority << 4 | (ip6->flow_lbl[0] >> 4), + .frag_off = bpf_htons(1 << 14), + .ttl = ip6->hop_limit, + .protocol = ip6->nexthdr == IPPROTO_ICMPV6 ? IPPROTO_ICMP : ip6->nexthdr, + .saddr = saddr, + .daddr = daddr, + .tot_len = bpf_htons(bpf_ntohs(ip6->payload_len) + IP_H_LEN), + }; + + ip->check = csum_fold_helper(bpf_csum_diff((__be32 *) ip, 0, (__be32 *) ip, IP_H_LEN, 0)); +} + +static __always_inline int +translate_icmpv6_header(const struct icmp6hdr *icmp6, struct icmphdr *icmp) +{ + /* These translations are defined in RFC6145 section 5.2 */ + switch (icmp6->icmp6_type) { + case ICMPV6_ECHO_REQUEST: + icmp->type = ICMP_ECHO; + break; + case ICMPV6_ECHO_REPLY: + icmp->type = ICMP_ECHOREPLY; + break; + case ICMPV6_DEST_UNREACH: + icmp->type = ICMP_DEST_UNREACH; + switch (icmp6->icmp6_code) { + case ICMPV6_NOROUTE: + case ICMPV6_NOT_NEIGHBOUR: + case ICMPV6_ADDR_UNREACH: + icmp->code = ICMP_HOST_UNREACH; + break; + case ICMPV6_ADM_PROHIBITED: + icmp->code = ICMP_HOST_ANO; + break; + case ICMPV6_PORT_UNREACH: + icmp->code = ICMP_PORT_UNREACH; + break; + default: + return -1; + } + break; + case ICMPV6_PKT_TOOBIG: + { + __u32 mtu; + + icmp->type = ICMP_DEST_UNREACH; + icmp->code = ICMP_FRAG_NEEDED; + + mtu = bpf_ntohl(icmp6->icmp6_mtu) - 20; + if (mtu > 0xffff) + return -1; + icmp->un.frag.mtu = bpf_htons(mtu); + break; + } + case ICMPV6_TIME_EXCEED: + icmp->type = ICMP_TIME_EXCEEDED; + break; + case ICMPV6_PARAMPROB: + switch (icmp6->icmp6_code) { + case 0: + { + __u32 ptr; + + icmp->type = ICMP_PARAMETERPROB; + icmp->code = 0; + + ptr = bpf_ntohl(icmp6->icmp6_pointer); + /* Figure 6 in RFC6145 - using if statements b/c of + * range at the bottom + */ + if (ptr == 0 || ptr == 1) + icmp->un.reserved[0] = ptr; + else if (ptr == 4 || ptr == 5) + icmp->un.reserved[0] = 2; + else if (ptr == 6) + icmp->un.reserved[0] = 9; + else if (ptr == 7) + icmp->un.reserved[0] = 8; + else if (ptr >= 8 && ptr <= 23) + icmp->un.reserved[0] = 12; + else if (ptr >= 24 && ptr <= 39) + icmp->un.reserved[0] = 16; + else + return -1; + break; + } + case 1: + icmp->type = ICMP_DEST_UNREACH; + icmp->code = ICMP_PROT_UNREACH; + break; + default: + return -1; + } + break; + default: + return -1; + } + + return 0; +} + +static __always_inline int +rewrite_icmpv6_inner(struct __sk_buff *skb, __u32 *csum_diff, bool has_eth) +{ + void *data_end = SKB_DATA_END(skb); + void *data = SKB_DATA(skb); + struct icmphdr *icmp; + struct icmp6hdr *icmp6; + struct icmphdr icmp_buf; /* buffer for the new ICMPv4 header */ + struct icmp6hdr icmp6_buf; /* copy of the old ICMPv6 header */ + + /* + * icmp6: v + * ------------------------------------------------------------------------- + * | Ethernet | IPv6 | ICMPv6 | IPv6 | ICMPv6 | ... + * ------------------------------------------------------------------------- + */ + + if (!ensure_header(&icmp6, + skb, + &data, + &data_end, + L2_H_LEN(has_eth) + 2 * IP6_H_LEN + ICMP6_H_LEN)) + return -1; + + icmp6_buf = *icmp6; + icmp = (void *) icmp6; + icmp_buf = *icmp; + + if (translate_icmpv6_header(icmp6, &icmp_buf)) + return -1; + + *icmp = icmp_buf; + update_icmp_checksum(skb, + (struct ipv6hdr *) (data + L2_H_LEN(has_eth)), + &icmp6_buf, + icmp, + has_eth, + false, + true, + 0); + + if (csum_diff) { + data_end = SKB_DATA_END(skb); + data = SKB_DATA(skb); + + if (!ensure_header(&icmp, + skb, + &data, + &data_end, + L2_H_LEN(has_eth) + 2 * IP6_H_LEN + ICMP6_H_LEN)) + return -1; + + /* Compute the checksum difference between the old ICMPv6 header and the new ICMPv4 one */ + *csum_diff = + bpf_csum_diff((__be32 *) &icmp6_buf, ICMP6_H_LEN, (__be32 *) &icmp6_buf, 0, *csum_diff); + *csum_diff = bpf_csum_diff((__be32 *) icmp, 0, (__be32 *) icmp, ICMP_H_LEN, *csum_diff); + } + return 0; +} + +static __always_inline int +rewrite_ipv6_inner(struct __sk_buff *skb, struct iphdr *dst_hdr, __u32 *csum_diff, bool has_eth) +{ + void *data_end = SKB_DATA_END(skb); + void *data = SKB_DATA(skb); + struct ipv6hdr *ip6h; + __be32 addr4; + struct in6_addr subnet_v6; + + /* + * ip6h: v + * ---------------------------------------------------------------- + * | Ethernet | IPv6 | ICMPv6 | IPv6 | ... + * ---------------------------------------------------------------- + */ + + if (!ensure_header(&ip6h, skb, &data, &data_end, L2_H_LEN(has_eth) + IP6_H_LEN + ICMP6_H_LEN)) + return -1; + + if (!v6addr_equal(&ip6h->saddr, &config.local_v6)) + return -1; + if (!v6addr_to_v4(&ip6h->daddr, config.pref64_len, &addr4, &subnet_v6)) + return -1; + if (!v6addr_equal(&subnet_v6, &config.pref64)) + return -1; + + translate_ipv6_header(ip6h, dst_hdr, config.local_v4.s_addr, addr4); + + if (csum_diff) { + /* Checksum difference between the old IPv6 header and the new IPv4 one */ + *csum_diff = bpf_csum_diff((__be32 *) ip6h, IP6_H_LEN, (__be32 *) ip6h, 0, *csum_diff); + + *csum_diff = bpf_csum_diff((__be32 *) dst_hdr, 0, (__be32 *) dst_hdr, IP_H_LEN, *csum_diff); + } + + switch (dst_hdr->protocol) { + case IPPROTO_ICMP: + if (rewrite_icmpv6_inner(skb, csum_diff, has_eth)) + return -1; + break; + case IPPROTO_TCP: + case IPPROTO_UDP: + update_l4_checksum(skb, ip6h, dst_hdr, has_eth, false, true, false, csum_diff); + break; + default: + break; + } + + return 0; +} + +static __always_inline int +rewrite_icmpv6(struct __sk_buff *skb, int *out_length_diff, bool has_eth) +{ + void *data_end = SKB_DATA_END(skb); + void *data = SKB_DATA(skb); + struct iphdr *ip; + struct icmp6hdr *icmp6; + struct icmphdr *icmp; + struct icmphdr icmp_buf; /* buffer for the new ICMPv4 header */ + struct icmp6hdr icmp6_buf; /* copy of the old ICMPv6 header */ + struct iphdr ip_in_buf; /* buffer for the new inner IPv4 header */ + __u32 csum_diff = 0; + + /* + * icmp6: v + * --------------------------------------------- + * | Ethernet | IPv6 | ICMPv6 | ... + * --------------------------------------------- + */ + + if (!ensure_header(&icmp6, skb, &data, &data_end, L2_H_LEN(has_eth) + IP6_H_LEN)) + return -1; + + icmp6_buf = *icmp6; + icmp = (void *) icmp6; + icmp_buf = *icmp; + + if (translate_icmpv6_header(icmp6, &icmp_buf)) + return -1; + + if (icmp6->icmp6_type >= 128) { + /* ICMPv6 non-error message: only translate the header */ + *icmp = icmp_buf; + update_icmp_checksum(skb, + (struct ipv6hdr *) (data + L2_H_LEN(has_eth)), + &icmp6_buf, + icmp, + has_eth, + false, + false, + 0); + return 0; + } + + /* ICMPv6 error messages: we need to rewrite the headers in the inner packet. + * Track in csum_diff the incremental changes to the checksum for the ICMPv4 + * header. */ + + if (rewrite_ipv6_inner(skb, &ip_in_buf, &csum_diff, has_eth)) + return -1; + + /* The inner IP header shrinks from 40 (IPv6) to 20 (IPv4) bytes; we need to move + * the L4 header and payload. BPF programs don't have an easy way to move a variable + * amount of packet data; use bpf_skb_adjust_room() which can add or remove data + * inside a packet. It doesn't support arbitrary offsets, but we can use BPF_ADJ_ROOM_NET + * to remove the bytes just after the L3 header, and rewrite the ICMP and the inner + * IP headers. + */ + if (bpf_skb_adjust_room(skb, (int) IP_H_LEN - (int) IP6_H_LEN, BPF_ADJ_ROOM_NET, 0)) + return -1; + + *out_length_diff = (int) IP_H_LEN - (int) IP6_H_LEN; + + data_end = SKB_DATA_END(skb); + data = SKB_DATA(skb); + + if (!ensure_header(&ip, skb, &data, &data_end, L2_H_LEN(has_eth) + IP6_H_LEN + ICMP_H_LEN)) + return -1; + + icmp = data + L2_H_LEN(has_eth) + IP6_H_LEN; + + /* Rewrite the ICMPv6 header with the translated ICMPv4 one */ + *icmp = icmp_buf; + /* Rewrite the inner IPv6 header with the translated IPv4 one */ + *ip = ip_in_buf; + + /* Update the ICMPv4 checksum according to all the changes in headers */ + update_icmp_checksum(skb, + (struct ipv6hdr *) (data + L2_H_LEN(has_eth)), + &icmp6_buf, + icmp, + has_eth, + false, + false, + csum_diff); + + return 0; +} + +/* ipv6 traffic from the PLAT, to be translated into ipv4 and sent to an application */ +static __always_inline int +clat_handle_v6(struct __sk_buff *skb, bool has_eth) +{ + int ret = TC_ACT_OK; + void *data_end = SKB_DATA_END(skb); + void *data = SKB_DATA(skb); + struct ethhdr *eth; + struct ipv6hdr *ip6h; + struct iphdr *iph; + struct iphdr dst_hdr; + struct in6_addr subnet_v6; + __be32 addr4; + int length_diff = 0; + bool fragmented = false; + + if (!ensure_header(&ip6h, skb, &data, &data_end, L2_H_LEN(has_eth))) + goto out; + + if (has_eth) { + eth = data; + if (eth->h_proto != bpf_htons(ETH_P_IPV6)) + goto out; + } + + if (ip6h->version != 6) + goto out; + + if (!v6addr_equal(&ip6h->daddr, &config.local_v6)) + goto out; + if (!v6addr_to_v4(&ip6h->saddr, config.pref64_len, &addr4, &subnet_v6)) + goto out; + if (!v6addr_equal(&subnet_v6, &config.pref64)) { + struct icmp6hdr *icmp6; + + /* Follow draft-ietf-v6ops-icmpext-xlat-v6only-source-01: + * + * "Whenever a translator translates an ICMPv6 Destination Unreachable, + * ICMPv6 Time Exceeded or ICMPv6 Packet Too Big ([RFC4443]) to the + * corresponding ICMPv4 ([RFC0792]) message, and the IPv6 source + * address in the outermost IPv6 header is untranslatable, the + * translator SHOULD use the dummy IPv4 address (192.0.0.8) as the IPv4 + * source address for the translated packet." + */ + if (ip6h->nexthdr != IPPROTO_ICMPV6) + goto out; + + if (!ensure_header(&icmp6, skb, &data, &data_end, L2_H_LEN(has_eth) + IP6_H_LEN)) + goto out; + + ip6h = data + L2_H_LEN(has_eth); + + if (icmp6->icmp6_type != ICMPV6_DEST_UNREACH && icmp6->icmp6_type != ICMPV6_TIME_EXCEED + && icmp6->icmp6_type != ICMPV6_PKT_TOOBIG) + goto out; + + DBG("v6: icmpv6 type %u from native address %pI6c, translating src to dummy ipv4\n", + icmp6->icmp6_type, + &ip6h->saddr); + + addr4 = __cpu_to_be32(INADDR_DUMMY); + } + + /* At this point we know the packet needs translation. If we can't + * rewrite it, it should be dropped. + */ + ret = TC_ACT_SHOT; + + if (ip6h->nexthdr == IPPROTO_TCP || ip6h->nexthdr == IPPROTO_UDP + || ip6h->nexthdr == IPPROTO_ICMPV6) { + translate_ipv6_header(ip6h, &dst_hdr, addr4, config.local_v4.s_addr); + DBG("v6: incoming pkt from src %pI6c (%pI4)\n", &ip6h->saddr, &addr4); + } else if (ip6h->nexthdr == IPPROTO_FRAGMENT) { + struct ip6_frag *frag; + int tot_len; + __u16 offset; + + if (!ensure_header(&frag, skb, &data, &data_end, L2_H_LEN(has_eth) + IP6_H_LEN)) + goto out; + + ip6h = data + L2_H_LEN(has_eth); + + /* Translate into an IPv4 fragmented packet, RFC 6145 5.1.1 */ + + tot_len = bpf_ntohs(ip6h->payload_len) + IP_H_LEN - IP6_FRAG_H_LEN; + + offset = bpf_ntohs(frag->offset); + offset = ((offset & 1) << 13) | /* More Fragments flag */ + (offset >> 3); /* Offset in 8-octet units */ + + dst_hdr = (struct iphdr) { + .version = 4, + .ihl = 5, + .id = bpf_htons(bpf_ntohl(frag->identification) & 0xffff), + .tos = ip6h->priority << 4 | (ip6h->flow_lbl[0] >> 4), + .frag_off = bpf_htons(offset), + .ttl = ip6h->hop_limit, + .protocol = frag->nexthdr == IPPROTO_ICMPV6 ? IPPROTO_ICMP : frag->nexthdr, + .saddr = addr4, + .daddr = config.local_v4.s_addr, + .tot_len = bpf_htons(tot_len), + }; + + dst_hdr.check = csum_fold_helper( + bpf_csum_diff((__be32 *) &dst_hdr, 0, (__be32 *) &dst_hdr, IP_H_LEN, 0)); + + fragmented = true; + + DBG("v6: incoming fragmented pkt from src %pI6c (%pI4), id 0x%x\n", + &ip6h->saddr, + &addr4, + bpf_ntohs(dst_hdr.id)); + } else { + DBG("v6: pkt src/dst %pI6c/%pI6c has nexthdr %u, dropping\n", &ip6h->saddr, &ip6h->daddr); + goto out; + } + + switch (dst_hdr.protocol) { + case IPPROTO_ICMP: + /* We can't update the checksum of ICMP fragmented packets: ICMPv4 doesn't use + * a pseudo header, while the ICMPv6 pseudo-header includes the total payload + * length, which is not known when parsing the first fragment. This makes it + * impossible for a stateless translator to compute the checksum delta. TCP and + * UDP don't have this problem because both the v4 and v6 pseudo-headers include + * the total length. */ + if (fragmented) + goto out; + + if (rewrite_icmpv6(skb, &length_diff, has_eth)) + goto out; + break; + case IPPROTO_TCP: + case IPPROTO_UDP: + /* Update the L4 headers only for non-fragmented packets or for the first + * fragment, which contains the L4 header. */ + if (!fragmented || (bpf_ntohs(dst_hdr.frag_off) & 0x1FFF) == 0) { + update_l4_checksum(skb, ip6h, &dst_hdr, has_eth, false, false, fragmented, NULL); + } + break; + default: + break; + } + + /* rewrite_icmpv6() can change the payload length when it rewrites the content of + * an ICMPv6 error packet. Update the length and the checksum. */ + if (length_diff != 0) { + data = SKB_DATA(skb); + data_end = SKB_DATA_END(skb); + + if (!ensure_header(&ip6h, skb, &data, &data_end, L2_H_LEN(has_eth))) + goto out; + + dst_hdr.tot_len = bpf_htons(bpf_ntohs(ip6h->payload_len) + length_diff + IP_H_LEN); + + dst_hdr.check = 0; + dst_hdr.check = csum_fold_helper( + bpf_csum_diff((__be32 *) &dst_hdr, 0, (__be32 *) &dst_hdr, IP_H_LEN, 0)); + } + + if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IP), 0)) + goto out; + + if (fragmented) { + /* Remove the IPv6 fragment header */ + if (bpf_skb_adjust_room(skb, -(__s32) IP6_FRAG_H_LEN, BPF_ADJ_ROOM_NET, 0)) + goto out; + } + + data = SKB_DATA(skb); + data_end = SKB_DATA_END(skb); + + if (!ensure_header(&iph, skb, &data, &data_end, L2_H_LEN(has_eth))) + goto out; + + if (has_eth) { + eth = data; + eth->h_proto = bpf_htons(ETH_P_IP); + } + + *iph = dst_hdr; + + if (fragmented) + __sync_fetch_and_add(&stats.ingress_fragment, 1); + switch (dst_hdr.protocol) { + case IPPROTO_ICMP: + __sync_fetch_and_add(&stats.ingress_icmp, 1); + break; + case IPPROTO_TCP: + __sync_fetch_and_add(&stats.ingress_tcp, 1); + break; + case IPPROTO_UDP: + __sync_fetch_and_add(&stats.ingress_udp, 1); + break; + default: + __sync_fetch_and_add(&stats.ingress_other, 1); + break; + } + + ret = bpf_redirect(skb->ifindex, BPF_F_INGRESS); +out: + if (ret == TC_ACT_SHOT) + __sync_fetch_and_add(&stats.ingress_dropped, 1); + return ret; +} + +/* Use separate entry points for interfaces with and without an + * Ethernet header. Since all functions are now marked as inline, + * the compiler is able to replace the value of the parametric + * L2_H_LEN() macros with an immediate constant. This avoids + * pointer arithmetic which is forbidden because we don't run with + * CAP_PERFMON. The loader attaches the right program pair based + * on the interface type. */ +SEC("tcx/egress") +int +nm_clat_egress_eth(struct __sk_buff *skb) +{ + return clat_handle_v4(skb, true); +} + +SEC("tcx/egress") +int +nm_clat_egress_rawip(struct __sk_buff *skb) +{ + return clat_handle_v4(skb, false); +} + +SEC("tcx/ingress") +int +nm_clat_ingress_eth(struct __sk_buff *skb) +{ + return clat_handle_v6(skb, true); +} + +SEC("tcx/ingress") +int +nm_clat_ingress_rawip(struct __sk_buff *skb) +{ + return clat_handle_v6(skb, false); +} |