/* SPDX-License-Identifier: GPL-2.0 */ /* Copyright 2021 Toke Høiland-Jørgensen */ /* Copyright 2025 Mary Strodl */ /* Copyright 2026 Beniamino Galvani */ /** * This is an implementation of a CLAT in eBPF. BPF is a different environment * than the rest of NetworkManager, and we don't have access to most of the * C standard library, so some things might look a little different from what * you're used to. * * Check out src/core/bpf/meson.build to see how this gets built. **/ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "clat.h" char _license[] SEC("license") = "GPL"; struct clat_config config; struct clat_stats stats; #ifdef DEBUG /* Note: when enabling debugging, you also need to add CAP_PERFMON * to the CapabilityBoundingSet of the NM systemd unit. The messages * will be printed to /sys/kernel/debug/tracing/trace_pipe */ #define DBG(fmt, ...) \ ({ \ char ____fmt[] = "clat: " fmt; \ bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \ }) #else #define DBG(fmt, ...) #endif /* Macros to read the sk_buff data* pointers, preventing the compiler * from generating a 32-bit register spill. */ #define SKB_ACCESS_MEMBER_32(_skb, member) \ ({ \ void *ptr; \ \ asm volatile("%0 = *(u32 *)(%1 + %2)" \ : "=r"(ptr) \ : "r"(_skb), "i"(offsetof(struct __sk_buff, member))); \ \ ptr; \ }) #define SKB_DATA(_skb) SKB_ACCESS_MEMBER_32(_skb, data) #define SKB_DATA_END(_skb) SKB_ACCESS_MEMBER_32(_skb, data_end) struct icmpv6_pseudo { struct in6_addr saddr; struct in6_addr daddr; __u32 len; __u8 padding[3]; __u8 nh; } __attribute__((packed)); struct ip6_frag { __u8 nexthdr; __u8 reserved; __u16 offset; __u32 identification; } __attribute__((packed)); #define L2_H_LEN(has_eth) (has_eth ? sizeof(struct ethhdr) : 0) #define IP_H_LEN (sizeof(struct iphdr)) #define IP6_H_LEN (sizeof(struct ipv6hdr)) #define IP6_FRAG_H_LEN (sizeof(struct ip6_frag)) #define ICMP_H_LEN (sizeof(struct icmphdr)) #define ICMP6_H_LEN (sizeof(struct icmp6hdr)) #define ensure_header(header, skb, data, data_end, offset) \ _ensure_header((void **) header, (skb), (data), (data_end), sizeof(**(header)), (offset)) /* * Verifies that the header at offset @offset and with size @size can * be accessed, and assigns the pointer to @header. In case the data * is not available, the function tries to pull it. Note that all packet * pointers must be refreshed after calling this function. */ static __always_inline bool _ensure_header(void **header, struct __sk_buff *skb, void **data, void **data_end, unsigned size, unsigned offset) { if (*data + offset + size > *data_end) { bpf_skb_pull_data(skb, offset + size); *data = SKB_DATA(skb); *data_end = SKB_DATA_END(skb); } if (*data + offset + size > *data_end) return false; *header = *data + offset; return true; } /* This function must be declared as inline because the BPF calling * convention only supports up to 5 function arguments. */ static __always_inline void update_l4_checksum(struct __sk_buff *skb, struct ipv6hdr *ip6h, struct iphdr *iph, bool has_eth, bool v4to6, bool is_inner, bool is_v6_fragment, __u32 *csum_diff) { int flags = BPF_F_PSEUDO_HDR; __u16 offset; __u32 csum; int ip_type; if (v4to6) { void *from_ptr = &iph->saddr; void *to_ptr = &ip6h->saddr; csum = bpf_csum_diff(from_ptr, 2 * sizeof(__u32), to_ptr, 2 * sizeof(struct in6_addr), 0); offset = L2_H_LEN(has_eth) + IP_H_LEN; ip_type = ip6h->nexthdr; } else { void *from_ptr = &ip6h->saddr; void *to_ptr = &iph->saddr; csum = bpf_csum_diff(from_ptr, 2 * sizeof(struct in6_addr), to_ptr, 2 * sizeof(__u32), 0); offset = L2_H_LEN(has_eth) + IP6_H_LEN; ip_type = iph->protocol; if (is_inner) { offset = offset + ICMP6_H_LEN + IP6_H_LEN; } } if (is_v6_fragment) { offset += IP6_FRAG_H_LEN; } switch (ip_type) { case IPPROTO_TCP: offset += offsetof(struct tcphdr, check); break; case IPPROTO_UDP: offset += offsetof(struct udphdr, check); flags |= BPF_F_MARK_MANGLED_0; break; default: return; } bpf_l4_csum_replace(skb, offset, 0, csum, flags); if (csum_diff) { *csum_diff = bpf_csum_diff((__be32 *) &csum, sizeof(csum), 0, 0, *csum_diff); } } static __always_inline void update_icmp_checksum(struct __sk_buff *skb, const struct ipv6hdr *ip6h, void *icmp_before, void *icmp_after, bool has_eth, bool v4to6, bool is_inner, __u32 seed) { struct icmpv6_pseudo ph = {.nh = IPPROTO_ICMPV6, .len = ip6h->payload_len}; __u16 h_before; __u16 h_after; __u16 offset; __u32 csum; __u32 u_before; __u32 u_after; __builtin_memcpy(&ph.saddr, &ip6h->saddr, sizeof(struct in6_addr)); __builtin_memcpy(&ph.daddr, &ip6h->daddr, sizeof(struct in6_addr)); /* Do checksum update in two passes: first compute the incremental * checksum update of the ICMPv6 pseudo header, update the checksum * using bpf_l4_csum_replace(), and then do a separate update for the * ICMP type and code (which is two consecutive bytes, so cast them to * u16). The bpf_csum_diff() helper can be used to compute the * incremental update of the full block, whereas the * bpf_l4_csum_replace() helper can do the two-byte diff and update by * itself. */ csum = bpf_csum_diff((__be32 *) &ph, v4to6 ? 0 : sizeof(ph), (__be32 *) &ph, v4to6 ? sizeof(ph) : 0, seed); if (v4to6) { offset = L2_H_LEN(has_eth) + IP_H_LEN + 2; } else { offset = L2_H_LEN(has_eth) + IP6_H_LEN + 2; if (is_inner) offset += ICMP6_H_LEN + IP6_H_LEN; } /* first two bytes of ICMP header, type and code */ h_before = *(__u16 *) icmp_before; h_after = *(__u16 *) icmp_after; /* last four bytes of ICMP header, the data union */ u_before = *(__u32 *) (icmp_before + 4); u_after = *(__u32 *) (icmp_after + 4); bpf_l4_csum_replace(skb, offset, 0, csum, BPF_F_PSEUDO_HDR); bpf_l4_csum_replace(skb, offset, h_before, h_after, 2); if (u_before != u_after) bpf_l4_csum_replace(skb, offset, u_before, u_after, 4); } static __always_inline int rewrite_icmp(struct __sk_buff *skb, const struct ipv6hdr *ip6h, bool has_eth) { void *data_end = SKB_DATA_END(skb); void *data = SKB_DATA(skb); struct icmphdr icmp_buf; /* copy of the old ICMPv4 header */ struct icmp6hdr icmp6_buf; /* buffer for the new ICMPv6 header */ struct icmphdr *icmp; struct icmp6hdr *icmp6; __u32 mtu; if (!ensure_header(&icmp, skb, &data, &data_end, L2_H_LEN(has_eth) + IP_H_LEN)) return -1; icmp_buf = *icmp; icmp6 = (void *) icmp; icmp6_buf = *icmp6; /* These translations are defined in RFC6145 section 4.2 */ switch (icmp->type) { case ICMP_ECHO: icmp6_buf.icmp6_type = ICMPV6_ECHO_REQUEST; break; case ICMP_ECHOREPLY: icmp6_buf.icmp6_type = ICMPV6_ECHO_REPLY; break; case ICMP_DEST_UNREACH: icmp6_buf.icmp6_type = ICMPV6_DEST_UNREACH; switch (icmp->code) { case ICMP_NET_UNREACH: case ICMP_HOST_UNREACH: case ICMP_SR_FAILED: case ICMP_NET_UNKNOWN: case ICMP_HOST_UNKNOWN: case ICMP_HOST_ISOLATED: case ICMP_NET_UNR_TOS: case ICMP_HOST_UNR_TOS: icmp6_buf.icmp6_code = ICMPV6_NOROUTE; break; case ICMP_PROT_UNREACH: icmp6_buf.icmp6_type = ICMPV6_PARAMPROB; icmp6_buf.icmp6_code = ICMPV6_UNK_NEXTHDR; icmp6_buf.icmp6_pointer = bpf_htonl(offsetof(struct ipv6hdr, nexthdr)); break; case ICMP_PORT_UNREACH: icmp6_buf.icmp6_code = ICMPV6_PORT_UNREACH; break; case ICMP_FRAG_NEEDED: icmp6_buf.icmp6_type = ICMPV6_PKT_TOOBIG; icmp6_buf.icmp6_code = 0; mtu = bpf_ntohs(icmp->un.frag.mtu) + 20; /* RFC6145 section 6, "second approach" - should not be * necessary, but might as well do this */ if (mtu < 1280) mtu = 1280; icmp6_buf.icmp6_mtu = bpf_htonl(mtu); break; case ICMP_NET_ANO: case ICMP_HOST_ANO: case ICMP_PKT_FILTERED: case ICMP_PREC_CUTOFF: icmp6_buf.icmp6_code = ICMPV6_ADM_PROHIBITED; break; default: return -1; } break; case ICMP_PARAMETERPROB: if (icmp->code == 1) return -1; icmp6_buf.icmp6_type = ICMPV6_PARAMPROB; icmp6_buf.icmp6_code = ICMPV6_HDR_FIELD; /* The pointer field not defined in the Linux header. This * translation is from Figure 3 of RFC6145. */ switch (icmp->un.reserved[0]) { case 0: /* version/IHL */ icmp6_buf.icmp6_pointer = 0; break; case 1: /* Type of Service */ icmp6_buf.icmp6_pointer = bpf_htonl(1); break; case 2: /* Total length */ case 3: icmp6_buf.icmp6_pointer = bpf_htonl(4); break; case 8: /* Time to Live */ icmp6_buf.icmp6_pointer = bpf_htonl(7); break; case 9: /* Protocol */ icmp6_buf.icmp6_pointer = bpf_htonl(6); break; case 12: /* Source address */ case 13: case 14: case 15: icmp6_buf.icmp6_pointer = bpf_htonl(8); break; case 16: /* Destination address */ case 17: case 18: case 19: icmp6_buf.icmp6_pointer = bpf_htonl(24); break; default: return -1; } break; default: return -1; } *icmp6 = icmp6_buf; update_icmp_checksum(skb, ip6h, &icmp_buf, icmp6, has_eth, true, false, 0); /* FIXME: also need to rewrite IP header embedded in ICMP error */ return 0; } /* * Convert an IPv4 address to the corresponding "IPv4-Embedded IPv6 Address" * according to RFC 6052 2.2. * * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ * |PL| 0-------------32--40--48--56--64--72--80--88--96--104---------| * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ * |32| prefix |v4(32) | u | suffix | * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ * |40| prefix |v4(24) | u |(8)| suffix | * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ * |48| prefix |v4(16) | u | (16) | suffix | * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ * |56| prefix |(8)| u | v4(24) | suffix | * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ * |64| prefix | u | v4(32) | suffix | * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ * |96| prefix | v4(32) | * +--+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ * */ static __always_inline bool v4addr_to_v6(__be32 addr4, struct in6_addr *addr6, const struct in6_addr *pref64, int pref64_len) { union { __be32 a32; __u8 a8[4]; } u; u.a32 = addr4; addr6->s6_addr32[0] = 0; addr6->s6_addr32[1] = 0; addr6->s6_addr32[2] = 0; addr6->s6_addr32[3] = 0; switch (pref64_len) { case 96: addr6->s6_addr32[0] = pref64->s6_addr32[0]; addr6->s6_addr32[1] = pref64->s6_addr32[1]; addr6->s6_addr32[2] = pref64->s6_addr32[2]; addr6->s6_addr32[3] = addr4; break; case 64: addr6->s6_addr32[0] = pref64->s6_addr32[0]; addr6->s6_addr32[1] = pref64->s6_addr32[1]; addr6->s6_addr[9] = u.a8[0]; addr6->s6_addr[10] = u.a8[1]; addr6->s6_addr[11] = u.a8[2]; addr6->s6_addr[12] = u.a8[3]; break; case 56: addr6->s6_addr32[0] = pref64->s6_addr32[0]; addr6->s6_addr32[1] = pref64->s6_addr32[1]; addr6->s6_addr[7] = u.a8[0]; addr6->s6_addr[9] = u.a8[1]; addr6->s6_addr[10] = u.a8[2]; addr6->s6_addr[11] = u.a8[3]; break; case 48: addr6->s6_addr32[0] = pref64->s6_addr32[0]; addr6->s6_addr16[2] = pref64->s6_addr16[2]; addr6->s6_addr[6] = u.a8[0]; addr6->s6_addr[7] = u.a8[1]; addr6->s6_addr[9] = u.a8[2]; addr6->s6_addr[10] = u.a8[3]; break; case 40: addr6->s6_addr32[0] = pref64->s6_addr32[0]; addr6->s6_addr[4] = pref64->s6_addr[4]; addr6->s6_addr[5] = u.a8[0]; addr6->s6_addr[6] = u.a8[1]; addr6->s6_addr[7] = u.a8[2]; addr6->s6_addr[9] = u.a8[3]; break; case 32: addr6->s6_addr32[0] = pref64->s6_addr32[0]; addr6->s6_addr32[1] = addr4; break; default: return false; } return true; } /* * Extract the IPv4 address @addr4 and the NAT64 prefix @pref64 from an IPv6 address, * given the known prefix length @pref64_len. See the table above. */ static __always_inline bool v6addr_to_v4(const struct in6_addr *addr6, int pref64_len, __be32 *addr4, struct in6_addr *pref64) { union { __be32 a32; __u8 a8[4]; } u; pref64->s6_addr32[0] = 0; pref64->s6_addr32[1] = 0; pref64->s6_addr32[2] = 0; pref64->s6_addr32[3] = 0; switch (pref64_len) { case 96: u.a32 = addr6->s6_addr32[3]; pref64->s6_addr32[0] = addr6->s6_addr32[0]; pref64->s6_addr32[1] = addr6->s6_addr32[1]; pref64->s6_addr32[2] = addr6->s6_addr32[2]; break; case 64: u.a8[0] = addr6->s6_addr[9]; u.a8[1] = addr6->s6_addr[10]; u.a8[2] = addr6->s6_addr[11]; u.a8[3] = addr6->s6_addr[12]; pref64->s6_addr32[0] = addr6->s6_addr32[0]; pref64->s6_addr32[1] = addr6->s6_addr32[1]; break; case 56: u.a8[0] = addr6->s6_addr[7]; u.a8[1] = addr6->s6_addr[9]; u.a8[2] = addr6->s6_addr[10]; u.a8[3] = addr6->s6_addr[11]; pref64->s6_addr32[0] = addr6->s6_addr32[0]; pref64->s6_addr32[1] = addr6->s6_addr32[1]; pref64->s6_addr[7] = 0; break; case 48: u.a8[0] = addr6->s6_addr[6]; u.a8[1] = addr6->s6_addr[7]; u.a8[2] = addr6->s6_addr[9]; u.a8[3] = addr6->s6_addr[10]; pref64->s6_addr32[0] = addr6->s6_addr32[0]; pref64->s6_addr32[1] = addr6->s6_addr32[1]; pref64->s6_addr16[3] = 0; break; case 40: u.a8[0] = addr6->s6_addr[5]; u.a8[1] = addr6->s6_addr[6]; u.a8[2] = addr6->s6_addr[7]; u.a8[3] = addr6->s6_addr[9]; pref64->s6_addr32[0] = addr6->s6_addr32[0]; pref64->s6_addr32[1] = addr6->s6_addr32[1]; pref64->s6_addr16[3] = 0; pref64->s6_addr[5] = 0; break; case 32: u.a32 = addr6->s6_addr32[1]; pref64->s6_addr32[0] = addr6->s6_addr32[0]; break; default: return false; } *addr4 = u.a32; return true; } /* ipv4 traffic in from application on this device, needs to be translated to v6 and sent to PLAT */ static __always_inline int clat_handle_v4(struct __sk_buff *skb, bool has_eth) { int ret = TC_ACT_OK; void *data_end = SKB_DATA_END(skb); void *data = SKB_DATA(skb); struct ipv6hdr *ip6h; struct ipv6hdr dst_hdr = { .version = 6, }; struct iphdr *iph; struct ethhdr *eth; if (!ensure_header(&iph, skb, &data, &data_end, L2_H_LEN(has_eth))) goto out; if (has_eth) { eth = data; if (eth->h_proto != bpf_htons(ETH_P_IP)) goto out; } if (iph->version != 4) goto out; if (iph->saddr != config.local_v4.s_addr) goto out; /* At this point we know the packet needs translation. If we can't * rewrite it, it should be dropped. */ ret = TC_ACT_SHOT; /* we don't bother dealing with IP options or fragmented packets. The * latter are identified by the 'frag_off' field having a value (either * the MF bit, or the fragment offset, or both). However, this field also * contains the "don't fragment" (DF) bit, which we ignore, so mask that * out. The DF is the second-most-significant bit (as bit 0 is * reserved). */ if (iph->ihl != 5 || (iph->frag_off & ~bpf_htons(1 << 14))) { DBG("v4: pkt src/dst %pI4/%pI4 has IP options or is fragmented, dropping\n", &iph->saddr, &iph->daddr); goto out; } if (!v4addr_to_v6(iph->daddr, &dst_hdr.daddr, &config.pref64, config.pref64_len)) goto out; dst_hdr.saddr = config.local_v6; dst_hdr.nexthdr = iph->protocol; dst_hdr.hop_limit = iph->ttl; /* weird definition in ipv6hdr */ dst_hdr.priority = (iph->tos & 0x70) >> 4; dst_hdr.flow_lbl[0] = iph->tos << 4; dst_hdr.payload_len = bpf_htons(bpf_ntohs(iph->tot_len) - IP_H_LEN); DBG("v4: outgoing pkt to dst %pI4 (%pI6c)\n", &iph->daddr, &dst_hdr.daddr); switch (dst_hdr.nexthdr) { case IPPROTO_ICMP: if (rewrite_icmp(skb, &dst_hdr, has_eth)) goto out; dst_hdr.nexthdr = IPPROTO_ICMPV6; break; case IPPROTO_TCP: case IPPROTO_UDP: update_l4_checksum(skb, &dst_hdr, iph, has_eth, true, false, false, NULL); break; default: break; } if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IPV6), 0)) goto out; data = SKB_DATA(skb); data_end = SKB_DATA_END(skb); if (!ensure_header(&ip6h, skb, &data, &data_end, L2_H_LEN(has_eth))) goto out; if (has_eth) { eth = data; eth->h_proto = bpf_htons(ETH_P_IPV6); } *ip6h = dst_hdr; switch (dst_hdr.nexthdr) { case IPPROTO_ICMPV6: __sync_fetch_and_add(&stats.egress_icmp, 1); break; case IPPROTO_TCP: __sync_fetch_and_add(&stats.egress_tcp, 1); break; case IPPROTO_UDP: __sync_fetch_and_add(&stats.egress_udp, 1); break; default: __sync_fetch_and_add(&stats.egress_other, 1); break; } ret = bpf_redirect(skb->ifindex, 0); out: if (ret == TC_ACT_SHOT) __sync_fetch_and_add(&stats.egress_dropped, 1); return ret; } static __always_inline __u16 csum_fold_helper(__u32 csum) { __u32 sum; sum = (csum >> 16) + (csum & 0xffff); sum += (sum >> 16); return ~sum; } static __always_inline bool v6addr_equal(const struct in6_addr *a, const struct in6_addr *b) { int i; for (i = 0; i < 4; i++) { if (a->s6_addr32[i] != b->s6_addr32[i]) return false; } return true; } static __always_inline void translate_ipv6_header(const struct ipv6hdr *ip6, struct iphdr *ip, __be32 saddr, __be32 daddr) { *ip = (struct iphdr) { .version = 4, .ihl = 5, .tos = ip6->priority << 4 | (ip6->flow_lbl[0] >> 4), .frag_off = bpf_htons(1 << 14), .ttl = ip6->hop_limit, .protocol = ip6->nexthdr == IPPROTO_ICMPV6 ? IPPROTO_ICMP : ip6->nexthdr, .saddr = saddr, .daddr = daddr, .tot_len = bpf_htons(bpf_ntohs(ip6->payload_len) + IP_H_LEN), }; ip->check = csum_fold_helper(bpf_csum_diff((__be32 *) ip, 0, (__be32 *) ip, IP_H_LEN, 0)); } static __always_inline int translate_icmpv6_header(const struct icmp6hdr *icmp6, struct icmphdr *icmp) { /* These translations are defined in RFC6145 section 5.2 */ switch (icmp6->icmp6_type) { case ICMPV6_ECHO_REQUEST: icmp->type = ICMP_ECHO; break; case ICMPV6_ECHO_REPLY: icmp->type = ICMP_ECHOREPLY; break; case ICMPV6_DEST_UNREACH: icmp->type = ICMP_DEST_UNREACH; switch (icmp6->icmp6_code) { case ICMPV6_NOROUTE: case ICMPV6_NOT_NEIGHBOUR: case ICMPV6_ADDR_UNREACH: icmp->code = ICMP_HOST_UNREACH; break; case ICMPV6_ADM_PROHIBITED: icmp->code = ICMP_HOST_ANO; break; case ICMPV6_PORT_UNREACH: icmp->code = ICMP_PORT_UNREACH; break; default: return -1; } break; case ICMPV6_PKT_TOOBIG: { __u32 mtu; icmp->type = ICMP_DEST_UNREACH; icmp->code = ICMP_FRAG_NEEDED; mtu = bpf_ntohl(icmp6->icmp6_mtu) - 20; if (mtu > 0xffff) return -1; icmp->un.frag.mtu = bpf_htons(mtu); break; } case ICMPV6_TIME_EXCEED: icmp->type = ICMP_TIME_EXCEEDED; break; case ICMPV6_PARAMPROB: switch (icmp6->icmp6_code) { case 0: { __u32 ptr; icmp->type = ICMP_PARAMETERPROB; icmp->code = 0; ptr = bpf_ntohl(icmp6->icmp6_pointer); /* Figure 6 in RFC6145 - using if statements b/c of * range at the bottom */ if (ptr == 0 || ptr == 1) icmp->un.reserved[0] = ptr; else if (ptr == 4 || ptr == 5) icmp->un.reserved[0] = 2; else if (ptr == 6) icmp->un.reserved[0] = 9; else if (ptr == 7) icmp->un.reserved[0] = 8; else if (ptr >= 8 && ptr <= 23) icmp->un.reserved[0] = 12; else if (ptr >= 24 && ptr <= 39) icmp->un.reserved[0] = 16; else return -1; break; } case 1: icmp->type = ICMP_DEST_UNREACH; icmp->code = ICMP_PROT_UNREACH; break; default: return -1; } break; default: return -1; } return 0; } static __always_inline int rewrite_icmpv6_inner(struct __sk_buff *skb, __u32 *csum_diff, bool has_eth) { void *data_end = SKB_DATA_END(skb); void *data = SKB_DATA(skb); struct icmphdr *icmp; struct icmp6hdr *icmp6; struct icmphdr icmp_buf; /* buffer for the new ICMPv4 header */ struct icmp6hdr icmp6_buf; /* copy of the old ICMPv6 header */ /* * icmp6: v * ------------------------------------------------------------------------- * | Ethernet | IPv6 | ICMPv6 | IPv6 | ICMPv6 | ... * ------------------------------------------------------------------------- */ if (!ensure_header(&icmp6, skb, &data, &data_end, L2_H_LEN(has_eth) + 2 * IP6_H_LEN + ICMP6_H_LEN)) return -1; icmp6_buf = *icmp6; icmp = (void *) icmp6; icmp_buf = *icmp; if (translate_icmpv6_header(icmp6, &icmp_buf)) return -1; *icmp = icmp_buf; update_icmp_checksum(skb, (struct ipv6hdr *) (data + L2_H_LEN(has_eth)), &icmp6_buf, icmp, has_eth, false, true, 0); if (csum_diff) { data_end = SKB_DATA_END(skb); data = SKB_DATA(skb); if (!ensure_header(&icmp, skb, &data, &data_end, L2_H_LEN(has_eth) + 2 * IP6_H_LEN + ICMP6_H_LEN)) return -1; /* Compute the checksum difference between the old ICMPv6 header and the new ICMPv4 one */ *csum_diff = bpf_csum_diff((__be32 *) &icmp6_buf, ICMP6_H_LEN, (__be32 *) &icmp6_buf, 0, *csum_diff); *csum_diff = bpf_csum_diff((__be32 *) icmp, 0, (__be32 *) icmp, ICMP_H_LEN, *csum_diff); } return 0; } static __always_inline int rewrite_ipv6_inner(struct __sk_buff *skb, struct iphdr *dst_hdr, __u32 *csum_diff, bool has_eth) { void *data_end = SKB_DATA_END(skb); void *data = SKB_DATA(skb); struct ipv6hdr *ip6h; __be32 addr4; struct in6_addr subnet_v6; /* * ip6h: v * ---------------------------------------------------------------- * | Ethernet | IPv6 | ICMPv6 | IPv6 | ... * ---------------------------------------------------------------- */ if (!ensure_header(&ip6h, skb, &data, &data_end, L2_H_LEN(has_eth) + IP6_H_LEN + ICMP6_H_LEN)) return -1; if (!v6addr_equal(&ip6h->saddr, &config.local_v6)) return -1; if (!v6addr_to_v4(&ip6h->daddr, config.pref64_len, &addr4, &subnet_v6)) return -1; if (!v6addr_equal(&subnet_v6, &config.pref64)) return -1; translate_ipv6_header(ip6h, dst_hdr, config.local_v4.s_addr, addr4); if (csum_diff) { /* Checksum difference between the old IPv6 header and the new IPv4 one */ *csum_diff = bpf_csum_diff((__be32 *) ip6h, IP6_H_LEN, (__be32 *) ip6h, 0, *csum_diff); *csum_diff = bpf_csum_diff((__be32 *) dst_hdr, 0, (__be32 *) dst_hdr, IP_H_LEN, *csum_diff); } switch (dst_hdr->protocol) { case IPPROTO_ICMP: if (rewrite_icmpv6_inner(skb, csum_diff, has_eth)) return -1; break; case IPPROTO_TCP: case IPPROTO_UDP: update_l4_checksum(skb, ip6h, dst_hdr, has_eth, false, true, false, csum_diff); break; default: break; } return 0; } static __always_inline int rewrite_icmpv6(struct __sk_buff *skb, int *out_length_diff, bool has_eth) { void *data_end = SKB_DATA_END(skb); void *data = SKB_DATA(skb); struct iphdr *ip; struct icmp6hdr *icmp6; struct icmphdr *icmp; struct icmphdr icmp_buf; /* buffer for the new ICMPv4 header */ struct icmp6hdr icmp6_buf; /* copy of the old ICMPv6 header */ struct iphdr ip_in_buf; /* buffer for the new inner IPv4 header */ __u32 csum_diff = 0; /* * icmp6: v * --------------------------------------------- * | Ethernet | IPv6 | ICMPv6 | ... * --------------------------------------------- */ if (!ensure_header(&icmp6, skb, &data, &data_end, L2_H_LEN(has_eth) + IP6_H_LEN)) return -1; icmp6_buf = *icmp6; icmp = (void *) icmp6; icmp_buf = *icmp; if (translate_icmpv6_header(icmp6, &icmp_buf)) return -1; if (icmp6->icmp6_type >= 128) { /* ICMPv6 non-error message: only translate the header */ *icmp = icmp_buf; update_icmp_checksum(skb, (struct ipv6hdr *) (data + L2_H_LEN(has_eth)), &icmp6_buf, icmp, has_eth, false, false, 0); return 0; } /* ICMPv6 error messages: we need to rewrite the headers in the inner packet. * Track in csum_diff the incremental changes to the checksum for the ICMPv4 * header. */ if (rewrite_ipv6_inner(skb, &ip_in_buf, &csum_diff, has_eth)) return -1; /* The inner IP header shrinks from 40 (IPv6) to 20 (IPv4) bytes; we need to move * the L4 header and payload. BPF programs don't have an easy way to move a variable * amount of packet data; use bpf_skb_adjust_room() which can add or remove data * inside a packet. It doesn't support arbitrary offsets, but we can use BPF_ADJ_ROOM_NET * to remove the bytes just after the L3 header, and rewrite the ICMP and the inner * IP headers. */ if (bpf_skb_adjust_room(skb, (int) IP_H_LEN - (int) IP6_H_LEN, BPF_ADJ_ROOM_NET, 0)) return -1; *out_length_diff = (int) IP_H_LEN - (int) IP6_H_LEN; data_end = SKB_DATA_END(skb); data = SKB_DATA(skb); if (!ensure_header(&ip, skb, &data, &data_end, L2_H_LEN(has_eth) + IP6_H_LEN + ICMP_H_LEN)) return -1; icmp = data + L2_H_LEN(has_eth) + IP6_H_LEN; /* Rewrite the ICMPv6 header with the translated ICMPv4 one */ *icmp = icmp_buf; /* Rewrite the inner IPv6 header with the translated IPv4 one */ *ip = ip_in_buf; /* Update the ICMPv4 checksum according to all the changes in headers */ update_icmp_checksum(skb, (struct ipv6hdr *) (data + L2_H_LEN(has_eth)), &icmp6_buf, icmp, has_eth, false, false, csum_diff); return 0; } /* ipv6 traffic from the PLAT, to be translated into ipv4 and sent to an application */ static __always_inline int clat_handle_v6(struct __sk_buff *skb, bool has_eth) { int ret = TC_ACT_OK; void *data_end = SKB_DATA_END(skb); void *data = SKB_DATA(skb); struct ethhdr *eth; struct ipv6hdr *ip6h; struct iphdr *iph; struct iphdr dst_hdr; struct in6_addr subnet_v6; __be32 addr4; int length_diff = 0; bool fragmented = false; if (!ensure_header(&ip6h, skb, &data, &data_end, L2_H_LEN(has_eth))) goto out; if (has_eth) { eth = data; if (eth->h_proto != bpf_htons(ETH_P_IPV6)) goto out; } if (ip6h->version != 6) goto out; if (!v6addr_equal(&ip6h->daddr, &config.local_v6)) goto out; if (!v6addr_to_v4(&ip6h->saddr, config.pref64_len, &addr4, &subnet_v6)) goto out; if (!v6addr_equal(&subnet_v6, &config.pref64)) { struct icmp6hdr *icmp6; /* Follow draft-ietf-v6ops-icmpext-xlat-v6only-source-01: * * "Whenever a translator translates an ICMPv6 Destination Unreachable, * ICMPv6 Time Exceeded or ICMPv6 Packet Too Big ([RFC4443]) to the * corresponding ICMPv4 ([RFC0792]) message, and the IPv6 source * address in the outermost IPv6 header is untranslatable, the * translator SHOULD use the dummy IPv4 address (192.0.0.8) as the IPv4 * source address for the translated packet." */ if (ip6h->nexthdr != IPPROTO_ICMPV6) goto out; if (!ensure_header(&icmp6, skb, &data, &data_end, L2_H_LEN(has_eth) + IP6_H_LEN)) goto out; ip6h = data + L2_H_LEN(has_eth); if (icmp6->icmp6_type != ICMPV6_DEST_UNREACH && icmp6->icmp6_type != ICMPV6_TIME_EXCEED && icmp6->icmp6_type != ICMPV6_PKT_TOOBIG) goto out; DBG("v6: icmpv6 type %u from native address %pI6c, translating src to dummy ipv4\n", icmp6->icmp6_type, &ip6h->saddr); addr4 = __cpu_to_be32(INADDR_DUMMY); } /* At this point we know the packet needs translation. If we can't * rewrite it, it should be dropped. */ ret = TC_ACT_SHOT; if (ip6h->nexthdr == IPPROTO_TCP || ip6h->nexthdr == IPPROTO_UDP || ip6h->nexthdr == IPPROTO_ICMPV6) { translate_ipv6_header(ip6h, &dst_hdr, addr4, config.local_v4.s_addr); DBG("v6: incoming pkt from src %pI6c (%pI4)\n", &ip6h->saddr, &addr4); } else if (ip6h->nexthdr == IPPROTO_FRAGMENT) { struct ip6_frag *frag; int tot_len; __u16 offset; if (!ensure_header(&frag, skb, &data, &data_end, L2_H_LEN(has_eth) + IP6_H_LEN)) goto out; ip6h = data + L2_H_LEN(has_eth); /* Translate into an IPv4 fragmented packet, RFC 6145 5.1.1 */ tot_len = bpf_ntohs(ip6h->payload_len) + IP_H_LEN - IP6_FRAG_H_LEN; offset = bpf_ntohs(frag->offset); offset = ((offset & 1) << 13) | /* More Fragments flag */ (offset >> 3); /* Offset in 8-octet units */ dst_hdr = (struct iphdr) { .version = 4, .ihl = 5, .id = bpf_htons(bpf_ntohl(frag->identification) & 0xffff), .tos = ip6h->priority << 4 | (ip6h->flow_lbl[0] >> 4), .frag_off = bpf_htons(offset), .ttl = ip6h->hop_limit, .protocol = frag->nexthdr == IPPROTO_ICMPV6 ? IPPROTO_ICMP : frag->nexthdr, .saddr = addr4, .daddr = config.local_v4.s_addr, .tot_len = bpf_htons(tot_len), }; dst_hdr.check = csum_fold_helper( bpf_csum_diff((__be32 *) &dst_hdr, 0, (__be32 *) &dst_hdr, IP_H_LEN, 0)); fragmented = true; DBG("v6: incoming fragmented pkt from src %pI6c (%pI4), id 0x%x\n", &ip6h->saddr, &addr4, bpf_ntohs(dst_hdr.id)); } else { DBG("v6: pkt src/dst %pI6c/%pI6c has nexthdr %u, dropping\n", &ip6h->saddr, &ip6h->daddr); goto out; } switch (dst_hdr.protocol) { case IPPROTO_ICMP: /* We can't update the checksum of ICMP fragmented packets: ICMPv4 doesn't use * a pseudo header, while the ICMPv6 pseudo-header includes the total payload * length, which is not known when parsing the first fragment. This makes it * impossible for a stateless translator to compute the checksum delta. TCP and * UDP don't have this problem because both the v4 and v6 pseudo-headers include * the total length. */ if (fragmented) goto out; if (rewrite_icmpv6(skb, &length_diff, has_eth)) goto out; break; case IPPROTO_TCP: case IPPROTO_UDP: /* Update the L4 headers only for non-fragmented packets or for the first * fragment, which contains the L4 header. */ if (!fragmented || (bpf_ntohs(dst_hdr.frag_off) & 0x1FFF) == 0) { update_l4_checksum(skb, ip6h, &dst_hdr, has_eth, false, false, fragmented, NULL); } break; default: break; } /* rewrite_icmpv6() can change the payload length when it rewrites the content of * an ICMPv6 error packet. Update the length and the checksum. */ if (length_diff != 0) { data = SKB_DATA(skb); data_end = SKB_DATA_END(skb); if (!ensure_header(&ip6h, skb, &data, &data_end, L2_H_LEN(has_eth))) goto out; dst_hdr.tot_len = bpf_htons(bpf_ntohs(ip6h->payload_len) + length_diff + IP_H_LEN); dst_hdr.check = 0; dst_hdr.check = csum_fold_helper( bpf_csum_diff((__be32 *) &dst_hdr, 0, (__be32 *) &dst_hdr, IP_H_LEN, 0)); } if (bpf_skb_change_proto(skb, bpf_htons(ETH_P_IP), 0)) goto out; if (fragmented) { /* Remove the IPv6 fragment header */ if (bpf_skb_adjust_room(skb, -(__s32) IP6_FRAG_H_LEN, BPF_ADJ_ROOM_NET, 0)) goto out; } data = SKB_DATA(skb); data_end = SKB_DATA_END(skb); if (!ensure_header(&iph, skb, &data, &data_end, L2_H_LEN(has_eth))) goto out; if (has_eth) { eth = data; eth->h_proto = bpf_htons(ETH_P_IP); } *iph = dst_hdr; if (fragmented) __sync_fetch_and_add(&stats.ingress_fragment, 1); switch (dst_hdr.protocol) { case IPPROTO_ICMP: __sync_fetch_and_add(&stats.ingress_icmp, 1); break; case IPPROTO_TCP: __sync_fetch_and_add(&stats.ingress_tcp, 1); break; case IPPROTO_UDP: __sync_fetch_and_add(&stats.ingress_udp, 1); break; default: __sync_fetch_and_add(&stats.ingress_other, 1); break; } ret = bpf_redirect(skb->ifindex, BPF_F_INGRESS); out: if (ret == TC_ACT_SHOT) __sync_fetch_and_add(&stats.ingress_dropped, 1); return ret; } /* Use separate entry points for interfaces with and without an * Ethernet header. Since all functions are now marked as inline, * the compiler is able to replace the value of the parametric * L2_H_LEN() macros with an immediate constant. This avoids * pointer arithmetic which is forbidden because we don't run with * CAP_PERFMON. The loader attaches the right program pair based * on the interface type. */ SEC("tcx/egress") int nm_clat_egress_eth(struct __sk_buff *skb) { return clat_handle_v4(skb, true); } SEC("tcx/egress") int nm_clat_egress_rawip(struct __sk_buff *skb) { return clat_handle_v4(skb, false); } SEC("tcx/ingress") int nm_clat_ingress_eth(struct __sk_buff *skb) { return clat_handle_v6(skb, true); } SEC("tcx/ingress") int nm_clat_ingress_rawip(struct __sk_buff *skb) { return clat_handle_v6(skb, false); }