net/ipv4/tcp_bpf.c

Source file repositories/reference/linux-study-clean/net/ipv4/tcp_bpf.c

File Facts

System
Linux kernel
Corpus path
net/ipv4/tcp_bpf.c
Extension
.c
Size
18020 bytes
Lines
760
Domain
Networking Core
Bucket
Sockets, Protocols, Packet Path, And Network Policy
Inferred role
Networking Core: exported/initcall integration point
Status
integration implementation candidate

Why This File Exists

Networking stack implementation surface: socket APIs, protocol dispatch, packet flow, routing, filtering, and network namespaces.

Dependency Surface

Detected Declarations

Annotated Snippet

if (!__sk_rmem_schedule(sk, size, false)) {
			if (!copied)
				ret = -ENOMEM;
			break;
		}

		sk_mem_charge(sk, size);
		atomic_add(size, &sk->sk_rmem_alloc);
		sk_msg_xfer(tmp, msg, i, size);
		copied += size;
		if (sge->length)
			get_page(sk_msg_page(tmp, i));
		sk_msg_iter_var_next(i);
		tmp->sg.end = i;
		if (apply) {
			apply_bytes -= size;
			if (!apply_bytes) {
				if (sge->length)
					sk_msg_iter_var_prev(i);
				break;
			}
		}
	} while (i != msg->sg.end);

	if (!ret) {
		msg->sg.start = i;
		if (!sk_psock_queue_msg(psock, tmp))
			atomic_sub(copied, &sk->sk_rmem_alloc);
		sk_psock_data_ready(sk, psock);
	} else {
		sk_msg_free(sk, tmp);
		kfree(tmp);
	}

	release_sock(sk);
	return ret;
}

static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,
			int flags, bool uncharge)
{
	struct msghdr msghdr = {};
	bool apply = apply_bytes;
	struct scatterlist *sge;
	struct page *page;
	int size, ret = 0;
	u32 off;

	while (1) {
		struct bio_vec bvec;
		bool has_tx_ulp;

		sge = sk_msg_elem(msg, msg->sg.start);
		size = (apply && apply_bytes < sge->length) ?
			apply_bytes : sge->length;
		off  = sge->offset;
		page = sg_page(sge);

		tcp_rate_check_app_limited(sk);
retry:
		msghdr.msg_flags = flags | MSG_SPLICE_PAGES;
		has_tx_ulp = tls_sw_has_ctx_tx(sk);
		if (has_tx_ulp)
			msghdr.msg_flags |= MSG_SENDPAGE_NOPOLICY;

		if (size < sge->length && msg->sg.start != msg->sg.end)
			msghdr.msg_flags |= MSG_MORE;

		bvec_set_page(&bvec, page, size, off);
		iov_iter_bvec(&msghdr.msg_iter, ITER_SOURCE, &bvec, 1, size);
		ret = tcp_sendmsg_locked(sk, &msghdr, size);
		if (ret <= 0)
			return ret;

		if (apply)
			apply_bytes -= ret;
		msg->sg.size -= ret;
		sge->offset += ret;
		sge->length -= ret;
		if (uncharge)
			sk_mem_uncharge(sk, ret);
		if (ret != size) {
			size -= ret;
			off  += ret;
			goto retry;
		}
		if (!sge->length) {
			put_page(page);
			sk_msg_iter_next(msg, start);
			sg_init_table(sge, 1);

Annotation

Implementation Notes