summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/arm64/include/asm/memory.h5
-rw-r--r--arch/arm64/net/bpf_jit_comp.c30
-rw-r--r--arch/mips/include/asm/uasm.h1
-rw-r--r--arch/mips/include/uapi/asm/inst.h3
-rw-r--r--arch/mips/mm/uasm-micromips.c1
-rw-r--r--arch/mips/mm/uasm-mips.c1
-rw-r--r--arch/mips/mm/uasm.c9
-rw-r--r--arch/mips/net/ebpf_jit.c4
-rw-r--r--arch/powerpc/include/asm/ppc-opcode.h2
-rw-r--r--arch/powerpc/net/bpf_jit.h4
-rw-r--r--arch/powerpc/net/bpf_jit_comp64.c6
-rw-r--r--arch/s390/net/bpf_jit_comp.c12
-rw-r--r--arch/x86/net/bpf_jit_comp.c2
-rw-r--r--drivers/media/rc/bpf-lirc.c24
-rw-r--r--drivers/net/ethernet/netronome/nfp/bpf/jit.c45
-rw-r--r--include/linux/bpf.h21
-rw-r--r--include/linux/bpf_verifier.h1
-rw-r--r--include/linux/btf.h1
-rw-r--r--include/linux/filter.h7
-rw-r--r--include/uapi/linux/bpf.h138
-rw-r--r--kernel/bpf/btf.c2
-rw-r--r--kernel/bpf/core.c186
-rw-r--r--kernel/bpf/syscall.c130
-rw-r--r--kernel/bpf/verifier.c239
-rw-r--r--net/bpf/test_run.c15
-rw-r--r--net/core/filter.c16
-rw-r--r--samples/bpf/bpf_load.c4
-rw-r--r--samples/bpf/xdp1_user.c27
-rw-r--r--tools/bpf/bpftool/Documentation/bpftool-prog.rst31
-rw-r--r--tools/bpf/bpftool/bash-completion/bpftool86
-rw-r--r--tools/bpf/bpftool/btf_dumper.c64
-rw-r--r--tools/bpf/bpftool/common.c7
-rw-r--r--tools/bpf/bpftool/jit_disasm.c34
-rw-r--r--tools/bpf/bpftool/main.h49
-rw-r--r--tools/bpf/bpftool/map.c50
-rw-r--r--tools/bpf/bpftool/prog.c149
-rw-r--r--tools/bpf/bpftool/tracelog.c157
-rw-r--r--tools/bpf/bpftool/xlated_dumper.c34
-rw-r--r--tools/bpf/bpftool/xlated_dumper.h7
-rw-r--r--tools/include/uapi/linux/bpf.h63
-rw-r--r--tools/lib/bpf/Build2
-rw-r--r--tools/lib/bpf/README.rst2
-rw-r--r--tools/lib/bpf/bpf.c124
-rw-r--r--tools/lib/bpf/bpf.h24
-rw-r--r--tools/lib/bpf/bpf_prog_linfo.c253
-rw-r--r--tools/lib/bpf/btf.c344
-rw-r--r--tools/lib/bpf/btf.h25
-rw-r--r--tools/lib/bpf/libbpf.c159
-rw-r--r--tools/lib/bpf/libbpf.h13
-rw-r--r--tools/lib/bpf/libbpf.map5
-rw-r--r--tools/testing/selftests/bpf/bpf_helpers.h2
-rw-r--r--tools/testing/selftests/bpf/test_align.c4
-rw-r--r--tools/testing/selftests/bpf/test_btf.c820
-rwxr-xr-xtools/testing/selftests/bpf/test_lirc_mode2.sh3
-rw-r--r--tools/testing/selftests/bpf/test_lirc_mode2_kern.c3
-rw-r--r--tools/testing/selftests/bpf/test_lirc_mode2_user.c65
-rw-r--r--tools/testing/selftests/bpf/test_progs.c55
-rw-r--r--tools/testing/selftests/bpf/test_verifier.c176
58 files changed, 3070 insertions, 676 deletions
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index b96442960aea..ee20fc63899c 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -62,8 +62,11 @@
#define PAGE_OFFSET (UL(0xffffffffffffffff) - \
(UL(1) << (VA_BITS - 1)) + 1)
#define KIMAGE_VADDR (MODULES_END)
+#define BPF_JIT_REGION_START (VA_START + KASAN_SHADOW_SIZE)
+#define BPF_JIT_REGION_SIZE (SZ_128M)
+#define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE)
#define MODULES_END (MODULES_VADDR + MODULES_VSIZE)
-#define MODULES_VADDR (VA_START + KASAN_SHADOW_SIZE)
+#define MODULES_VADDR (BPF_JIT_REGION_END)
#define MODULES_VSIZE (SZ_128M)
#define VMEMMAP_START (PAGE_OFFSET - VMEMMAP_SIZE)
#define PCI_IO_END (VMEMMAP_START - SZ_2M)
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 89198017e8e6..0a7371a86139 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -134,10 +134,9 @@ static inline void emit_a64_mov_i64(const int reg, const u64 val,
}
/*
- * This is an unoptimized 64 immediate emission used for BPF to BPF call
- * addresses. It will always do a full 64 bit decomposition as otherwise
- * more complexity in the last extra pass is required since we previously
- * reserved 4 instructions for the address.
+ * Kernel addresses in the vmalloc space use at most 48 bits, and the
+ * remaining bits are guaranteed to be 0x1. So we can compose the address
+ * with a fixed length movn/movk/movk sequence.
*/
static inline void emit_addr_mov_i64(const int reg, const u64 val,
struct jit_ctx *ctx)
@@ -145,8 +144,8 @@ static inline void emit_addr_mov_i64(const int reg, const u64 val,
u64 tmp = val;
int shift = 0;
- emit(A64_MOVZ(1, reg, tmp & 0xffff, shift), ctx);
- for (;shift < 48;) {
+ emit(A64_MOVN(1, reg, ~tmp & 0xffff, shift), ctx);
+ while (shift < 32) {
tmp >>= 16;
shift += 16;
emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx);
@@ -634,11 +633,7 @@ emit_cond_jmp:
&func_addr, &func_addr_fixed);
if (ret < 0)
return ret;
- if (func_addr_fixed)
- /* We can use optimized emission here. */
- emit_a64_mov_i64(tmp, func_addr, ctx);
- else
- emit_addr_mov_i64(tmp, func_addr, ctx);
+ emit_addr_mov_i64(tmp, func_addr, ctx);
emit(A64_BLR(tmp), ctx);
emit(A64_MOV(1, r0, A64_R(0)), ctx);
break;
@@ -948,3 +943,16 @@ out:
tmp : orig_prog);
return prog;
}
+
+void *bpf_jit_alloc_exec(unsigned long size)
+{
+ return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
+ BPF_JIT_REGION_END, GFP_KERNEL,
+ PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE,
+ __builtin_return_address(0));
+}
+
+void bpf_jit_free_exec(void *addr)
+{
+ return vfree(addr);
+}
diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h
index 59dae37f6b8d..b1990dd75f27 100644
--- a/arch/mips/include/asm/uasm.h
+++ b/arch/mips/include/asm/uasm.h
@@ -157,6 +157,7 @@ Ip_u2u1s3(_slti);
Ip_u2u1s3(_sltiu);
Ip_u3u1u2(_sltu);
Ip_u2u1u3(_sra);
+Ip_u3u2u1(_srav);
Ip_u2u1u3(_srl);
Ip_u3u2u1(_srlv);
Ip_u3u1u2(_subu);
diff --git a/arch/mips/include/uapi/asm/inst.h b/arch/mips/include/uapi/asm/inst.h
index c05dcf5ab414..40fbb5dd66df 100644
--- a/arch/mips/include/uapi/asm/inst.h
+++ b/arch/mips/include/uapi/asm/inst.h
@@ -369,8 +369,9 @@ enum mm_32a_minor_op {
mm_ext_op = 0x02c,
mm_pool32axf_op = 0x03c,
mm_srl32_op = 0x040,
+ mm_srlv32_op = 0x050,
mm_sra_op = 0x080,
- mm_srlv32_op = 0x090,
+ mm_srav_op = 0x090,
mm_rotr_op = 0x0c0,
mm_lwxs_op = 0x118,
mm_addu32_op = 0x150,
diff --git a/arch/mips/mm/uasm-micromips.c b/arch/mips/mm/uasm-micromips.c
index 24e5b0d06899..75ef90486fe6 100644
--- a/arch/mips/mm/uasm-micromips.c
+++ b/arch/mips/mm/uasm-micromips.c
@@ -104,6 +104,7 @@ static const struct insn insn_table_MM[insn_invalid] = {
[insn_sltiu] = {M(mm_sltiu32_op, 0, 0, 0, 0, 0), RT | RS | SIMM},
[insn_sltu] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_sltu_op), RT | RS | RD},
[insn_sra] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_sra_op), RT | RS | RD},
+ [insn_srav] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srav_op), RT | RS | RD},
[insn_srl] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srl32_op), RT | RS | RD},
[insn_srlv] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srlv32_op), RT | RS | RD},
[insn_rotr] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_rotr_op), RT | RS | RD},
diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c
index 60ceb93c71a0..6abe40fc413d 100644
--- a/arch/mips/mm/uasm-mips.c
+++ b/arch/mips/mm/uasm-mips.c
@@ -171,6 +171,7 @@ static const struct insn insn_table[insn_invalid] = {
[insn_sltiu] = {M(sltiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
[insn_sltu] = {M(spec_op, 0, 0, 0, 0, sltu_op), RS | RT | RD},
[insn_sra] = {M(spec_op, 0, 0, 0, 0, sra_op), RT | RD | RE},
+ [insn_srav] = {M(spec_op, 0, 0, 0, 0, srav_op), RS | RT | RD},
[insn_srl] = {M(spec_op, 0, 0, 0, 0, srl_op), RT | RD | RE},
[insn_srlv] = {M(spec_op, 0, 0, 0, 0, srlv_op), RS | RT | RD},
[insn_subu] = {M(spec_op, 0, 0, 0, 0, subu_op), RS | RT | RD},
diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c
index 57570c0649b4..45b6264ff308 100644
--- a/arch/mips/mm/uasm.c
+++ b/arch/mips/mm/uasm.c
@@ -61,10 +61,10 @@ enum opcode {
insn_mthc0, insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_nor,
insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb,
insn_sc, insn_scd, insn_sd, insn_sh, insn_sll, insn_sllv,
- insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, insn_srl,
- insn_srlv, insn_subu, insn_sw, insn_sync, insn_syscall, insn_tlbp,
- insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait, insn_wsbh, insn_xor,
- insn_xori, insn_yield,
+ insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, insn_srav,
+ insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync, insn_syscall,
+ insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait, insn_wsbh,
+ insn_xor, insn_xori, insn_yield,
insn_invalid /* insn_invalid must be last */
};
@@ -353,6 +353,7 @@ I_u2u1s3(_slti)
I_u2u1s3(_sltiu)
I_u3u1u2(_sltu)
I_u2u1u3(_sra)
+I_u3u2u1(_srav)
I_u2u1u3(_srl)
I_u3u2u1(_srlv)
I_u2u1u3(_rotr)
diff --git a/arch/mips/net/ebpf_jit.c b/arch/mips/net/ebpf_jit.c
index aeb7b1b0f202..b16710a8a9e7 100644
--- a/arch/mips/net/ebpf_jit.c
+++ b/arch/mips/net/ebpf_jit.c
@@ -854,6 +854,7 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
case BPF_ALU | BPF_MOD | BPF_X: /* ALU_REG */
case BPF_ALU | BPF_LSH | BPF_X: /* ALU_REG */
case BPF_ALU | BPF_RSH | BPF_X: /* ALU_REG */
+ case BPF_ALU | BPF_ARSH | BPF_X: /* ALU_REG */
src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
if (src < 0 || dst < 0)
@@ -913,6 +914,9 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
case BPF_RSH:
emit_instr(ctx, srlv, dst, dst, src);
break;
+ case BPF_ARSH:
+ emit_instr(ctx, srav, dst, dst, src);
+ break;
default:
pr_err("ALU_REG NOT HANDLED\n");
return -EINVAL;
diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h
index a6e9e314c707..901459226eca 100644
--- a/arch/powerpc/include/asm/ppc-opcode.h
+++ b/arch/powerpc/include/asm/ppc-opcode.h
@@ -342,6 +342,8 @@
#define PPC_INST_SLW 0x7c000030
#define PPC_INST_SLD 0x7c000036
#define PPC_INST_SRW 0x7c000430
+#define PPC_INST_SRAW 0x7c000630
+#define PPC_INST_SRAWI 0x7c000670
#define PPC_INST_SRD 0x7c000436
#define PPC_INST_SRAD 0x7c000634
#define PPC_INST_SRADI 0x7c000674
diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h
index 47fc6660845d..c2d5192ed64f 100644
--- a/arch/powerpc/net/bpf_jit.h
+++ b/arch/powerpc/net/bpf_jit.h
@@ -152,6 +152,10 @@
___PPC_RS(a) | ___PPC_RB(s))
#define PPC_SRW(d, a, s) EMIT(PPC_INST_SRW | ___PPC_RA(d) | \
___PPC_RS(a) | ___PPC_RB(s))
+#define PPC_SRAW(d, a, s) EMIT(PPC_INST_SRAW | ___PPC_RA(d) | \
+ ___PPC_RS(a) | ___PPC_RB(s))
+#define PPC_SRAWI(d, a, i) EMIT(PPC_INST_SRAWI | ___PPC_RA(d) | \
+ ___PPC_RS(a) | __PPC_SH(i))
#define PPC_SRD(d, a, s) EMIT(PPC_INST_SRD | ___PPC_RA(d) | \
___PPC_RS(a) | ___PPC_RB(s))
#define PPC_SRAD(d, a, s) EMIT(PPC_INST_SRAD | ___PPC_RA(d) | \
diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index 9393e231cbc2..7ce57657d3b8 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -529,9 +529,15 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
if (imm != 0)
PPC_SRDI(dst_reg, dst_reg, imm);
break;
+ case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
+ PPC_SRAW(dst_reg, dst_reg, src_reg);
+ goto bpf_alu32_trunc;
case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
PPC_SRAD(dst_reg, dst_reg, src_reg);
break;
+ case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
+ PPC_SRAWI(dst_reg, dst_reg, imm);
+ goto bpf_alu32_trunc;
case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
if (imm != 0)
PPC_SRADI(dst_reg, dst_reg, imm);
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index d7052cbe984f..3ff758eeb71d 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -821,10 +821,22 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
/*
* BPF_ARSH
*/
+ case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= src */
+ /* sra %dst,%dst,0(%src) */
+ EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0);
+ EMIT_ZERO(dst_reg);
+ break;
case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */
/* srag %dst,%dst,0(%src) */
EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0);
break;
+ case BPF_ALU | BPF_ARSH | BPF_K: /* ((s32) dst >> imm */
+ if (imm == 0)
+ break;
+ /* sra %dst,imm(%r0) */
+ EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm);
+ EMIT_ZERO(dst_reg);
+ break;
case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */
if (imm == 0)
break;
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 2580cd2e98b1..5542303c43d9 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1181,6 +1181,8 @@ out_image:
}
if (!image || !prog->is_func || extra_pass) {
+ if (image)
+ bpf_prog_fill_jited_linfo(prog, addrs);
out_addrs:
kfree(addrs);
kfree(jit_data);
diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c
index 8b97fd1f0cea..390a722e6211 100644
--- a/drivers/media/rc/bpf-lirc.c
+++ b/drivers/media/rc/bpf-lirc.c
@@ -59,6 +59,28 @@ static const struct bpf_func_proto rc_keydown_proto = {
.arg4_type = ARG_ANYTHING,
};
+BPF_CALL_3(bpf_rc_pointer_rel, u32*, sample, s32, rel_x, s32, rel_y)
+{
+ struct ir_raw_event_ctrl *ctrl;
+
+ ctrl = container_of(sample, struct ir_raw_event_ctrl, bpf_sample);
+
+ input_report_rel(ctrl->dev->input_dev, REL_X, rel_x);
+ input_report_rel(ctrl->dev->input_dev, REL_Y, rel_y);
+ input_sync(ctrl->dev->input_dev);
+
+ return 0;
+}
+
+static const struct bpf_func_proto rc_pointer_rel_proto = {
+ .func = bpf_rc_pointer_rel,
+ .gpl_only = true,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_ANYTHING,
+ .arg3_type = ARG_ANYTHING,
+};
+
static const struct bpf_func_proto *
lirc_mode2_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
@@ -67,6 +89,8 @@ lirc_mode2_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
return &rc_repeat_proto;
case BPF_FUNC_rc_keydown:
return &rc_keydown_proto;
+ case BPF_FUNC_rc_pointer_rel:
+ return &rc_pointer_rel_proto;
case BPF_FUNC_map_lookup_elem:
return &bpf_map_lookup_elem_proto;
case BPF_FUNC_map_update_elem:
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
index 97d33bb4d84d..662cbc21d909 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
@@ -2382,6 +2382,49 @@ static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
return 0;
}
+static int __ashr_imm(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
+{
+ /* Set signedness bit (MSB of result). */
+ emit_alu(nfp_prog, reg_none(), reg_a(dst), ALU_OP_OR, reg_imm(0));
+ emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, reg_b(dst),
+ SHF_SC_R_SHF, shift_amt);
+ wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+
+ return 0;
+}
+
+static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+ const struct bpf_insn *insn = &meta->insn;
+ u64 umin, umax;
+ u8 dst, src;
+
+ dst = insn->dst_reg * 2;
+ umin = meta->umin_src;
+ umax = meta->umax_src;
+ if (umin == umax)
+ return __ashr_imm(nfp_prog, dst, umin);
+
+ src = insn->src_reg * 2;
+ /* NOTE: the first insn will set both indirect shift amount (source A)
+ * and signedness bit (MSB of result).
+ */
+ emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst));
+ emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
+ reg_b(dst), SHF_SC_R_SHF);
+ wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+
+ return 0;
+}
+
+static int ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+ const struct bpf_insn *insn = &meta->insn;
+ u8 dst = insn->dst_reg * 2;
+
+ return __ashr_imm(nfp_prog, dst, insn->imm);
+}
+
static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
{
const struct bpf_insn *insn = &meta->insn;
@@ -3286,6 +3329,8 @@ static const instr_cb_t instr_cb[256] = {
[BPF_ALU | BPF_DIV | BPF_K] = div_imm,
[BPF_ALU | BPF_NEG] = neg_reg,
[BPF_ALU | BPF_LSH | BPF_K] = shl_imm,
+ [BPF_ALU | BPF_ARSH | BPF_X] = ashr_reg,
+ [BPF_ALU | BPF_ARSH | BPF_K] = ashr_imm,
[BPF_ALU | BPF_END | BPF_X] = end_reg32,
[BPF_LD | BPF_IMM | BPF_DW] = imm_ld8,
[BPF_LD | BPF_ABS | BPF_B] = data_ld1,
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index e82b7039fc66..0c992b86eb2c 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -319,7 +319,28 @@ struct bpf_prog_aux {
struct bpf_prog_offload *offload;
struct btf *btf;
struct bpf_func_info *func_info;
+ /* bpf_line_info loaded from userspace. linfo->insn_off
+ * has the xlated insn offset.
+ * Both the main and sub prog share the same linfo.
+ * The subprog can access its first linfo by
+ * using the linfo_idx.
+ */
+ struct bpf_line_info *linfo;
+ /* jited_linfo is the jited addr of the linfo. It has a
+ * one to one mapping to linfo:
+ * jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
+ * Both the main and sub prog share the same jited_linfo.
+ * The subprog can access its first jited_linfo by
+ * using the linfo_idx.
+ */
+ void **jited_linfo;
u32 func_info_cnt;
+ u32 nr_linfo;
+ /* subprog can use linfo_idx to access its first linfo and
+ * jited_linfo.
+ * main prog always has linfo_idx == 0
+ */
+ u32 linfo_idx;
union {
struct work_struct work;
struct rcu_head rcu;
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 11f5df1092d9..c736945be7c5 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -203,6 +203,7 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
struct bpf_subprog_info {
u32 start; /* insn idx of function entry point */
+ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */
u16 stack_depth; /* max. stack depth used by this function */
};
diff --git a/include/linux/btf.h b/include/linux/btf.h
index 8c2199b5d250..b98405a56383 100644
--- a/include/linux/btf.h
+++ b/include/linux/btf.h
@@ -46,6 +46,7 @@ void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
struct seq_file *m);
int btf_get_fd_by_id(u32 id);
u32 btf_id(const struct btf *btf);
+bool btf_name_offset_valid(const struct btf *btf, u32 offset);
#ifdef CONFIG_BPF_SYSCALL
const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 5a26a7caa98f..537e9e7c6e6f 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -725,6 +725,13 @@ void bpf_prog_free(struct bpf_prog *fp);
bool bpf_opcode_in_insntable(u8 code);
+void bpf_prog_free_linfo(struct bpf_prog *prog);
+void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
+ const u32 *insn_to_jit_off);
+int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog);
+void bpf_prog_free_jited_linfo(struct bpf_prog *prog);
+void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog);
+
struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);
struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
gfp_t gfp_extra_flags);
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index ec8b40ff386e..92e962ba0c47 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -232,6 +232,20 @@ enum bpf_attach_type {
*/
#define BPF_F_STRICT_ALIGNMENT (1U << 0)
+/* If BPF_F_ANY_ALIGNMENT is used in BPF_PROF_LOAD command, the
+ * verifier will allow any alignment whatsoever. On platforms
+ * with strict alignment requirements for loads ands stores (such
+ * as sparc and mips) the verifier validates that all loads and
+ * stores provably follow this requirement. This flag turns that
+ * checking and enforcement off.
+ *
+ * It is mostly used for testing when we want to validate the
+ * context and memory access aspects of the verifier, but because
+ * of an unaligned access the alignment check would trigger before
+ * the one we are interested in.
+ */
+#define BPF_F_ANY_ALIGNMENT (1U << 1)
+
/* when bpf_ldimm64->src_reg == BPF_PSEUDO_MAP_FD, bpf_ldimm64->imm == fd */
#define BPF_PSEUDO_MAP_FD 1
@@ -342,6 +356,9 @@ union bpf_attr {
__u32 func_info_rec_size; /* userspace bpf_func_info size */
__aligned_u64 func_info; /* func info */
__u32 func_info_cnt; /* number of bpf_func_info records */
+ __u32 line_info_rec_size; /* userspace bpf_line_info size */
+ __aligned_u64 line_info; /* line info */
+ __u32 line_info_cnt; /* number of bpf_line_info records */
};
struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -360,8 +377,11 @@ union bpf_attr {
struct { /* anonymous struct used by BPF_PROG_TEST_RUN command */
__u32 prog_fd;
__u32 retval;
- __u32 data_size_in;
- __u32 data_size_out;
+ __u32 data_size_in; /* input: len of data_in */
+ __u32 data_size_out; /* input/output: len of data_out
+ * returns ENOSPC if data_out
+ * is too small.
+ */
__aligned_u64 data_in;
__aligned_u64 data_out;
__u32 repeat;
@@ -482,18 +502,6 @@ union bpf_attr {
* Return
* 0 on success, or a negative error in case of failure.
*
- * int bpf_map_pop_elem(struct bpf_map *map, void *value)
- * Description
- * Pop an element from *map*.
- * Return
- * 0 on success, or a negative error in case of failure.
- *
- * int bpf_map_peek_elem(struct bpf_map *map, void *value)
- * Description
- * Get an element from *map* without removing it.
- * Return
- * 0 on success, or a negative error in case of failure.
- *
* int bpf_probe_read(void *dst, u32 size, const void *src)
* Description
* For tracing programs, safely attempt to read *size* bytes from
@@ -1917,9 +1925,9 @@ union bpf_attr {
* is set to metric from route (IPv4/IPv6 only), and ifindex
* is set to the device index of the nexthop from the FIB lookup.
*
- * *plen* argument is the size of the passed in struct.
- * *flags* argument can be a combination of one or more of the
- * following values:
+ * *plen* argument is the size of the passed in struct.
+ * *flags* argument can be a combination of one or more of the
+ * following values:
*
* **BPF_FIB_LOOKUP_DIRECT**
* Do a direct table lookup vs full lookup using FIB
@@ -1928,9 +1936,9 @@ union bpf_attr {
* Perform lookup from an egress perspective (default is
* ingress).
*
- * *ctx* is either **struct xdp_md** for XDP programs or
- * **struct sk_buff** tc cls_act programs.
- * Return
+ * *ctx* is either **struct xdp_md** for XDP programs or
+ * **struct sk_buff** tc cls_act programs.
+ * Return
* * < 0 if any input argument is invalid
* * 0 on success (packet is forwarded, nexthop neighbor exists)
* * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the
@@ -2075,8 +2083,8 @@ union bpf_attr {
* translated to a keycode using the rc keymap, and reported as
* an input key down event. After a period a key up event is
* generated. This period can be extended by calling either
- * **bpf_rc_keydown** () again with the same values, or calling
- * **bpf_rc_repeat** ().
+ * **bpf_rc_keydown**\ () again with the same values, or calling
+ * **bpf_rc_repeat**\ ().
*
* Some protocols include a toggle bit, in case the button was
* released and pressed again between consecutive scancodes.
@@ -2159,21 +2167,22 @@ union bpf_attr {
* The *flags* meaning is specific for each map type,
* and has to be 0 for cgroup local storage.
*
- * Depending on the bpf program type, a local storage area
- * can be shared between multiple instances of the bpf program,
+ * Depending on the BPF program type, a local storage area
+ * can be shared between multiple instances of the BPF program,
* running simultaneously.
*
* A user should care about the synchronization by himself.
- * For example, by using the BPF_STX_XADD instruction to alter
+ * For example, by using the **BPF_STX_XADD** instruction to alter
* the shared data.
* Return
- * Pointer to the local storage area.
+ * A pointer to the local storage area.
*
* int bpf_sk_select_reuseport(struct sk_reuseport_md *reuse, struct bpf_map *map, void *key, u64 flags)
* Description
- * Select a SO_REUSEPORT sk from a BPF_MAP_TYPE_REUSEPORT_ARRAY map
- * It checks the selected sk is matching the incoming
- * request in the skb.
+ * Select a **SO_REUSEPORT** socket from a
+ * **BPF_MAP_TYPE_REUSEPORT_ARRAY** *map*.
+ * It checks the selected socket is matching the incoming
+ * request in the socket buffer.
* Return
* 0 on success, or a negative error in case of failure.
*
@@ -2181,7 +2190,7 @@ union bpf_attr {
* Description
* Look for TCP socket matching *tuple*, optionally in a child
* network namespace *netns*. The return value must be checked,
- * and if non-NULL, released via **bpf_sk_release**\ ().
+ * and if non-**NULL**, released via **bpf_sk_release**\ ().
*
* The *ctx* should point to the context of the program, such as
* the skb or socket (depending on the hook in use). This is used
@@ -2217,7 +2226,7 @@ union bpf_attr {
* Description
* Look for UDP socket matching *tuple*, optionally in a child
* network namespace *netns*. The return value must be checked,
- * and if non-NULL, released via **bpf_sk_release**\ ().
+ * and if non-**NULL**, released via **bpf_sk_release**\ ().
*
* The *ctx* should point to the context of the program, such as
* the skb or socket (depending on the hook in use). This is used
@@ -2249,42 +2258,67 @@ union bpf_attr {
* For sockets with reuseport option, the *struct bpf_sock*
* result is from reuse->socks[] using the hash of the tuple.
*
- * int bpf_sk_release(struct bpf_sock *sk)
+ * int bpf_sk_release(struct bpf_sock *sock)
* Description
- * Release the reference held by *sock*. *sock* must be a non-NULL
- * pointer that was returned from bpf_sk_lookup_xxx\ ().
+ * Release the reference held by *sock*. *sock* must be a
+ * non-**NULL** pointer that was returned from
+ * **bpf_sk_lookup_xxx**\ ().
* Return
* 0 on success, or a negative error in case of failure.
*
+ * int bpf_map_pop_elem(struct bpf_map *map, void *value)
+ * Description
+ * Pop an element from *map*.
+ * Return
+ * 0 on success, or a negative error in case of failure.
+ *
+ * int bpf_map_peek_elem(struct bpf_map *map, void *value)
+ * Description
+ * Get an element from *map* without removing it.
+ * Return
+ * 0 on success, or a negative error in case of failure.
+ *
* int bpf_msg_push_data(struct sk_buff *skb, u32 start, u32 len, u64 flags)
* Description
- * For socket policies, insert *len* bytes into msg at offset
+ * For socket policies, insert *len* bytes into *msg* at offset
* *start*.
*
* If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a
- * *msg* it may want to insert metadata or options into the msg.
+ * *msg* it may want to insert metadata or options into the *msg*.
* This can later be read and used by any of the lower layer BPF
* hooks.
*
* This helper may fail if under memory pressure (a malloc
* fails) in these cases BPF programs will get an appropriate
* error and BPF programs will need to handle them.
- *
* Return
* 0 on success, or a negative error in case of failure.
*
* int bpf_msg_pop_data(struct sk_msg_buff *msg, u32 start, u32 pop, u64 flags)
- * Description
+ * Description
* Will remove *pop* bytes from a *msg* starting at byte *start*.
* This may result in **ENOMEM** errors under certain situations if
* an allocation and copy are required due to a full ring buffer.
* However, the helper will try to avoid doing the allocation
* if possible. Other errors can occur if input parameters are
- * invalid either due to *start* byte not being valid part of msg
+ * invalid either due to *start* byte not being valid part of *msg*
* payload and/or *pop* value being to large.
+ * Return
+ * 0 on success, or a negative error in case of failure.
+ *
+ * int bpf_rc_pointer_rel(void *ctx, s32 rel_x, s32 rel_y)
+ * Description
+ * This helper is used in programs implementing IR decoding, to
+ * report a successfully decoded pointer movement.
*
+ * The *ctx* should point to the lirc sample as passed into
+ * the program.
+ *
+ * This helper is only available is the kernel was compiled with
+ * the **CONFIG_BPF_LIRC_MODE2** configuration option set to
+ * "**y**".
* Return
- * 0 on success, or a negative erro in case of failure.
+ * 0
*/
#define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \
@@ -2378,7 +2412,8 @@ union bpf_attr {
FN(map_pop_elem), \
FN(map_peek_elem), \
FN(msg_push_data), \
- FN(msg_pop_data),
+ FN(msg_pop_data), \
+ FN(rc_pointer_rel),
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
* function eBPF program intends to call
@@ -2496,6 +2531,7 @@ struct __sk_buff {
__u32 data_meta;
__bpf_md_ptr(struct bpf_flow_keys *, flow_keys);
__u64 tstamp;
+ __u32 wire_len;
};
struct bpf_tunnel_key {
@@ -2674,7 +2710,13 @@ struct bpf_prog_info {
__u32 btf_id;
__u32 func_info_rec_size;
__aligned_u64 func_info;
- __u32 func_info_cnt;
+ __u32 nr_func_info;
+ __u32 nr_line_info;
+ __aligned_u64 line_info;
+ __aligned_u64 jited_line_info;
+ __u32 nr_jited_line_info;
+ __u32 line_info_rec_size;
+ __u32 jited_line_info_rec_size;
} __attribute__((aligned(8)));
struct bpf_map_info {
@@ -2987,8 +3029,18 @@ struct bpf_flow_keys {
};
struct bpf_func_info {
- __u32 insn_offset;
+ __u32 insn_off;
__u32 type_id;
};
+#define BPF_LINE_INFO_LINE_NUM(line_col) ((line_col) >> 10)
+#define BPF_LINE_INFO_LINE_COL(line_col) ((line_col) & 0x3ff)
+
+struct bpf_line_info {
+ __u32 insn_off;
+ __u32 file_name_off;
+ __u32 line_off;
+ __u32 line_col;
+};
+
#endif /* _UAPI__LINUX_BPF_H__ */
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 793acba40b4c..bf34933cc413 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -444,7 +444,7 @@ static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
return kind_ops[BTF_INFO_KIND(t->info)];
}
-static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
+bool btf_name_offset_valid(const struct btf *btf, u32 offset)
{
return BTF_STR_OFFSET_VALID(offset) &&
offset < btf->hdr.str_len;
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index f93ed667546f..5cdd8da0e7f2 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -105,6 +105,91 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)
}
EXPORT_SYMBOL_GPL(bpf_prog_alloc);
+int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)
+{
+ if (!prog->aux->nr_linfo || !prog->jit_requested)
+ return 0;
+
+ prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo,
+ sizeof(*prog->aux->jited_linfo),
+ GFP_KERNEL | __GFP_NOWARN);
+ if (!prog->aux->jited_linfo)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void bpf_prog_free_jited_linfo(struct bpf_prog *prog)
+{
+ kfree(prog->aux->jited_linfo);
+ prog->aux->jited_linfo = NULL;
+}
+
+void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog)
+{
+ if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0])
+ bpf_prog_free_jited_linfo(prog);
+}
+
+/* The jit engine is responsible to provide an array
+ * for insn_off to the jited_off mapping (insn_to_jit_off).
+ *
+ * The idx to this array is the insn_off. Hence, the insn_off
+ * here is relative to the prog itself instead of the main prog.
+ * This array has one entry for each xlated bpf insn.
+ *
+ * jited_off is the byte off to the last byte of the jited insn.
+ *
+ * Hence, with
+ * insn_start:
+ * The first bpf insn off of the prog. The insn off
+ * here is relative to the main prog.
+ * e.g. if prog is a subprog, insn_start > 0
+ * linfo_idx:
+ * The prog's idx to prog->aux->linfo and jited_linfo
+ *
+ * jited_linfo[linfo_idx] = prog->bpf_func
+ *
+ * For i > linfo_idx,
+ *
+ * jited_linfo[i] = prog->bpf_func +
+ * insn_to_jit_off[linfo[i].insn_off - insn_start - 1]
+ */
+void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
+ const u32 *insn_to_jit_off)
+{
+ u32 linfo_idx, insn_start, insn_end, nr_linfo, i;
+ const struct bpf_line_info *linfo;
+ void **jited_linfo;
+
+ if (!prog->aux->jited_linfo)
+ /* Userspace did not provide linfo */
+ return;
+
+ linfo_idx = prog->aux->linfo_idx;
+ linfo = &prog->aux->linfo[linfo_idx];
+ insn_start = linfo[0].insn_off;
+ insn_end = insn_start + prog->len;
+
+ jited_linfo = &prog->aux->jited_linfo[linfo_idx];
+ jited_linfo[0] = prog->bpf_func;
+
+ nr_linfo = prog->aux->nr_linfo - linfo_idx;
+
+ for (i = 1; i < nr_linfo && linfo[i].insn_off < insn_end; i++)
+ /* The verifier ensures that linfo[i].insn_off is
+ * strictly increasing
+ */
+ jited_linfo[i] = prog->bpf_func +
+ insn_to_jit_off[linfo[i].insn_off - insn_start - 1];
+}
+
+void bpf_prog_free_linfo(struct bpf_prog *prog)
+{
+ bpf_prog_free_jited_linfo(prog);
+ kvfree(prog->aux->linfo);
+}
+
struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
gfp_t gfp_extra_flags)
{
@@ -294,6 +379,26 @@ static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta,
return ret;
}
+static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta)
+{
+ struct bpf_line_info *linfo;
+ u32 i, nr_linfo;
+
+ nr_linfo = prog->aux->nr_linfo;
+ if (!nr_linfo || !delta)
+ return;
+
+ linfo = prog->aux->linfo;
+
+ for (i = 0; i < nr_linfo; i++)
+ if (off < linfo[i].insn_off)
+ break;
+
+ /* Push all off < linfo[i].insn_off by delta */
+ for (; i < nr_linfo; i++)
+ linfo[i].insn_off += delta;
+}
+
struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
const struct bpf_insn *patch, u32 len)
{
@@ -349,6 +454,8 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
*/
BUG_ON(bpf_adj_branches(prog_adj, off, insn_delta, false));
+ bpf_adj_linfo(prog_adj, off, insn_delta);
+
return prog_adj;
}
@@ -410,7 +517,7 @@ static void bpf_get_prog_name(const struct bpf_prog *prog, char *sym)
sym = bin2hex(sym, prog->tag, sizeof(prog->tag));
/* prog->aux->name will be ignored if full btf name is available */
- if (prog->aux->btf) {
+ if (prog->aux->func_info_cnt) {
type = btf_type_by_id(prog->aux->btf,
prog->aux->func_info[prog->aux->func_idx].type_id);
func_name = btf_name_by_offset(prog->aux->btf, type->name_off);
@@ -623,6 +730,16 @@ static void bpf_jit_uncharge_modmem(u32 pages)
atomic_long_sub(pages, &bpf_jit_current);
}
+void *__weak bpf_jit_alloc_exec(unsigned long size)
+{
+ return module_alloc(size);
+}
+
+void __weak bpf_jit_free_exec(void *addr)
+{
+ module_memfree(addr);
+}
+
struct bpf_binary_header *
bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
unsigned int alignment,
@@ -640,7 +757,7 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
if (bpf_jit_charge_modmem(pages))
return NULL;
- hdr = module_alloc(size);
+ hdr = bpf_jit_alloc_exec(size);
if (!hdr) {
bpf_jit_uncharge_modmem(pages);
return NULL;
@@ -664,7 +781,7 @@ void bpf_jit_binary_free(struct bpf_binary_header *hdr)
{
u32 pages = hdr->pages;
- module_memfree(hdr);
+ bpf_jit_free_exec(hdr);
bpf_jit_uncharge_modmem(pages);
}
@@ -923,32 +1040,34 @@ EXPORT_SYMBOL_GPL(__bpf_call_base);
#define BPF_INSN_MAP(INSN_2, INSN_3) \
/* 32 bit ALU operations. */ \
/* Register based. */ \
- INSN_3(ALU, ADD, X), \
- INSN_3(ALU, SUB, X), \
- INSN_3(ALU, AND, X), \
- INSN_3(ALU, OR, X), \
- INSN_3(ALU, LSH, X), \
- INSN_3(ALU, RSH, X), \
- INSN_3(ALU, XOR, X), \
- INSN_3(ALU, MUL, X), \
- INSN_3(ALU, MOV, X), \
- INSN_3(ALU, DIV, X), \
- INSN_3(ALU, MOD, X), \
+ INSN_3(ALU, ADD, X), \
+ INSN_3(ALU, SUB, X), \
+ INSN_3(ALU, AND, X), \
+ INSN_3(ALU, OR, X), \
+ INSN_3(ALU, LSH, X), \
+ INSN_3(ALU, RSH, X), \
+ INSN_3(ALU, XOR, X), \
+ INSN_3(ALU, MUL, X), \
+ INSN_3(ALU, MOV, X), \
+ INSN_3(ALU, ARSH, X), \
+ INSN_3(ALU, DIV, X), \
+ INSN_3(ALU, MOD, X), \
INSN_2(ALU, NEG), \
INSN_3(ALU, END, TO_BE), \
INSN_3(ALU, END, TO_LE), \
/* Immediate based. */ \
- INSN_3(ALU, ADD, K), \
- INSN_3(ALU, SUB, K), \
- INSN_3(ALU, AND, K), \
- INSN_3(ALU, OR, K), \
- INSN_3(ALU, LSH, K), \
- INSN_3(ALU, RSH, K), \
- INSN_3(ALU, XOR, K), \
- INSN_3(ALU, MUL, K), \
- INSN_3(ALU, MOV, K), \
- INSN_3(ALU, DIV, K), \
- INSN_3(ALU, MOD, K), \
+ INSN_3(ALU, ADD, K), \
+ INSN_3(ALU, SUB, K), \
+ INSN_3(ALU, AND, K), \
+ INSN_3(ALU, OR, K), \
+ INSN_3(ALU, LSH, K), \
+ INSN_3(ALU, RSH, K), \
+ INSN_3(ALU, XOR, K), \
+ INSN_3(ALU, MUL, K), \
+ INSN_3(ALU, MOV, K), \
+ INSN_3(ALU, ARSH, K), \
+ INSN_3(ALU, DIV, K), \
+ INSN_3(ALU, MOD, K), \
/* 64 bit ALU operations. */ \
/* Register based. */ \
INSN_3(ALU64, ADD, X), \
@@ -1127,6 +1246,12 @@ select_insn:
DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32;
insn++;
CONT;
+ ALU_ARSH_X:
+ DST = (u64) (u32) ((*(s32 *) &DST) >> SRC);
+ CONT;
+ ALU_ARSH_K:
+ DST = (u64) (u32) ((*(s32 *) &DST) >> IMM);
+ CONT;
ALU64_ARSH_X:
(*(s64 *) &DST) >>= SRC;
CONT;
@@ -1573,13 +1698,20 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
* be JITed, but falls back to the interpreter.
*/
if (!bpf_prog_is_dev_bound(fp->aux)) {
+ *err = bpf_prog_alloc_jited_linfo(fp);
+ if (*err)
+ return fp;
+
fp = bpf_int_jit_compile(fp);
-#ifdef CONFIG_BPF_JIT_ALWAYS_ON
if (!fp->jited) {
+ bpf_prog_free_jited_linfo(fp);
+#ifdef CONFIG_BPF_JIT_ALWAYS_ON
*err = -ENOTSUPP;
return fp;
- }
#endif
+ } else {
+ bpf_prog_free_unused_jited_linfo(fp);
+ }
} else {
*err = bpf_prog_offload_compile(fp);
if (*err)
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 85cbeec06e50..5745c7837621 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1215,6 +1215,7 @@ static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
bpf_prog_kallsyms_del_all(prog);
btf_put(prog->aux->btf);
kvfree(prog->aux->func_info);
+ bpf_prog_free_linfo(prog);
call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
}
@@ -1439,7 +1440,7 @@ bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type,
}
/* last field in 'union bpf_attr' used by this command */
-#define BPF_PROG_LOAD_LAST_FIELD func_info_cnt
+#define BPF_PROG_LOAD_LAST_FIELD line_info_cnt
static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
{
@@ -1452,9 +1453,14 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
if (CHECK_ATTR(BPF_PROG_LOAD))
return -EINVAL;
- if (attr->prog_flags & ~BPF_F_STRICT_ALIGNMENT)
+ if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | BPF_F_ANY_ALIGNMENT))
return -EINVAL;
+ if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
+ (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
+ !capable(CAP_SYS_ADMIN))
+ return -EPERM;
+
/* copy eBPF program license from user space */
if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
sizeof(license) - 1) < 0)
@@ -1555,6 +1561,9 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
return err;
free_used_maps:
+ bpf_prog_free_linfo(prog);
+ kvfree(prog->aux->func_info);
+ btf_put(prog->aux->btf);
bpf_prog_kallsyms_del_subprogs(prog);
free_used_maps(prog->aux);
free_prog:
@@ -2034,6 +2043,37 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog)
return insns;
}
+static int set_info_rec_size(struct bpf_prog_info *info)
+{
+ /*
+ * Ensure info.*_rec_size is the same as kernel expected size
+ *
+ * or
+ *
+ * Only allow zero *_rec_size if both _rec_size and _cnt are
+ * zero. In this case, the kernel will set the expected
+ * _rec_size back to the info.
+ */
+
+ if ((info->nr_func_info || info->func_info_rec_size) &&
+ info->func_info_rec_size != sizeof(struct bpf_func_info))
+ return -EINVAL;
+
+ if ((info->nr_line_info || info->line_info_rec_size) &&
+ info->line_info_rec_size != sizeof(struct bpf_line_info))
+ return -EINVAL;
+
+ if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
+ info->jited_line_info_rec_size != sizeof(__u64))
+ return -EINVAL;
+
+ info->func_info_rec_size = sizeof(struct bpf_func_info);
+ info->line_info_rec_size = sizeof(struct bpf_line_info);
+ info->jited_line_info_rec_size = sizeof(__u64);
+
+ return 0;
+}
+
static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
const union bpf_attr *attr,
union bpf_attr __user *uattr)
@@ -2076,12 +2116,18 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
return -EFAULT;
}
+ err = set_info_rec_size(&info);
+ if (err)
+ return err;
+
if (!capable(CAP_SYS_ADMIN)) {
info.jited_prog_len = 0;
info.xlated_prog_len = 0;
info.nr_jited_ksyms = 0;
info.nr_jited_func_lens = 0;
- info.func_info_cnt = 0;
+ info.nr_func_info = 0;
+ info.nr_line_info = 0;
+ info.nr_jited_line_info = 0;
goto done;
}
@@ -2163,7 +2209,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
ulen = info.nr_jited_ksyms;
info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
- if (info.nr_jited_ksyms && ulen) {
+ if (ulen) {
if (bpf_dump_raw_ok()) {
unsigned long ksym_addr;
u64 __user *user_ksyms;
@@ -2194,7 +2240,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
ulen = info.nr_jited_func_lens;
info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
- if (info.nr_jited_func_lens && ulen) {
+ if (ulen) {
if (bpf_dump_raw_ok()) {
u32 __user *user_lens;
u32 func_len, i;
@@ -2219,35 +2265,61 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
}
}
- if (prog->aux->btf) {
- u32 krec_size = sizeof(struct bpf_func_info);
- u32 ucnt, urec_size;
-
+ if (prog->aux->btf)
info.btf_id = btf_id(prog->aux->btf);
- ucnt = info.func_info_cnt;
- info.func_info_cnt = prog->aux->func_info_cnt;
- urec_size = info.func_info_rec_size;
- info.func_info_rec_size = krec_size;
- if (ucnt) {
- /* expect passed-in urec_size is what the kernel expects */
- if (urec_size != info.func_info_rec_size)
- return -EINVAL;
-
- if (bpf_dump_raw_ok()) {
- char __user *user_finfo;
-
- user_finfo = u64_to_user_ptr(info.func_info);
- ucnt = min_t(u32, info.func_info_cnt, ucnt);
- if (copy_to_user(user_finfo, prog->aux->func_info,
- krec_size * ucnt))
+ ulen = info.nr_func_info;
+ info.nr_func_info = prog->aux->func_info_cnt;
+ if (info.nr_func_info && ulen) {
+ if (bpf_dump_raw_ok()) {
+ char __user *user_finfo;
+
+ user_finfo = u64_to_user_ptr(info.func_info);
+ ulen = min_t(u32, info.nr_func_info, ulen);
+ if (copy_to_user(user_finfo, prog->aux->func_info,
+ info.func_info_rec_size * ulen))
+ return -EFAULT;
+ } else {
+ info.func_info = 0;
+ }
+ }
+
+ ulen = info.nr_line_info;
+ info.nr_line_info = prog->aux->nr_linfo;
+ if (info.nr_line_info && ulen) {
+ if (bpf_dump_raw_ok()) {
+ __u8 __user *user_linfo;
+
+ user_linfo = u64_to_user_ptr(info.line_info);
+ ulen = min_t(u32, info.nr_line_info, ulen);
+ if (copy_to_user(user_linfo, prog->aux->linfo,
+ info.line_info_rec_size * ulen))
+ return -EFAULT;
+ } else {
+ info.line_info = 0;
+ }
+ }
+
+ ulen = info.nr_jited_line_info;
+ if (prog->aux->jited_linfo)
+ info.nr_jited_line_info = prog->aux->nr_linfo;
+ else
+ info.nr_jited_line_info = 0;
+ if (info.nr_jited_line_info && ulen) {
+ if (bpf_dump_raw_ok()) {
+ __u64 __user *user_linfo;
+ u32 i;
+
+ user_linfo = u64_to_user_ptr(info.jited_line_info);
+ ulen = min_t(u32, info.nr_jited_line_info, ulen);
+ for (i = 0; i < ulen; i++) {
+ if (put_user((__u64)(long)prog->aux->jited_linfo[i],
+ &user_linfo[i]))
return -EFAULT;
- } else {
- info.func_info_cnt = 0;
}
+ } else {
+ info.jited_line_info = 0;
}
- } else {
- info.func_info_cnt = 0;
}
done:
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index ee30effdf98a..8b511a4fe84a 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -3584,12 +3584,15 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
return err;
if (BPF_SRC(insn->code) == BPF_X) {
+ struct bpf_reg_state *src_reg = regs + insn->src_reg;
+ struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
+
if (BPF_CLASS(insn->code) == BPF_ALU64) {
/* case: R1 = R2
* copy register state to dest reg
*/
- regs[insn->dst_reg] = regs[insn->src_reg];
- regs[insn->dst_reg].live |= REG_LIVE_WRITTEN;
+ *dst_reg = *src_reg;
+ dst_reg->live |= REG_LIVE_WRITTEN;
} else {
/* R1 = (u32) R2 */
if (is_pointer_value(env, insn->src_reg)) {
@@ -3597,9 +3600,14 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
"R%d partial copy of pointer\n",
insn->src_reg);
return -EACCES;
+ } else if (src_reg->type == SCALAR_VALUE) {
+ *dst_reg = *src_reg;
+ dst_reg->live |= REG_LIVE_WRITTEN;
+ } else {
+ mark_reg_unknown(env, regs,
+ insn->dst_reg);
}
- mark_reg_unknown(env, regs, insn->dst_reg);
- coerce_reg_to_size(&regs[insn->dst_reg], 4);
+ coerce_reg_to_size(dst_reg, 4);
}
} else {
/* case: R = imm
@@ -3650,11 +3658,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
return -EINVAL;
}
- if (opcode == BPF_ARSH && BPF_CLASS(insn->code) != BPF_ALU64) {
- verbose(env, "BPF_ARSH not supported for 32 bit ALU\n");
- return -EINVAL;
- }
-
if ((opcode == BPF_LSH || opcode == BPF_RSH ||
opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
@@ -4713,15 +4716,17 @@ err_free:
#define MIN_BPF_FUNCINFO_SIZE 8
#define MAX_FUNCINFO_REC_SIZE 252
-static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env,
- union bpf_attr *attr, union bpf_attr __user *uattr)
+static int check_btf_func(struct bpf_verifier_env *env,
+ const union bpf_attr *attr,
+ union bpf_attr __user *uattr)
{
u32 i, nfuncs, urec_size, min_size, prev_offset;
u32 krec_size = sizeof(struct bpf_func_info);
- struct bpf_func_info *krecord = NULL;
+ struct bpf_func_info *krecord;
const struct btf_type *type;
+ struct bpf_prog *prog;
+ const struct btf *btf;
void __user *urecord;
- struct btf *btf;
int ret = 0;
nfuncs = attr->func_info_cnt;
@@ -4741,20 +4746,15 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env,
return -EINVAL;
}
- btf = btf_get_by_fd(attr->prog_btf_fd);
- if (IS_ERR(btf)) {
- verbose(env, "unable to get btf from fd\n");
- return PTR_ERR(btf);
- }
+ prog = env->prog;
+ btf = prog->aux->btf;
urecord = u64_to_user_ptr(attr->func_info);
min_size = min_t(u32, krec_size, urec_size);
krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
- if (!krecord) {
- ret = -ENOMEM;
- goto free_btf;
- }
+ if (!krecord)
+ return -ENOMEM;
for (i = 0; i < nfuncs; i++) {
ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
@@ -4767,35 +4767,35 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env,
if (put_user(min_size, &uattr->func_info_rec_size))
ret = -EFAULT;
}
- goto free_btf;
+ goto err_free;
}
if (copy_from_user(&krecord[i], urecord, min_size)) {
ret = -EFAULT;
- goto free_btf;
+ goto err_free;
}
- /* check insn_offset */
+ /* check insn_off */
if (i == 0) {
- if (krecord[i].insn_offset) {
+ if (krecord[i].insn_off) {
verbose(env,
- "nonzero insn_offset %u for the first func info record",
- krecord[i].insn_offset);
+ "nonzero insn_off %u for the first func info record",
+ krecord[i].insn_off);
ret = -EINVAL;
- goto free_btf;
+ goto err_free;
}
- } else if (krecord[i].insn_offset <= prev_offset) {
+ } else if (krecord[i].insn_off <= prev_offset) {
verbose(env,
"same or smaller insn offset (%u) than previous func info record (%u)",
- krecord[i].insn_offset, prev_offset);
+ krecord[i].insn_off, prev_offset);
ret = -EINVAL;
- goto free_btf;
+ goto err_free;
}
- if (env->subprog_info[i].start != krecord[i].insn_offset) {
+ if (env->subprog_info[i].start != krecord[i].insn_off) {
verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
ret = -EINVAL;
- goto free_btf;
+ goto err_free;
}
/* check type_id */
@@ -4804,20 +4804,18 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env,
verbose(env, "invalid type id %d in func info",
krecord[i].type_id);
ret = -EINVAL;
- goto free_btf;
+ goto err_free;
}
- prev_offset = krecord[i].insn_offset;
+ prev_offset = krecord[i].insn_off;
urecord += urec_size;
}
- prog->aux->btf = btf;
prog->aux->func_info = krecord;
prog->aux->func_info_cnt = nfuncs;
return 0;
-free_btf:
- btf_put(btf);
+err_free:
kvfree(krecord);
return ret;
}
@@ -4830,7 +4828,151 @@ static void adjust_btf_func(struct bpf_verifier_env *env)
return;
for (i = 0; i < env->subprog_cnt; i++)
- env->prog->aux->func_info[i].insn_offset = env->subprog_info[i].start;
+ env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start;
+}
+
+#define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \
+ sizeof(((struct bpf_line_info *)(0))->line_col))
+#define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE
+
+static int check_btf_line(struct bpf_verifier_env *env,
+ const union bpf_attr *attr,
+ union bpf_attr __user *uattr)
+{
+ u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
+ struct bpf_subprog_info *sub;
+ struct bpf_line_info *linfo;
+ struct bpf_prog *prog;
+ const struct btf *btf;
+ void __user *ulinfo;
+ int err;
+
+ nr_linfo = attr->line_info_cnt;
+ if (!nr_linfo)
+ return 0;
+
+ rec_size = attr->line_info_rec_size;
+ if (rec_size < MIN_BPF_LINEINFO_SIZE ||
+ rec_size > MAX_LINEINFO_REC_SIZE ||
+ rec_size & (sizeof(u32) - 1))
+ return -EINVAL;
+
+ /* Need to zero it in case the userspace may
+ * pass in a smaller bpf_line_info object.
+ */
+ linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
+ GFP_KERNEL | __GFP_NOWARN);
+ if (!linfo)
+ return -ENOMEM;
+
+ prog = env->prog;
+ btf = prog->aux->btf;
+
+ s = 0;
+ sub = env->subprog_info;
+ ulinfo = u64_to_user_ptr(attr->line_info);
+ expected_size = sizeof(struct bpf_line_info);
+ ncopy = min_t(u32, expected_size, rec_size);
+ for (i = 0; i < nr_linfo; i++) {
+ err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
+ if (err) {
+ if (err == -E2BIG) {
+ verbose(env, "nonzero tailing record in line_info");
+ if (put_user(expected_size,
+ &uattr->line_info_rec_size))
+ err = -EFAULT;
+ }
+ goto err_free;
+ }
+
+ if (copy_from_user(&linfo[i], ulinfo, ncopy)) {
+ err = -EFAULT;
+ goto err_free;
+ }
+
+ /*
+ * Check insn_off to ensure
+ * 1) strictly increasing AND
+ * 2) bounded by prog->len
+ *
+ * The linfo[0].insn_off == 0 check logically falls into
+ * the later "missing bpf_line_info for func..." case
+ * because the first linfo[0].insn_off must be the
+ * first sub also and the first sub must have
+ * subprog_info[0].start == 0.
+ */
+ if ((i && linfo[i].insn_off <= prev_offset) ||
+ linfo[i].insn_off >= prog->len) {
+ verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
+ i, linfo[i].insn_off, prev_offset,
+ prog->len);
+ err = -EINVAL;
+ goto err_free;
+ }
+
+ if (!btf_name_offset_valid(btf, linfo[i].line_off) ||
+ !btf_name_offset_valid(btf, linfo[i].file_name_off)) {
+ verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
+ err = -EINVAL;
+ goto err_free;
+ }
+
+ if (s != env->subprog_cnt) {
+ if (linfo[i].insn_off == sub[s].start) {
+ sub[s].linfo_idx = i;
+ s++;
+ } else if (sub[s].start < linfo[i].insn_off) {
+ verbose(env, "missing bpf_line_info for func#%u\n", s);
+ err = -EINVAL;
+ goto err_free;
+ }
+ }
+
+ prev_offset = linfo[i].insn_off;
+ ulinfo += rec_size;
+ }
+
+ if (s != env->subprog_cnt) {
+ verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
+ env->subprog_cnt - s, s);
+ err = -EINVAL;
+ goto err_free;
+ }
+
+ prog->aux->linfo = linfo;
+ prog->aux->nr_linfo = nr_linfo;
+
+ return 0;
+
+err_free:
+ kvfree(linfo);
+ return err;
+}
+
+static int check_btf_info(struct bpf_verifier_env *env,
+ const union bpf_attr *attr,
+ union bpf_attr __user *uattr)
+{
+ struct btf *btf;
+ int err;
+
+ if (!attr->func_info_cnt && !attr->line_info_cnt)
+ return 0;
+
+ btf = btf_get_by_fd(attr->prog_btf_fd);
+ if (IS_ERR(btf))
+ return PTR_ERR(btf);
+ env->prog->aux->btf = btf;
+
+ err = check_btf_func(env, attr, uattr);
+ if (err)
+ return err;
+
+ err = check_btf_line(env, attr, uattr);
+ if (err)
+ return err;
+
+ return 0;
}
/* check %cur's range satisfies %old's */
@@ -6084,7 +6226,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
int i, j, subprog_start, subprog_end = 0, len, subprog;
struct bpf_insn *insn;
void *old_bpf_func;
- int err = -ENOMEM;
+ int err;
if (env->subprog_cnt <= 1)
return 0;
@@ -6115,6 +6257,11 @@ static int jit_subprogs(struct bpf_verifier_env *env)
insn->imm = 1;
}
+ err = bpf_prog_alloc_jited_linfo(prog);
+ if (err)
+ goto out_undo_insn;
+
+ err = -ENOMEM;
func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
if (!func)
goto out_undo_insn;
@@ -6145,6 +6292,10 @@ static int jit_subprogs(struct bpf_verifier_env *env)
func[i]->aux->name[0] = 'F';
func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
func[i]->jit_requested = 1;
+ func[i]->aux->linfo = prog->aux->linfo;
+ func[i]->aux->nr_linfo = prog->aux->nr_linfo;
+ func[i]->aux->jited_linfo = prog->aux->jited_linfo;
+ func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
func[i] = bpf_int_jit_compile(func[i]);
if (!func[i]->jited) {
err = -ENOTSUPP;
@@ -6218,6 +6369,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
prog->bpf_func = func[0]->bpf_func;
prog->aux->func = func;
prog->aux->func_cnt = env->subprog_cnt;
+ bpf_prog_free_unused_jited_linfo(prog);
return 0;
out_free:
for (i = 0; i < env->subprog_cnt; i++)
@@ -6234,6 +6386,7 @@ out_undo_insn:
insn->off = 0;
insn->imm = env->insn_aux_data[i].call_imm;
}
+ bpf_prog_free_jited_linfo(prog);
return err;
}
@@ -6580,6 +6733,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
env->strict_alignment = true;
+ if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
+ env->strict_alignment = false;
ret = replace_map_fd_with_map_ptr(env);
if (ret < 0)
@@ -6604,7 +6759,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
if (ret < 0)
goto skip_full_check;
- ret = check_btf_func(env->prog, env, attr, uattr);
+ ret = check_btf_info(env, attr, uattr);
if (ret < 0)
goto skip_full_check;
diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index 25001913d03b..fa2644d276ef 100644
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -75,8 +75,18 @@ static int bpf_test_finish(const union bpf_attr *kattr,
{
void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
int err = -EFAULT;
+ u32 copy_size = size;
+
+ /* Clamp copy if the user has provided a size hint, but copy the full
+ * buffer if not to retain old behaviour.
+ */
+ if (kattr->test.data_size_out &&
+ copy_size > kattr->test.data_size_out) {
+ copy_size = kattr->test.data_size_out;
+ err = -ENOSPC;
+ }
- if (data_out && copy_to_user(data_out, data, size))
+ if (data_out && copy_to_user(data_out, data, copy_size))
goto out;
if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
goto out;
@@ -84,7 +94,8 @@ static int bpf_test_finish(const union bpf_attr *kattr,
goto out;
if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
goto out;
- err = 0;
+ if (err != -ENOSPC)
+ err = 0;
out:
return err;
}
diff --git a/net/core/filter.c b/net/core/filter.c
index 8659b40172d1..f9348806e843 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -5774,6 +5774,7 @@ static bool sk_filter_is_valid_access(int off, int size,
case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
case bpf_ctx_range_till(struct __sk_buff, family, local_port):
case bpf_ctx_range(struct __sk_buff, tstamp):
+ case bpf_ctx_range(struct __sk_buff, wire_len):
return false;
}
@@ -5798,6 +5799,7 @@ static bool cg_skb_is_valid_access(int off, int size,
case bpf_ctx_range(struct __sk_buff, tc_classid):
case bpf_ctx_range(struct __sk_buff, data_meta):
case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
+ case bpf_ctx_range(struct __sk_buff, wire_len):
return false;
case bpf_ctx_range(struct __sk_buff, data):
case bpf_ctx_range(struct __sk_buff, data_end):
@@ -5844,6 +5846,7 @@ static bool lwt_is_valid_access(int off, int size,
case bpf_ctx_range(struct __sk_buff, data_meta):
case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
case bpf_ctx_range(struct __sk_buff, tstamp):
+ case bpf_ctx_range(struct __sk_buff, wire_len):
return false;
}
@@ -6274,6 +6277,7 @@ static bool sk_skb_is_valid_access(int off, int size,
case bpf_ctx_range(struct __sk_buff, data_meta):
case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
case bpf_ctx_range(struct __sk_buff, tstamp):
+ case bpf_ctx_range(struct __sk_buff, wire_len):
return false;
}
@@ -6361,6 +6365,7 @@ static bool flow_dissector_is_valid_access(int off, int size,
case bpf_ctx_range(struct __sk_buff, data_meta):
case bpf_ctx_range_till(struct __sk_buff, family, local_port):
case bpf_ctx_range(struct __sk_buff, tstamp):
+ case bpf_ctx_range(struct __sk_buff, wire_len):
return false;
}
@@ -6686,6 +6691,17 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type,
bpf_target_off(struct sk_buff,
tstamp, 8,
target_size));
+ break;
+
+ case offsetof(struct __sk_buff, wire_len):
+ BUILD_BUG_ON(FIELD_SIZEOF(struct qdisc_skb_cb, pkt_len) != 4);
+
+ off = si->off;
+ off -= offsetof(struct __sk_buff, wire_len);
+ off += offsetof(struct sk_buff, cb);
+ off += offsetof(struct qdisc_skb_cb, pkt_len);
+ *target_size = 4;
+ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, off);
}
return insn - insn_buf;
diff --git a/samples/bpf/bpf_load.c b/samples/bpf/bpf_load.c
index 434ea34a5954..eae7b635343d 100644
--- a/samples/bpf/bpf_load.c
+++ b/samples/bpf/bpf_load.c
@@ -58,7 +58,9 @@ static int write_kprobe_events(const char *val)
{
int fd, ret, flags;
- if ((val != NULL) && (val[0] == '\0'))
+ if (val == NULL)
+ return -1;
+ else if (val[0] == '\0')
flags = O_WRONLY | O_TRUNC;
else
flags = O_WRONLY | O_APPEND;
diff --git a/samples/bpf/xdp1_user.c b/samples/bpf/xdp1_user.c
index b02c531510ed..0a197f86ac43 100644
--- a/samples/bpf/xdp1_user.c
+++ b/samples/bpf/xdp1_user.c
@@ -15,6 +15,7 @@
#include <unistd.h>
#include <libgen.h>
#include <sys/resource.h>
+#include <net/if.h>
#include "bpf_util.h"
#include "bpf/bpf.h"
@@ -34,26 +35,24 @@ static void int_exit(int sig)
static void poll_stats(int map_fd, int interval)
{
unsigned int nr_cpus = bpf_num_possible_cpus();
- const unsigned int nr_keys = 256;
- __u64 values[nr_cpus], prev[nr_keys][nr_cpus];
- __u32 key;
+ __u64 values[nr_cpus], prev[UINT8_MAX] = { 0 };
int i;
- memset(prev, 0, sizeof(prev));
-
while (1) {
+ __u32 key = UINT32_MAX;
+
sleep(interval);
- for (key = 0; key < nr_keys; key++) {
+ while (bpf_map_get_next_key(map_fd, &key, &key) != -1) {
__u64 sum = 0;
assert(bpf_map_lookup_elem(map_fd, &key, values) == 0);
for (i = 0; i < nr_cpus; i++)
- sum += (values[i] - prev[key][i]);
- if (sum)
+ sum += values[i];
+ if (sum > prev[key])
printf("proto %u: %10llu pkt/s\n",
- key, sum / interval);
- memcpy(prev[key], values, sizeof(values));
+ key, (sum - prev[key]) / interval);
+ prev[key] = sum;
}
}
}
@@ -61,7 +60,7 @@ static void poll_stats(int map_fd, int interval)
static void usage(const char *prog)
{
fprintf(stderr,
- "usage: %s [OPTS] IFINDEX\n\n"
+ "usage: %s [OPTS] IFACE\n\n"
"OPTS:\n"
" -S use skb-mode\n"
" -N enforce native mode\n",
@@ -104,7 +103,11 @@ int main(int argc, char **argv)
return 1;
}
- ifindex = strtoul(argv[optind], NULL, 0);
+ ifindex = if_nametoindex(argv[1]);
+ if (!ifindex) {
+ perror("if_nametoindex");
+ return 1;
+ }
snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
prog_load_attr.file = filename;
diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst
index ab36e920e552..bb1aeb98b6da 100644
--- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst
+++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst
@@ -22,12 +22,13 @@ MAP COMMANDS
=============
| **bpftool** **prog { show | list }** [*PROG*]
-| **bpftool** **prog dump xlated** *PROG* [{**file** *FILE* | **opcodes** | **visual**}]
-| **bpftool** **prog dump jited** *PROG* [{**file** *FILE* | **opcodes**}]
+| **bpftool** **prog dump xlated** *PROG* [{**file** *FILE* | **opcodes** | **visual** | **linum**}]
+| **bpftool** **prog dump jited** *PROG* [{**file** *FILE* | **opcodes** | **linum**}]
| **bpftool** **prog pin** *PROG* *FILE*
| **bpftool** **prog { load | loadall }** *OBJ* *PATH* [**type** *TYPE*] [**map** {**idx** *IDX* | **name** *NAME*} *MAP*] [**dev** *NAME*]
-| **bpftool** **prog attach** *PROG* *ATTACH_TYPE* [*MAP*]
-| **bpftool** **prog detach** *PROG* *ATTACH_TYPE* [*MAP*]
+| **bpftool** **prog attach** *PROG* *ATTACH_TYPE* [*MAP*]
+| **bpftool** **prog detach** *PROG* *ATTACH_TYPE* [*MAP*]
+| **bpftool** **prog tracelog**
| **bpftool** **prog help**
|
| *MAP* := { **id** *MAP_ID* | **pinned** *FILE* }
@@ -55,7 +56,7 @@ DESCRIPTION
Output will start with program ID followed by program type and
zero or more named attributes (depending on kernel version).
- **bpftool prog dump xlated** *PROG* [{ **file** *FILE* | **opcodes** | **visual** }]
+ **bpftool prog dump xlated** *PROG* [{ **file** *FILE* | **opcodes** | **visual** | **linum** }]
Dump eBPF instructions of the program from the kernel. By
default, eBPF will be disassembled and printed to standard
output in human-readable format. In this case, **opcodes**
@@ -68,13 +69,23 @@ DESCRIPTION
built instead, and eBPF instructions will be presented with
CFG in DOT format, on standard output.
- **bpftool prog dump jited** *PROG* [{ **file** *FILE* | **opcodes** }]
+ If the prog has line_info available, the source line will
+ be displayed by default. If **linum** is specified,
+ the filename, line number and line column will also be
+ displayed on top of the source line.
+
+ **bpftool prog dump jited** *PROG* [{ **file** *FILE* | **opcodes** | **linum** }]
Dump jited image (host machine code) of the program.
If *FILE* is specified image will be written to a file,
otherwise it will be disassembled and printed to stdout.
**opcodes** controls if raw opcodes will be printed.
+ If the prog has line_info available, the source line will
+ be displayed by default. If **linum** is specified,
+ the filename, line number and line column will also be
+ displayed on top of the source line.
+
**bpftool prog pin** *PROG* *FILE*
Pin program *PROG* as *FILE*.
@@ -117,6 +128,14 @@ DESCRIPTION
parameter, with the exception of *flow_dissector* which is
detached from the current networking name space.
+ **bpftool prog tracelog**
+ Dump the trace pipe of the system to the console (stdout).
+ Hit <Ctrl+C> to stop printing. BPF programs can write to this
+ trace pipe at runtime with the **bpf_trace_printk()** helper.
+ This should be used only for debugging purposes. For
+ streaming data from BPF programs to user space, one can use
+ perf events (see also **bpftool-map**\ (8)).
+
**bpftool prog help**
Print short help message.
diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool
index 45c2db257d2b..a57febd6abb1 100644
--- a/tools/bpf/bpftool/bash-completion/bpftool
+++ b/tools/bpf/bpftool/bash-completion/bpftool
@@ -191,7 +191,7 @@ _bpftool()
# Deal with simplest keywords
case $prev in
- help|hex|opcodes|visual)
+ help|hex|opcodes|visual|linum)
return 0
;;
tag)
@@ -243,16 +243,20 @@ _bpftool()
# Completion depends on object and command in use
case $object in
prog)
- if [[ $command != "load" && $command != "loadall" ]]; then
- case $prev in
- id)
- _bpftool_get_prog_ids
- return 0
- ;;
- esac
- fi
+ # Complete id, only for subcommands that use prog (but no map) ids
+ case $command in
+ show|list|dump|pin)
+ case $prev in
+ id)
+ _bpftool_get_prog_ids
+ return 0
+ ;;
+ esac
+ ;;
+ esac
local PROG_TYPE='id pinned tag'
+ local MAP_TYPE='id pinned'
case $command in
show|list)
[[ $prev != "$command" ]] && return 0
@@ -274,10 +278,10 @@ _bpftool()
*)
_bpftool_once_attr 'file'
if _bpftool_search_list 'xlated'; then
- COMPREPLY+=( $( compgen -W 'opcodes visual' -- \
+ COMPREPLY+=( $( compgen -W 'opcodes visual linum' -- \
"$cur" ) )
else
- COMPREPLY+=( $( compgen -W 'opcodes' -- \
+ COMPREPLY+=( $( compgen -W 'opcodes linum' -- \
"$cur" ) )
fi
return 0
@@ -293,22 +297,43 @@ _bpftool()
return 0
;;
attach|detach)
- if [[ ${#words[@]} == 7 ]]; then
- COMPREPLY=( $( compgen -W "id pinned" -- "$cur" ) )
- return 0
- fi
-
- if [[ ${#words[@]} == 6 ]]; then
- COMPREPLY=( $( compgen -W "msg_verdict skb_verdict \
- skb_parse flow_dissector" -- "$cur" ) )
- return 0
- fi
-
- if [[ $prev == "$command" ]]; then
- COMPREPLY=( $( compgen -W "id pinned" -- "$cur" ) )
- return 0
- fi
- return 0
+ case $cword in
+ 3)
+ COMPREPLY=( $( compgen -W "$PROG_TYPE" -- "$cur" ) )
+ return 0
+ ;;
+ 4)
+ case $prev in
+ id)
+ _bpftool_get_prog_ids
+ ;;
+ pinned)
+ _filedir
+ ;;
+ esac
+ return 0
+ ;;
+ 5)
+ COMPREPLY=( $( compgen -W 'msg_verdict skb_verdict \
+ skb_parse flow_dissector' -- "$cur" ) )
+ return 0
+ ;;
+ 6)
+ COMPREPLY=( $( compgen -W "$MAP_TYPE" -- "$cur" ) )
+ return 0
+ ;;
+ 7)
+ case $prev in
+ id)
+ _bpftool_get_map_ids
+ ;;
+ pinned)
+ _filedir
+ ;;
+ esac
+ return 0
+ ;;
+ esac
;;
load|loadall)
local obj
@@ -373,10 +398,13 @@ _bpftool()
;;
esac
;;
+ tracelog)
+ return 0
+ ;;
*)
[[ $prev == $object ]] && \
COMPREPLY=( $( compgen -W 'dump help pin attach detach load \
- show list' -- "$cur" ) )
+ show list tracelog' -- "$cur" ) )
;;
esac
;;
@@ -411,7 +439,7 @@ _bpftool()
lru_percpu_hash lpm_trie array_of_maps \
hash_of_maps devmap sockmap cpumap xskmap \
sockhash cgroup_storage reuseport_sockarray \
- percpu_cgroup_storage' -- \
+ percpu_cgroup_storage queue stack' -- \
"$cur" ) )
return 0
;;
diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c
index 0f1df9019dfc..2392ccdc918f 100644
--- a/tools/bpf/bpftool/btf_dumper.c
+++ b/tools/bpf/bpftool/btf_dumper.c
@@ -385,3 +385,67 @@ void btf_dumper_type_only(const struct btf *btf, __u32 type_id, char *func_sig,
if (err < 0)
func_sig[0] = '\0';
}
+
+static const char *ltrim(const char *s)
+{
+ while (isspace(*s))
+ s++;
+
+ return s;
+}
+
+void btf_dump_linfo_plain(const struct btf *btf,
+ const struct bpf_line_info *linfo,
+ const char *prefix, bool linum)
+{
+ const char *line = btf__name_by_offset(btf, linfo->line_off);
+
+ if (!line)
+ return;
+ line = ltrim(line);
+
+ if (!prefix)
+ prefix = "";
+
+ if (linum) {
+ const char *file = btf__name_by_offset(btf, linfo->file_name_off);
+
+ /* More forgiving on file because linum option is
+ * expected to provide more info than the already
+ * available src line.
+ */
+ if (!file)
+ file = "";
+
+ printf("%s%s [file:%s line_num:%u line_col:%u]\n",
+ prefix, line, file,
+ BPF_LINE_INFO_LINE_NUM(linfo->line_col),
+ BPF_LINE_INFO_LINE_COL(linfo->line_col));
+ } else {
+ printf("%s%s\n", prefix, line);
+ }
+}
+
+void btf_dump_linfo_json(const struct btf *btf,
+ const struct bpf_line_info *linfo, bool linum)
+{
+ const char *line = btf__name_by_offset(btf, linfo->line_off);
+
+ if (line)
+ jsonw_string_field(json_wtr, "src", ltrim(line));
+
+ if (linum) {
+ const char *file = btf__name_by_offset(btf, linfo->file_name_off);
+
+ if (file)
+ jsonw_string_field(json_wtr, "file", file);
+
+ if (BPF_LINE_INFO_LINE_NUM(linfo->line_col))
+ jsonw_int_field(json_wtr, "line_num",
+ BPF_LINE_INFO_LINE_NUM(linfo->line_col));
+
+ if (BPF_LINE_INFO_LINE_COL(linfo->line_col))
+ jsonw_int_field(json_wtr, "line_col",
+ BPF_LINE_INFO_LINE_COL(linfo->line_col));
+ }
+}
diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c
index 4e217d57118e..172d3761d9ab 100644
--- a/tools/bpf/bpftool/common.c
+++ b/tools/bpf/bpftool/common.c
@@ -48,7 +48,6 @@
#include <sys/mount.h>
#include <sys/resource.h>
#include <sys/stat.h>
-#include <sys/types.h>
#include <sys/vfs.h>
#include <bpf.h>
@@ -276,7 +275,7 @@ int get_fd_type(int fd)
char buf[512];
ssize_t n;
- snprintf(path, sizeof(path), "/proc/%d/fd/%d", getpid(), fd);
+ snprintf(path, sizeof(path), "/proc/self/fd/%d", fd);
n = readlink(path, buf, sizeof(buf));
if (n < 0) {
@@ -304,7 +303,7 @@ char *get_fdinfo(int fd, const char *key)
ssize_t n;
FILE *fdi;
- snprintf(path, sizeof(path), "/proc/%d/fdinfo/%d", getpid(), fd);
+ snprintf(path, sizeof(path), "/proc/self/fdinfo/%d", fd);
fdi = fopen(path, "r");
if (!fdi) {
@@ -605,7 +604,7 @@ void print_dev_plain(__u32 ifindex, __u64 ns_dev, __u64 ns_inode)
if (!ifindex)
return;
- printf(" dev ");
+ printf(" offloaded_to ");
if (ifindex_to_name_ns(ifindex, ns_dev, ns_inode, name))
printf("%s", name);
else
diff --git a/tools/bpf/bpftool/jit_disasm.c b/tools/bpf/bpftool/jit_disasm.c
index b2ed5ee1af5f..f381f8628ce9 100644
--- a/tools/bpf/bpftool/jit_disasm.c
+++ b/tools/bpf/bpftool/jit_disasm.c
@@ -19,29 +19,21 @@
#include <string.h>
#include <bfd.h>
#include <dis-asm.h>
-#include <sys/types.h>
#include <sys/stat.h>
#include <limits.h>
+#include <libbpf.h>
#include "json_writer.h"
#include "main.h"
static void get_exec_path(char *tpath, size_t size)
{
+ const char *path = "/proc/self/exe";
ssize_t len;
- char *path;
-
- snprintf(tpath, size, "/proc/%d/exe", (int) getpid());
- tpath[size - 1] = 0;
-
- path = strdup(tpath);
- assert(path);
len = readlink(path, tpath, size - 1);
assert(len > 0);
tpath[len] = 0;
-
- free(path);
}
static int oper_count;
@@ -77,10 +69,16 @@ static int fprintf_json(void *out, const char *fmt, ...)
}
void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
- const char *arch, const char *disassembler_options)
+ const char *arch, const char *disassembler_options,
+ const struct btf *btf,
+ const struct bpf_prog_linfo *prog_linfo,
+ __u64 func_ksym, unsigned int func_idx,
+ bool linum)
{
+ const struct bpf_line_info *linfo = NULL;
disassembler_ftype disassemble;
struct disassemble_info info;
+ unsigned int nr_skip = 0;
int count, i, pc = 0;
char tpath[PATH_MAX];
bfd *bfdf;
@@ -136,12 +134,26 @@ void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
if (json_output)
jsonw_start_array(json_wtr);
do {
+ if (prog_linfo) {
+ linfo = bpf_prog_linfo__lfind_addr_func(prog_linfo,
+ func_ksym + pc,
+ func_idx,
+ nr_skip);
+ if (linfo)
+ nr_skip++;
+ }
+
if (json_output) {
jsonw_start_object(json_wtr);
oper_count = 0;
+ if (linfo)
+ btf_dump_linfo_json(btf, linfo, linum);
jsonw_name(json_wtr, "pc");
jsonw_printf(json_wtr, "\"0x%x\"", pc);
} else {
+ if (linfo)
+ btf_dump_linfo_plain(btf, linfo, "; ",
+ linum);
printf("%4x:\t", pc);
}
diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h
index 7431669fae0a..0b37599f8cda 100644
--- a/tools/bpf/bpftool/main.h
+++ b/tools/bpf/bpftool/main.h
@@ -78,6 +78,32 @@
#define HELP_SPEC_MAP \
"MAP := { id MAP_ID | pinned FILE }"
+static const char * const prog_type_name[] = {
+ [BPF_PROG_TYPE_UNSPEC] = "unspec",
+ [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter",
+ [BPF_PROG_TYPE_KPROBE] = "kprobe",
+ [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls",
+ [BPF_PROG_TYPE_SCHED_ACT] = "sched_act",
+ [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint",
+ [BPF_PROG_TYPE_XDP] = "xdp",
+ [BPF_PROG_TYPE_PERF_EVENT] = "perf_event",
+ [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb",
+ [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock",
+ [BPF_PROG_TYPE_LWT_IN] = "lwt_in",
+ [BPF_PROG_TYPE_LWT_OUT] = "lwt_out",
+ [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit",
+ [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops",
+ [BPF_PROG_TYPE_SK_SKB] = "sk_skb",
+ [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device",
+ [BPF_PROG_TYPE_SK_MSG] = "sk_msg",
+ [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
+ [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr",
+ [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local",
+ [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2",
+ [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport",
+ [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector",
+};
+
enum bpf_obj_type {
BPF_OBJ_UNKNOWN,
BPF_OBJ_PROG,
@@ -112,6 +138,9 @@ struct pinned_obj {
struct hlist_node hash;
};
+struct btf;
+struct bpf_line_info;
+
int build_pinned_obj_table(struct pinned_obj_table *table,
enum bpf_obj_type type);
void delete_pinned_obj_table(struct pinned_obj_table *tab);
@@ -141,6 +170,7 @@ int do_event_pipe(int argc, char **argv);
int do_cgroup(int argc, char **arg);
int do_perf(int argc, char **arg);
int do_net(int argc, char **arg);
+int do_tracelog(int argc, char **arg);
int parse_u32_arg(int *argc, char ***argv, __u32 *val, const char *what);
int prog_parse_fd(int *argc, char ***argv);
@@ -148,13 +178,22 @@ int map_parse_fd(int *argc, char ***argv);
int map_parse_fd_and_info(int *argc, char ***argv, void *info, __u32 *info_len);
#ifdef HAVE_LIBBFD_SUPPORT
+struct bpf_prog_linfo;
void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
- const char *arch, const char *disassembler_options);
+ const char *arch, const char *disassembler_options,
+ const struct btf *btf,
+ const struct bpf_prog_linfo *prog_linfo,
+ __u64 func_ksym, unsigned int func_idx,
+ bool linum);
int disasm_init(void);
#else
static inline
void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
- const char *arch, const char *disassembler_options)
+ const char *arch, const char *disassembler_options,
+ const struct btf *btf,
+ const struct bpf_prog_linfo *prog_linfo,
+ __u64 func_ksym, unsigned int func_idx,
+ bool linum)
{
}
static inline int disasm_init(void)
@@ -190,6 +229,12 @@ int btf_dumper_type(const struct btf_dumper *d, __u32 type_id,
void btf_dumper_type_only(const struct btf *btf, __u32 func_type_id,
char *func_only, int size);
+void btf_dump_linfo_plain(const struct btf *btf,
+ const struct bpf_line_info *linfo,
+ const char *prefix, bool linum);
+void btf_dump_linfo_json(const struct btf *btf,
+ const struct bpf_line_info *linfo, bool linum);
+
struct nlattr;
struct ifinfomsg;
struct tcmsg;
diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c
index 3850f8d65703..8469ea6cf1c8 100644
--- a/tools/bpf/bpftool/map.c
+++ b/tools/bpf/bpftool/map.c
@@ -487,7 +487,6 @@ static int show_map_close_json(int fd, struct bpf_map_info *info)
char *memlock;
memlock = get_fdinfo(fd, "memlock");
- close(fd);
jsonw_start_object(json_wtr);
@@ -514,6 +513,30 @@ static int show_map_close_json(int fd, struct bpf_map_info *info)
jsonw_int_field(json_wtr, "bytes_memlock", atoi(memlock));
free(memlock);
+ if (info->type == BPF_MAP_TYPE_PROG_ARRAY) {
+ char *owner_prog_type = get_fdinfo(fd, "owner_prog_type");
+ char *owner_jited = get_fdinfo(fd, "owner_jited");
+
+ if (owner_prog_type) {
+ unsigned int prog_type = atoi(owner_prog_type);
+
+ if (prog_type < ARRAY_SIZE(prog_type_name))
+ jsonw_string_field(json_wtr, "owner_prog_type",
+ prog_type_name[prog_type]);
+ else
+ jsonw_uint_field(json_wtr, "owner_prog_type",
+ prog_type);
+ }
+ if (atoi(owner_jited))
+ jsonw_bool_field(json_wtr, "owner_jited", true);
+ else
+ jsonw_bool_field(json_wtr, "owner_jited", false);
+
+ free(owner_prog_type);
+ free(owner_jited);
+ }
+ close(fd);
+
if (!hash_empty(map_table.table)) {
struct pinned_obj *obj;
@@ -536,7 +559,6 @@ static int show_map_close_plain(int fd, struct bpf_map_info *info)
char *memlock;
memlock = get_fdinfo(fd, "memlock");
- close(fd);
printf("%u: ", info->id);
if (info->type < ARRAY_SIZE(map_type_name))
@@ -557,6 +579,30 @@ static int show_map_close_plain(int fd, struct bpf_map_info *info)
printf(" memlock %sB", memlock);
free(memlock);
+ if (info->type == BPF_MAP_TYPE_PROG_ARRAY) {
+ char *owner_prog_type = get_fdinfo(fd, "owner_prog_type");
+ char *owner_jited = get_fdinfo(fd, "owner_jited");
+
+ printf("\n\t");
+ if (owner_prog_type) {
+ unsigned int prog_type = atoi(owner_prog_type);
+
+ if (prog_type < ARRAY_SIZE(prog_type_name))
+ printf("owner_prog_type %s ",
+ prog_type_name[prog_type]);
+ else
+ printf("owner_prog_type %d ", prog_type);
+ }
+ if (atoi(owner_jited))
+ printf("owner jited");
+ else
+ printf("owner not jited");
+
+ free(owner_prog_type);
+ free(owner_jited);
+ }
+ close(fd);
+
printf("\n");
if (!hash_empty(map_table.table)) {
struct pinned_obj *obj;
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index 4e04699a2350..b73b4e473948 100644
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -54,30 +54,6 @@
#include "main.h"
#include "xlated_dumper.h"
-static const char * const prog_type_name[] = {
- [BPF_PROG_TYPE_UNSPEC] = "unspec",
- [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter",
- [BPF_PROG_TYPE_KPROBE] = "kprobe",
- [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls",
- [BPF_PROG_TYPE_SCHED_ACT] = "sched_act",
- [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint",
- [BPF_PROG_TYPE_XDP] = "xdp",
- [BPF_PROG_TYPE_PERF_EVENT] = "perf_event",
- [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb",
- [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock",
- [BPF_PROG_TYPE_LWT_IN] = "lwt_in",
- [BPF_PROG_TYPE_LWT_OUT] = "lwt_out",
- [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit",
- [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops",
- [BPF_PROG_TYPE_SK_SKB] = "sk_skb",
- [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device",
- [BPF_PROG_TYPE_SK_MSG] = "sk_msg",
- [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
- [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr",
- [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2",
- [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector",
-};
-
static const char * const attach_type_strings[] = {
[BPF_SK_SKB_STREAM_PARSER] = "stream_parser",
[BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",
@@ -447,24 +423,26 @@ static int do_show(int argc, char **argv)
static int do_dump(int argc, char **argv)
{
+ unsigned int finfo_rec_size, linfo_rec_size, jited_linfo_rec_size;
+ void *func_info = NULL, *linfo = NULL, *jited_linfo = NULL;
+ unsigned int nr_finfo, nr_linfo = 0, nr_jited_linfo = 0;
+ struct bpf_prog_linfo *prog_linfo = NULL;
unsigned long *func_ksyms = NULL;
struct bpf_prog_info info = {};
unsigned int *func_lens = NULL;
const char *disasm_opt = NULL;
- unsigned int finfo_rec_size;
unsigned int nr_func_ksyms;
unsigned int nr_func_lens;
struct dump_data dd = {};
__u32 len = sizeof(info);
struct btf *btf = NULL;
- void *func_info = NULL;
- unsigned int finfo_cnt;
unsigned int buf_size;
char *filepath = NULL;
bool opcodes = false;
bool visual = false;
char func_sig[1024];
unsigned char *buf;
+ bool linum = false;
__u32 *member_len;
__u64 *member_ptr;
ssize_t n;
@@ -508,6 +486,9 @@ static int do_dump(int argc, char **argv)
} else if (is_prefix(*argv, "visual")) {
visual = true;
NEXT_ARG();
+ } else if (is_prefix(*argv, "linum")) {
+ linum = true;
+ NEXT_ARG();
}
if (argc) {
@@ -556,10 +537,10 @@ static int do_dump(int argc, char **argv)
}
}
- finfo_cnt = info.func_info_cnt;
+ nr_finfo = info.nr_func_info;
finfo_rec_size = info.func_info_rec_size;
- if (finfo_cnt && finfo_rec_size) {
- func_info = malloc(finfo_cnt * finfo_rec_size);
+ if (nr_finfo && finfo_rec_size) {
+ func_info = malloc(nr_finfo * finfo_rec_size);
if (!func_info) {
p_err("mem alloc failed");
close(fd);
@@ -567,6 +548,32 @@ static int do_dump(int argc, char **argv)
}
}
+ linfo_rec_size = info.line_info_rec_size;
+ if (info.nr_line_info && linfo_rec_size && info.btf_id) {
+ nr_linfo = info.nr_line_info;
+ linfo = malloc(nr_linfo * linfo_rec_size);
+ if (!linfo) {
+ p_err("mem alloc failed");
+ close(fd);
+ goto err_free;
+ }
+ }
+
+ jited_linfo_rec_size = info.jited_line_info_rec_size;
+ if (info.nr_jited_line_info &&
+ jited_linfo_rec_size &&
+ info.nr_jited_ksyms &&
+ info.nr_jited_func_lens &&
+ info.btf_id) {
+ nr_jited_linfo = info.nr_jited_line_info;
+ jited_linfo = malloc(nr_jited_linfo * jited_linfo_rec_size);
+ if (!jited_linfo) {
+ p_err("mem alloc failed");
+ close(fd);
+ goto err_free;
+ }
+ }
+
memset(&info, 0, sizeof(info));
*member_ptr = ptr_to_u64(buf);
@@ -575,9 +582,15 @@ static int do_dump(int argc, char **argv)
info.nr_jited_ksyms = nr_func_ksyms;
info.jited_func_lens = ptr_to_u64(func_lens);
info.nr_jited_func_lens = nr_func_lens;
- info.func_info_cnt = finfo_cnt;
+ info.nr_func_info = nr_finfo;
info.func_info_rec_size = finfo_rec_size;
info.func_info = ptr_to_u64(func_info);
+ info.nr_line_info = nr_linfo;
+ info.line_info_rec_size = linfo_rec_size;
+ info.line_info = ptr_to_u64(linfo);
+ info.nr_jited_line_info = nr_jited_linfo;
+ info.jited_line_info_rec_size = jited_linfo_rec_size;
+ info.jited_line_info = ptr_to_u64(jited_linfo);
err = bpf_obj_get_info_by_fd(fd, &info, &len);
close(fd);
@@ -601,9 +614,9 @@ static int do_dump(int argc, char **argv)
goto err_free;
}
- if (info.func_info_cnt != finfo_cnt) {
- p_err("incorrect func_info_cnt %d vs. expected %d",
- info.func_info_cnt, finfo_cnt);
+ if (info.nr_func_info != nr_finfo) {
+ p_err("incorrect nr_func_info %d vs. expected %d",
+ info.nr_func_info, nr_finfo);
goto err_free;
}
@@ -613,6 +626,37 @@ static int do_dump(int argc, char **argv)
goto err_free;
}
+ if (func_info && !info.func_info) {
+ /* kernel.kptr_restrict is set. No func_info available. */
+ free(func_info);
+ func_info = NULL;
+ nr_finfo = 0;
+ }
+
+ if (linfo && info.nr_line_info != nr_linfo) {
+ p_err("incorrect nr_line_info %u vs. expected %u",
+ info.nr_line_info, nr_linfo);
+ goto err_free;
+ }
+
+ if (info.line_info_rec_size != linfo_rec_size) {
+ p_err("incorrect line_info_rec_size %u vs. expected %u",
+ info.line_info_rec_size, linfo_rec_size);
+ goto err_free;
+ }
+
+ if (jited_linfo && info.nr_jited_line_info != nr_jited_linfo) {
+ p_err("incorrect nr_jited_line_info %u vs. expected %u",
+ info.nr_jited_line_info, nr_jited_linfo);
+ goto err_free;
+ }
+
+ if (info.jited_line_info_rec_size != jited_linfo_rec_size) {
+ p_err("incorrect jited_line_info_rec_size %u vs. expected %u",
+ info.jited_line_info_rec_size, jited_linfo_rec_size);
+ goto err_free;
+ }
+
if ((member_len == &info.jited_prog_len &&
info.jited_prog_insns == 0) ||
(member_len == &info.xlated_prog_len &&
@@ -626,6 +670,12 @@ static int do_dump(int argc, char **argv)
goto err_free;
}
+ if (nr_linfo) {
+ prog_linfo = bpf_prog_linfo__new(&info);
+ if (!prog_linfo)
+ p_info("error in processing bpf_line_info. continue without it.");
+ }
+
if (filepath) {
fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600);
if (fd < 0) {
@@ -707,8 +757,11 @@ static int do_dump(int argc, char **argv)
printf("%s:\n", sym_name);
}
- disasm_print_insn(img, lens[i], opcodes, name,
- disasm_opt);
+ disasm_print_insn(img, lens[i], opcodes,
+ name, disasm_opt, btf,
+ prog_linfo, ksyms[i], i,
+ linum);
+
img += lens[i];
if (json_output)
@@ -721,7 +774,7 @@ static int do_dump(int argc, char **argv)
jsonw_end_array(json_wtr);
} else {
disasm_print_insn(buf, *member_len, opcodes, name,
- disasm_opt);
+ disasm_opt, btf, NULL, 0, 0, false);
}
} else if (visual) {
if (json_output)
@@ -735,11 +788,14 @@ static int do_dump(int argc, char **argv)
dd.btf = btf;
dd.func_info = func_info;
dd.finfo_rec_size = finfo_rec_size;
+ dd.prog_linfo = prog_linfo;
if (json_output)
- dump_xlated_json(&dd, buf, *member_len, opcodes);
+ dump_xlated_json(&dd, buf, *member_len, opcodes,
+ linum);
else
- dump_xlated_plain(&dd, buf, *member_len, opcodes);
+ dump_xlated_plain(&dd, buf, *member_len, opcodes,
+ linum);
kernel_syms_destroy(&dd);
}
@@ -747,6 +803,9 @@ static int do_dump(int argc, char **argv)
free(func_ksyms);
free(func_lens);
free(func_info);
+ free(linfo);
+ free(jited_linfo);
+ bpf_prog_linfo__free(prog_linfo);
return 0;
err_free:
@@ -754,6 +813,9 @@ err_free:
free(func_ksyms);
free(func_lens);
free(func_info);
+ free(linfo);
+ free(jited_linfo);
+ bpf_prog_linfo__free(prog_linfo);
return -1;
}
@@ -1155,8 +1217,8 @@ static int do_help(int argc, char **argv)
fprintf(stderr,
"Usage: %s %s { show | list } [PROG]\n"
- " %s %s dump xlated PROG [{ file FILE | opcodes | visual }]\n"
- " %s %s dump jited PROG [{ file FILE | opcodes }]\n"
+ " %s %s dump xlated PROG [{ file FILE | opcodes | visual | linum }]\n"
+ " %s %s dump jited PROG [{ file FILE | opcodes | linum }]\n"
" %s %s pin PROG FILE\n"
" %s %s { load | loadall } OBJ PATH \\\n"
" [type TYPE] [dev NAME] \\\n"
@@ -1164,6 +1226,7 @@ static int do_help(int argc, char **argv)
" [pinmaps MAP_DIR]\n"
" %s %s attach PROG ATTACH_TYPE [MAP]\n"
" %s %s detach PROG ATTACH_TYPE [MAP]\n"
+ " %s %s tracelog\n"
" %s %s help\n"
"\n"
" " HELP_SPEC_MAP "\n"
@@ -1172,6 +1235,7 @@ static int do_help(int argc, char **argv)
" tracepoint | raw_tracepoint | xdp | perf_event | cgroup/skb |\n"
" cgroup/sock | cgroup/dev | lwt_in | lwt_out | lwt_xmit |\n"
" lwt_seg6local | sockops | sk_skb | sk_msg | lirc_mode2 |\n"
+ " sk_reuseport | flow_dissector |\n"
" cgroup/bind4 | cgroup/bind6 | cgroup/post_bind4 |\n"
" cgroup/post_bind6 | cgroup/connect4 | cgroup/connect6 |\n"
" cgroup/sendmsg4 | cgroup/sendmsg6 }\n"
@@ -1181,7 +1245,7 @@ static int do_help(int argc, char **argv)
"",
bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2],
bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2],
- bin_name, argv[-2], bin_name, argv[-2]);
+ bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2]);
return 0;
}
@@ -1196,6 +1260,7 @@ static const struct cmd cmds[] = {
{ "loadall", do_loadall },
{ "attach", do_attach },
{ "detach", do_detach },
+ { "tracelog", do_tracelog },
{ 0 }
};
diff --git a/tools/bpf/bpftool/tracelog.c b/tools/bpf/bpftool/tracelog.c
new file mode 100644
index 000000000000..1fa8e513f590
--- /dev/null
+++ b/tools/bpf/bpftool/tracelog.c
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/* Copyright (c) 2015-2017 Daniel Borkmann */
+/* Copyright (c) 2018 Netronome Systems, Inc. */
+
+#include <errno.h>
+#include <limits.h>
+#include <signal.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <linux/magic.h>
+#include <sys/fcntl.h>
+#include <sys/vfs.h>
+
+#include "main.h"
+
+#ifndef TRACEFS_MAGIC
+# define TRACEFS_MAGIC 0x74726163
+#endif
+
+#define _textify(x) #x
+#define textify(x) _textify(x)
+
+FILE *trace_pipe_fd;
+char *buff;
+
+static int validate_tracefs_mnt(const char *mnt, unsigned long magic)
+{
+ struct statfs st_fs;
+
+ if (statfs(mnt, &st_fs) < 0)
+ return -ENOENT;
+ if ((unsigned long)st_fs.f_type != magic)
+ return -ENOENT;
+
+ return 0;
+}
+
+static bool
+find_tracefs_mnt_single(unsigned long magic, char *mnt, const char *mntpt)
+{
+ size_t src_len;
+
+ if (validate_tracefs_mnt(mntpt, magic))
+ return false;
+
+ src_len = strlen(mntpt);
+ if (src_len + 1 >= PATH_MAX) {
+ p_err("tracefs mount point name too long");
+ return false;
+ }
+
+ strcpy(mnt, mntpt);
+ return true;
+}
+
+static bool find_tracefs_pipe(char *mnt)
+{
+ static const char * const known_mnts[] = {
+ "/sys/kernel/debug/tracing",
+ "/sys/kernel/tracing",
+ "/tracing",
+ "/trace",
+ };
+ const char *pipe_name = "/trace_pipe";
+ const char *fstype = "tracefs";
+ char type[100], format[32];
+ const char * const *ptr;
+ bool found = false;
+ FILE *fp;
+
+ for (ptr = known_mnts; ptr < known_mnts + ARRAY_SIZE(known_mnts); ptr++)
+ if (find_tracefs_mnt_single(TRACEFS_MAGIC, mnt, *ptr))
+ goto exit_found;
+
+ fp = fopen("/proc/mounts", "r");
+ if (!fp)
+ return false;
+
+ /* Allow room for NULL terminating byte and pipe file name */
+ snprintf(format, sizeof(format), "%%*s %%%zds %%99s %%*s %%*d %%*d\\n",
+ PATH_MAX - strlen(pipe_name) - 1);
+ while (fscanf(fp, format, mnt, type) == 2)
+ if (strcmp(type, fstype) == 0) {
+ found = true;
+ break;
+ }
+ fclose(fp);
+
+ /* The string from fscanf() might be truncated, check mnt is valid */
+ if (!found || validate_tracefs_mnt(mnt, TRACEFS_MAGIC))
+ return false;
+
+exit_found:
+ strcat(mnt, pipe_name);
+ return true;
+}
+
+static void exit_tracelog(int signum)
+{
+ fclose(trace_pipe_fd);
+ free(buff);
+
+ if (json_output) {
+ jsonw_end_array(json_wtr);
+ jsonw_destroy(&json_wtr);
+ }
+
+ exit(0);
+}
+
+int do_tracelog(int argc, char **argv)
+{
+ const struct sigaction act = {
+ .sa_handler = exit_tracelog
+ };
+ char trace_pipe[PATH_MAX];
+ bool found_trace_pipe;
+ size_t buff_len = 0;
+
+ if (json_output)
+ jsonw_start_array(json_wtr);
+
+ found_trace_pipe = find_tracefs_pipe(trace_pipe);
+ if (!found_trace_pipe) {
+ p_err("could not find trace pipe, tracefs not mounted?");
+ return -1;
+ }
+
+ trace_pipe_fd = fopen(trace_pipe, "r");
+ if (!trace_pipe_fd) {
+ p_err("could not open trace pipe: %s", strerror(errno));
+ return -1;
+ }
+
+ sigaction(SIGHUP, &act, NULL);
+ sigaction(SIGINT, &act, NULL);
+ sigaction(SIGTERM, &act, NULL);
+ while (1) {
+ ssize_t ret;
+
+ ret = getline(&buff, &buff_len, trace_pipe_fd);
+ if (ret <= 0) {
+ p_err("failed to read content from trace pipe: %s",
+ strerror(errno));
+ break;
+ }
+ if (json_output)
+ jsonw_string(json_wtr, buff);
+ else
+ printf("%s", buff);
+ }
+
+ fclose(trace_pipe_fd);
+ free(buff);
+ return -1;
+}
diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c
index e06ac0286a75..aef628dcccb6 100644
--- a/tools/bpf/bpftool/xlated_dumper.c
+++ b/tools/bpf/bpftool/xlated_dumper.c
@@ -41,6 +41,7 @@
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
+#include <libbpf.h>
#include "disasm.h"
#include "json_writer.h"
@@ -234,8 +235,9 @@ static const char *print_imm(void *private_data,
}
void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len,
- bool opcodes)
+ bool opcodes, bool linum)
{
+ const struct bpf_prog_linfo *prog_linfo = dd->prog_linfo;
const struct bpf_insn_cbs cbs = {
.cb_print = print_insn_json,
.cb_call = print_call,
@@ -246,6 +248,7 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len,
struct bpf_insn *insn = buf;
struct btf *btf = dd->btf;
bool double_insn = false;
+ unsigned int nr_skip = 0;
char func_sig[1024];
unsigned int i;
@@ -261,7 +264,7 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len,
jsonw_start_object(json_wtr);
if (btf && record) {
- if (record->insn_offset == i) {
+ if (record->insn_off == i) {
btf_dumper_type_only(btf, record->type_id,
func_sig,
sizeof(func_sig));
@@ -273,6 +276,16 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len,
}
}
+ if (prog_linfo) {
+ const struct bpf_line_info *linfo;
+
+ linfo = bpf_prog_linfo__lfind(prog_linfo, i, nr_skip);
+ if (linfo) {
+ btf_dump_linfo_json(btf, linfo, linum);
+ nr_skip++;
+ }
+ }
+
jsonw_name(json_wtr, "disasm");
print_bpf_insn(&cbs, insn + i, true);
@@ -307,8 +320,9 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len,
}
void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len,
- bool opcodes)
+ bool opcodes, bool linum)
{
+ const struct bpf_prog_linfo *prog_linfo = dd->prog_linfo;
const struct bpf_insn_cbs cbs = {
.cb_print = print_insn,
.cb_call = print_call,
@@ -318,6 +332,7 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len,
struct bpf_func_info *record;
struct bpf_insn *insn = buf;
struct btf *btf = dd->btf;
+ unsigned int nr_skip = 0;
bool double_insn = false;
char func_sig[1024];
unsigned int i;
@@ -330,7 +345,7 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len,
}
if (btf && record) {
- if (record->insn_offset == i) {
+ if (record->insn_off == i) {
btf_dumper_type_only(btf, record->type_id,
func_sig,
sizeof(func_sig));
@@ -340,6 +355,17 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len,
}
}
+ if (prog_linfo) {
+ const struct bpf_line_info *linfo;
+
+ linfo = bpf_prog_linfo__lfind(prog_linfo, i, nr_skip);
+ if (linfo) {
+ btf_dump_linfo_plain(btf, linfo, "; ",
+ linum);
+ nr_skip++;
+ }
+ }
+
double_insn = insn[i].code == (BPF_LD | BPF_IMM | BPF_DW);
printf("% 4d: ", i);
diff --git a/tools/bpf/bpftool/xlated_dumper.h b/tools/bpf/bpftool/xlated_dumper.h
index aec31723e1e5..a24f89df8cb2 100644
--- a/tools/bpf/bpftool/xlated_dumper.h
+++ b/tools/bpf/bpftool/xlated_dumper.h
@@ -40,6 +40,8 @@
#define SYM_MAX_NAME 256
+struct bpf_prog_linfo;
+
struct kernel_sym {
unsigned long address;
char name[SYM_MAX_NAME];
@@ -54,6 +56,7 @@ struct dump_data {
struct btf *btf;
void *func_info;
__u32 finfo_rec_size;
+ const struct bpf_prog_linfo *prog_linfo;
char scratch_buff[SYM_MAX_NAME + 8];
};
@@ -61,9 +64,9 @@ void kernel_syms_load(struct dump_data *dd);
void kernel_syms_destroy(struct dump_data *dd);
struct kernel_sym *kernel_syms_search(struct dump_data *dd, unsigned long key);
void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len,
- bool opcodes);
+ bool opcodes, bool linum);
void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len,
- bool opcodes);
+ bool opcodes, bool linum);
void dump_xlated_for_graph(struct dump_data *dd, void *buf, void *buf_end,
unsigned int start_index);
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index ec8b40ff386e..94c002584068 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -232,6 +232,20 @@ enum bpf_attach_type {
*/
#define BPF_F_STRICT_ALIGNMENT (1U << 0)
+/* If BPF_F_ANY_ALIGNMENT is used in BPF_PROF_LOAD command, the
+ * verifier will allow any alignment whatsoever. On platforms
+ * with strict alignment requirements for loads ands stores (such
+ * as sparc and mips) the verifier validates that all loads and
+ * stores provably follow this requirement. This flag turns that
+ * checking and enforcement off.
+ *
+ * It is mostly used for testing when we want to validate the
+ * context and memory access aspects of the verifier, but because
+ * of an unaligned access the alignment check would trigger before
+ * the one we are interested in.
+ */
+#define BPF_F_ANY_ALIGNMENT (1U << 1)
+
/* when bpf_ldimm64->src_reg == BPF_PSEUDO_MAP_FD, bpf_ldimm64->imm == fd */
#define BPF_PSEUDO_MAP_FD 1
@@ -342,6 +356,9 @@ union bpf_attr {
__u32 func_info_rec_size; /* userspace bpf_func_info size */
__aligned_u64 func_info; /* func info */
__u32 func_info_cnt; /* number of bpf_func_info records */
+ __u32 line_info_rec_size; /* userspace bpf_line_info size */
+ __aligned_u64 line_info; /* line info */
+ __u32 line_info_cnt; /* number of bpf_line_info records */
};
struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -360,8 +377,11 @@ union bpf_attr {
struct { /* anonymous struct used by BPF_PROG_TEST_RUN command */
__u32 prog_fd;
__u32 retval;
- __u32 data_size_in;
- __u32 data_size_out;
+ __u32 data_size_in; /* input: len of data_in */
+ __u32 data_size_out; /* input/output: len of data_out
+ * returns ENOSPC if data_out
+ * is too small.
+ */
__aligned_u64 data_in;
__aligned_u64 data_out;
__u32 repeat;
@@ -2282,9 +2302,22 @@ union bpf_attr {
* if possible. Other errors can occur if input parameters are
* invalid either due to *start* byte not being valid part of msg
* payload and/or *pop* value being to large.
+ * Return
+ * 0 on success, or a negative error in case of failure.
+ *
+ * int bpf_rc_pointer_rel(void *ctx, s32 rel_x, s32 rel_y)
+ * Description
+ * This helper is used in programs implementing IR decoding, to
+ * report a successfully decoded pointer movement.
+ *
+ * The *ctx* should point to the lirc sample as passed into
+ * the program.
*
+ * This helper is only available is the kernel was compiled with
+ * the **CONFIG_BPF_LIRC_MODE2** configuration option set to
+ * "**y**".
* Return
- * 0 on success, or a negative erro in case of failure.
+ * 0
*/
#define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \
@@ -2378,7 +2411,8 @@ union bpf_attr {
FN(map_pop_elem), \
FN(map_peek_elem), \
FN(msg_push_data), \
- FN(msg_pop_data),
+ FN(msg_pop_data), \
+ FN(rc_pointer_rel),
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
* function eBPF program intends to call
@@ -2496,6 +2530,7 @@ struct __sk_buff {
__u32 data_meta;
__bpf_md_ptr(struct bpf_flow_keys *, flow_keys);
__u64 tstamp;
+ __u32 wire_len;
};
struct bpf_tunnel_key {
@@ -2674,7 +2709,13 @@ struct bpf_prog_info {
__u32 btf_id;
__u32 func_info_rec_size;
__aligned_u64 func_info;
- __u32 func_info_cnt;
+ __u32 nr_func_info;
+ __u32 nr_line_info;
+ __aligned_u64 line_info;
+ __aligned_u64 jited_line_info;
+ __u32 nr_jited_line_info;
+ __u32 line_info_rec_size;
+ __u32 jited_line_info_rec_size;
} __attribute__((aligned(8)));
struct bpf_map_info {
@@ -2987,8 +3028,18 @@ struct bpf_flow_keys {
};
struct bpf_func_info {
- __u32 insn_offset;
+ __u32 insn_off;
__u32 type_id;
};
+#define BPF_LINE_INFO_LINE_NUM(line_col) ((line_col) >> 10)
+#define BPF_LINE_INFO_LINE_COL(line_col) ((line_col) & 0x3ff)
+
+struct bpf_line_info {
+ __u32 insn_off;
+ __u32 file_name_off;
+ __u32 line_off;
+ __u32 line_col;
+};
+
#endif /* _UAPI__LINUX_BPF_H__ */
diff --git a/tools/lib/bpf/Build b/tools/lib/bpf/Build
index 7bc31c905018..197b40f5b5c6 100644
--- a/tools/lib/bpf/Build
+++ b/tools/lib/bpf/Build
@@ -1 +1 @@
-libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o netlink.o
+libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o netlink.o bpf_prog_linfo.o
diff --git a/tools/lib/bpf/README.rst b/tools/lib/bpf/README.rst
index 2ced9e061c4b..056f38310722 100644
--- a/tools/lib/bpf/README.rst
+++ b/tools/lib/bpf/README.rst
@@ -1,4 +1,4 @@
-.. SPDX-License-Identifier: GPL-2.0
+.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
libbpf API naming convention
============================
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
index ce1822194590..3caaa3428774 100644
--- a/tools/lib/bpf/bpf.c
+++ b/tools/lib/bpf/bpf.c
@@ -173,11 +173,36 @@ int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name,
-1);
}
+static void *
+alloc_zero_tailing_info(const void *orecord, __u32 cnt,
+ __u32 actual_rec_size, __u32 expected_rec_size)
+{
+ __u64 info_len = actual_rec_size * cnt;
+ void *info, *nrecord;
+ int i;
+
+ info = malloc(info_len);
+ if (!info)
+ return NULL;
+
+ /* zero out bytes kernel does not understand */
+ nrecord = info;
+ for (i = 0; i < cnt; i++) {
+ memcpy(nrecord, orecord, expected_rec_size);
+ memset(nrecord + expected_rec_size, 0,
+ actual_rec_size - expected_rec_size);
+ orecord += actual_rec_size;
+ nrecord += actual_rec_size;
+ }
+
+ return info;
+}
+
int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
char *log_buf, size_t log_buf_sz)
{
+ void *finfo = NULL, *linfo = NULL;
union bpf_attr attr;
- void *finfo = NULL;
__u32 name_len;
int fd;
@@ -201,53 +226,58 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
attr.func_info_rec_size = load_attr->func_info_rec_size;
attr.func_info_cnt = load_attr->func_info_cnt;
attr.func_info = ptr_to_u64(load_attr->func_info);
+ attr.line_info_rec_size = load_attr->line_info_rec_size;
+ attr.line_info_cnt = load_attr->line_info_cnt;
+ attr.line_info = ptr_to_u64(load_attr->line_info);
memcpy(attr.prog_name, load_attr->name,
min(name_len, BPF_OBJ_NAME_LEN - 1));
fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
- if (fd >= 0 || !log_buf || !log_buf_sz)
+ if (fd >= 0)
return fd;
/* After bpf_prog_load, the kernel may modify certain attributes
* to give user space a hint how to deal with loading failure.
* Check to see whether we can make some changes and load again.
*/
- if (errno == E2BIG && attr.func_info_cnt &&
- attr.func_info_rec_size < load_attr->func_info_rec_size) {
- __u32 actual_rec_size = load_attr->func_info_rec_size;
- __u32 expected_rec_size = attr.func_info_rec_size;
- __u32 finfo_cnt = load_attr->func_info_cnt;
- __u64 finfo_len = actual_rec_size * finfo_cnt;
- const void *orecord;
- void *nrecord;
- int i;
-
- finfo = malloc(finfo_len);
- if (!finfo)
- /* further try with log buffer won't help */
- return fd;
-
- /* zero out bytes kernel does not understand */
- orecord = load_attr->func_info;
- nrecord = finfo;
- for (i = 0; i < load_attr->func_info_cnt; i++) {
- memcpy(nrecord, orecord, expected_rec_size);
- memset(nrecord + expected_rec_size, 0,
- actual_rec_size - expected_rec_size);
- orecord += actual_rec_size;
- nrecord += actual_rec_size;
+ while (errno == E2BIG && (!finfo || !linfo)) {
+ if (!finfo && attr.func_info_cnt &&
+ attr.func_info_rec_size < load_attr->func_info_rec_size) {
+ /* try with corrected func info records */
+ finfo = alloc_zero_tailing_info(load_attr->func_info,
+ load_attr->func_info_cnt,
+ load_attr->func_info_rec_size,
+ attr.func_info_rec_size);
+ if (!finfo)
+ goto done;
+
+ attr.func_info = ptr_to_u64(finfo);
+ attr.func_info_rec_size = load_attr->func_info_rec_size;
+ } else if (!linfo && attr.line_info_cnt &&
+ attr.line_info_rec_size <
+ load_attr->line_info_rec_size) {
+ linfo = alloc_zero_tailing_info(load_attr->line_info,
+ load_attr->line_info_cnt,
+ load_attr->line_info_rec_size,
+ attr.line_info_rec_size);
+ if (!linfo)
+ goto done;
+
+ attr.line_info = ptr_to_u64(linfo);
+ attr.line_info_rec_size = load_attr->line_info_rec_size;
+ } else {
+ break;
}
- /* try with corrected func info records */
- attr.func_info = ptr_to_u64(finfo);
- attr.func_info_rec_size = load_attr->func_info_rec_size;
-
fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
- if (fd >= 0 || !log_buf || !log_buf_sz)
+ if (fd >= 0)
goto done;
}
+ if (!log_buf || !log_buf_sz)
+ goto done;
+
/* Try again with log */
attr.log_buf = ptr_to_u64(log_buf);
attr.log_size = log_buf_sz;
@@ -256,6 +286,7 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
done:
free(finfo);
+ free(linfo);
return fd;
}
@@ -279,9 +310,9 @@ int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns,
}
int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns,
- size_t insns_cnt, int strict_alignment,
- const char *license, __u32 kern_version,
- char *log_buf, size_t log_buf_sz, int log_level)
+ size_t insns_cnt, __u32 prog_flags, const char *license,
+ __u32 kern_version, char *log_buf, size_t log_buf_sz,
+ int log_level)
{
union bpf_attr attr;
@@ -295,7 +326,7 @@ int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns,
attr.log_level = log_level;
log_buf[0] = 0;
attr.kern_version = kern_version;
- attr.prog_flags = strict_alignment ? BPF_F_STRICT_ALIGNMENT : 0;
+ attr.prog_flags = prog_flags;
return sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
}
@@ -463,6 +494,29 @@ int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size,
return ret;
}
+int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr)
+{
+ union bpf_attr attr;
+ int ret;
+
+ if (!test_attr->data_out && test_attr->data_size_out > 0)
+ return -EINVAL;
+
+ bzero(&attr, sizeof(attr));
+ attr.test.prog_fd = test_attr->prog_fd;
+ attr.test.data_in = ptr_to_u64(test_attr->data_in);
+ attr.test.data_out = ptr_to_u64(test_attr->data_out);
+ attr.test.data_size_in = test_attr->data_size_in;
+ attr.test.data_size_out = test_attr->data_size_out;
+ attr.test.repeat = test_attr->repeat;
+
+ ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr));
+ test_attr->data_size_out = attr.test.data_size_out;
+ test_attr->retval = attr.test.retval;
+ test_attr->duration = attr.test.duration;
+ return ret;
+}
+
int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id)
{
union bpf_attr attr;
diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h
index 09e8bbe111d4..8f09de482839 100644
--- a/tools/lib/bpf/bpf.h
+++ b/tools/lib/bpf/bpf.h
@@ -82,6 +82,9 @@ struct bpf_load_program_attr {
__u32 func_info_rec_size;
const void *func_info;
__u32 func_info_cnt;
+ __u32 line_info_rec_size;
+ const void *line_info;
+ __u32 line_info_cnt;
};
/* Flags to direct loading requirements */
@@ -98,7 +101,7 @@ LIBBPF_API int bpf_load_program(enum bpf_prog_type type,
char *log_buf, size_t log_buf_sz);
LIBBPF_API int bpf_verify_program(enum bpf_prog_type type,
const struct bpf_insn *insns,
- size_t insns_cnt, int strict_alignment,
+ size_t insns_cnt, __u32 prog_flags,
const char *license, __u32 kern_version,
char *log_buf, size_t log_buf_sz,
int log_level);
@@ -118,6 +121,25 @@ LIBBPF_API int bpf_prog_attach(int prog_fd, int attachable_fd,
LIBBPF_API int bpf_prog_detach(int attachable_fd, enum bpf_attach_type type);
LIBBPF_API int bpf_prog_detach2(int prog_fd, int attachable_fd,
enum bpf_attach_type type);
+
+struct bpf_prog_test_run_attr {
+ int prog_fd;
+ int repeat;
+ const void *data_in;
+ __u32 data_size_in;
+ void *data_out; /* optional */
+ __u32 data_size_out; /* in: max length of data_out
+ * out: length of data_out */
+ __u32 retval; /* out: return code of the BPF program */
+ __u32 duration; /* out: average per repetition in ns */
+};
+
+LIBBPF_API int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr);
+
+/*
+ * bpf_prog_test_run does not check that data_out is large enough. Consider
+ * using bpf_prog_test_run_xattr instead.
+ */
LIBBPF_API int bpf_prog_test_run(int prog_fd, int repeat, void *data,
__u32 size, void *data_out, __u32 *size_out,
__u32 *retval, __u32 *duration);
diff --git a/tools/lib/bpf/bpf_prog_linfo.c b/tools/lib/bpf/bpf_prog_linfo.c
new file mode 100644
index 000000000000..addd6e9971cc
--- /dev/null
+++ b/tools/lib/bpf/bpf_prog_linfo.c
@@ -0,0 +1,253 @@
+// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+/* Copyright (c) 2018 Facebook */
+
+#include <string.h>
+#include <stdlib.h>
+#include <linux/err.h>
+#include <linux/bpf.h>
+#include "libbpf.h"
+
+#ifndef min
+#define min(x, y) ((x) < (y) ? (x) : (y))
+#endif
+
+struct bpf_prog_linfo {
+ void *raw_linfo;
+ void *raw_jited_linfo;
+ __u32 *nr_jited_linfo_per_func;
+ __u32 *jited_linfo_func_idx;
+ __u32 nr_linfo;
+ __u32 nr_jited_func;
+ __u32 rec_size;
+ __u32 jited_rec_size;
+};
+
+static int dissect_jited_func(struct bpf_prog_linfo *prog_linfo,
+ const __u64 *ksym_func, const __u32 *ksym_len)
+{
+ __u32 nr_jited_func, nr_linfo;
+ const void *raw_jited_linfo;
+ const __u64 *jited_linfo;
+ __u64 last_jited_linfo;
+ /*
+ * Index to raw_jited_linfo:
+ * i: Index for searching the next ksym_func
+ * prev_i: Index to the last found ksym_func
+ */
+ __u32 i, prev_i;
+ __u32 f; /* Index to ksym_func */
+
+ raw_jited_linfo = prog_linfo->raw_jited_linfo;
+ jited_linfo = raw_jited_linfo;
+ if (ksym_func[0] != *jited_linfo)
+ goto errout;
+
+ prog_linfo->jited_linfo_func_idx[0] = 0;
+ nr_jited_func = prog_linfo->nr_jited_func;
+ nr_linfo = prog_linfo->nr_linfo;
+
+ for (prev_i = 0, i = 1, f = 1;
+ i < nr_linfo && f < nr_jited_func;
+ i++) {
+ raw_jited_linfo += prog_linfo->jited_rec_size;
+ last_jited_linfo = *jited_linfo;
+ jited_linfo = raw_jited_linfo;
+
+ if (ksym_func[f] == *jited_linfo) {
+ prog_linfo->jited_linfo_func_idx[f] = i;
+
+ /* Sanity check */
+ if (last_jited_linfo - ksym_func[f - 1] + 1 >
+ ksym_len[f - 1])
+ goto errout;
+
+ prog_linfo->nr_jited_linfo_per_func[f - 1] =
+ i - prev_i;
+ prev_i = i;
+
+ /*
+ * The ksym_func[f] is found in jited_linfo.
+ * Look for the next one.
+ */
+ f++;
+ } else if (*jited_linfo <= last_jited_linfo) {
+ /* Ensure the addr is increasing _within_ a func */
+ goto errout;
+ }
+ }
+
+ if (f != nr_jited_func)
+ goto errout;
+
+ prog_linfo->nr_jited_linfo_per_func[nr_jited_func - 1] =
+ nr_linfo - prev_i;
+
+ return 0;
+
+errout:
+ return -EINVAL;
+}
+
+void bpf_prog_linfo__free(struct bpf_prog_linfo *prog_linfo)
+{
+ if (!prog_linfo)
+ return;
+
+ free(prog_linfo->raw_linfo);
+ free(prog_linfo->raw_jited_linfo);
+ free(prog_linfo->nr_jited_linfo_per_func);
+ free(prog_linfo->jited_linfo_func_idx);
+ free(prog_linfo);
+}
+
+struct bpf_prog_linfo *bpf_prog_linfo__new(const struct bpf_prog_info *info)
+{
+ struct bpf_prog_linfo *prog_linfo;
+ __u32 nr_linfo, nr_jited_func;
+
+ nr_linfo = info->nr_line_info;
+
+ /*
+ * Test !info->line_info because the kernel may NULL
+ * the ptr if kernel.kptr_restrict is set.
+ */
+ if (!nr_linfo || !info->line_info)
+ return NULL;
+
+ /*
+ * The min size that bpf_prog_linfo has to access for
+ * searching purpose.
+ */
+ if (info->line_info_rec_size <
+ offsetof(struct bpf_line_info, file_name_off))
+ return NULL;
+
+ prog_linfo = calloc(1, sizeof(*prog_linfo));
+ if (!prog_linfo)
+ return NULL;
+
+ /* Copy xlated line_info */
+ prog_linfo->nr_linfo = nr_linfo;
+ prog_linfo->rec_size = info->line_info_rec_size;
+ prog_linfo->raw_linfo = malloc(nr_linfo * prog_linfo->rec_size);
+ if (!prog_linfo->raw_linfo)
+ goto err_free;
+ memcpy(prog_linfo->raw_linfo, (void *)(long)info->line_info,
+ nr_linfo * prog_linfo->rec_size);
+
+ nr_jited_func = info->nr_jited_ksyms;
+ if (!nr_jited_func ||
+ !info->jited_line_info ||
+ info->nr_jited_line_info != nr_linfo ||
+ info->jited_line_info_rec_size < sizeof(__u64) ||
+ info->nr_jited_func_lens != nr_jited_func ||
+ !info->jited_ksyms ||
+ !info->jited_func_lens)
+ /* Not enough info to provide jited_line_info */
+ return prog_linfo;
+
+ /* Copy jited_line_info */
+ prog_linfo->nr_jited_func = nr_jited_func;
+ prog_linfo->jited_rec_size = info->jited_line_info_rec_size;
+ prog_linfo->raw_jited_linfo = malloc(nr_linfo *
+ prog_linfo->jited_rec_size);
+ if (!prog_linfo->raw_jited_linfo)
+ goto err_free;
+ memcpy(prog_linfo->raw_jited_linfo,
+ (void *)(long)info->jited_line_info,
+ nr_linfo * prog_linfo->jited_rec_size);
+
+ /* Number of jited_line_info per jited func */
+ prog_linfo->nr_jited_linfo_per_func = malloc(nr_jited_func *
+ sizeof(__u32));
+ if (!prog_linfo->nr_jited_linfo_per_func)
+ goto err_free;
+
+ /*
+ * For each jited func,
+ * the start idx to the "linfo" and "jited_linfo" array,
+ */
+ prog_linfo->jited_linfo_func_idx = malloc(nr_jited_func *
+ sizeof(__u32));
+ if (!prog_linfo->jited_linfo_func_idx)
+ goto err_free;
+
+ if (dissect_jited_func(prog_linfo,
+ (__u64 *)(long)info->jited_ksyms,
+ (__u32 *)(long)info->jited_func_lens))
+ goto err_free;
+
+ return prog_linfo;
+
+err_free:
+ bpf_prog_linfo__free(prog_linfo);
+ return NULL;
+}
+
+const struct bpf_line_info *
+bpf_prog_linfo__lfind_addr_func(const struct bpf_prog_linfo *prog_linfo,
+ __u64 addr, __u32 func_idx, __u32 nr_skip)
+{
+ __u32 jited_rec_size, rec_size, nr_linfo, start, i;
+ const void *raw_jited_linfo, *raw_linfo;
+ const __u64 *jited_linfo;
+
+ if (func_idx >= prog_linfo->nr_jited_func)
+ return NULL;
+
+ nr_linfo = prog_linfo->nr_jited_linfo_per_func[func_idx];
+ if (nr_skip >= nr_linfo)
+ return NULL;
+
+ start = prog_linfo->jited_linfo_func_idx[func_idx] + nr_skip;
+ jited_rec_size = prog_linfo->jited_rec_size;
+ raw_jited_linfo = prog_linfo->raw_jited_linfo +
+ (start * jited_rec_size);
+ jited_linfo = raw_jited_linfo;
+ if (addr < *jited_linfo)
+ return NULL;
+
+ nr_linfo -= nr_skip;
+ rec_size = prog_linfo->rec_size;
+ raw_linfo = prog_linfo->raw_linfo + (start * rec_size);
+ for (i = 0; i < nr_linfo; i++) {
+ if (addr < *jited_linfo)
+ break;
+
+ raw_linfo += rec_size;
+ raw_jited_linfo += jited_rec_size;
+ jited_linfo = raw_jited_linfo;
+ }
+
+ return raw_linfo - rec_size;
+}
+
+const struct bpf_line_info *
+bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo,
+ __u32 insn_off, __u32 nr_skip)
+{
+ const struct bpf_line_info *linfo;
+ __u32 rec_size, nr_linfo, i;
+ const void *raw_linfo;
+
+ nr_linfo = prog_linfo->nr_linfo;
+ if (nr_skip >= nr_linfo)
+ return NULL;
+
+ rec_size = prog_linfo->rec_size;
+ raw_linfo = prog_linfo->raw_linfo + (nr_skip * rec_size);
+ linfo = raw_linfo;
+ if (insn_off < linfo->insn_off)
+ return NULL;
+
+ nr_linfo -= nr_skip;
+ for (i = 0; i < nr_linfo; i++) {
+ if (insn_off < linfo->insn_off)
+ break;
+
+ raw_linfo += rec_size;
+ linfo = raw_linfo;
+ }
+
+ return raw_linfo - rec_size;
+}
diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
index c2d641f3e16e..d682d3b8f7b9 100644
--- a/tools/lib/bpf/btf.c
+++ b/tools/lib/bpf/btf.c
@@ -37,18 +37,43 @@ struct btf {
int fd;
};
+struct btf_ext_info {
+ /*
+ * info points to a deep copy of the individual info section
+ * (e.g. func_info and line_info) from the .BTF.ext.
+ * It does not include the __u32 rec_size.
+ */
+ void *info;
+ __u32 rec_size;
+ __u32 len;
+};
+
struct btf_ext {
- void *func_info;
- __u32 func_info_rec_size;
- __u32 func_info_len;
+ struct btf_ext_info func_info;
+ struct btf_ext_info line_info;
+};
+
+struct btf_ext_info_sec {
+ __u32 sec_name_off;
+ __u32 num_info;
+ /* Followed by num_info * record_size number of bytes */
+ __u8 data[0];
};
/* The minimum bpf_func_info checked by the loader */
struct bpf_func_info_min {
- __u32 insn_offset;
+ __u32 insn_off;
__u32 type_id;
};
+/* The minimum bpf_line_info checked by the loader */
+struct bpf_line_info_min {
+ __u32 insn_off;
+ __u32 file_name_off;
+ __u32 line_off;
+ __u32 line_col;
+};
+
static inline __u64 ptr_to_u64(const void *ptr)
{
return (__u64) (unsigned long) ptr;
@@ -479,71 +504,147 @@ exit_free:
return err;
}
-static int btf_ext_validate_func_info(const void *finfo, __u32 size,
- btf_print_fn_t err_log)
+struct btf_ext_sec_copy_param {
+ __u32 off;
+ __u32 len;
+ __u32 min_rec_size;
+ struct btf_ext_info *ext_info;
+ const char *desc;
+};
+
+static int btf_ext_copy_info(struct btf_ext *btf_ext,
+ __u8 *data, __u32 data_size,
+ struct btf_ext_sec_copy_param *ext_sec,
+ btf_print_fn_t err_log)
{
- int sec_hdrlen = sizeof(struct btf_sec_func_info);
- __u32 size_left, num_records, record_size;
- const struct btf_sec_func_info *sinfo;
- __u64 total_record_size;
-
- /* At least a func_info record size */
- if (size < sizeof(__u32)) {
- elog("BTF.ext func_info record size not found");
+ const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
+ const struct btf_ext_info_sec *sinfo;
+ struct btf_ext_info *ext_info;
+ __u32 info_left, record_size;
+ /* The start of the info sec (including the __u32 record_size). */
+ const void *info;
+
+ /* data and data_size do not include btf_ext_header from now on */
+ data = data + hdr->hdr_len;
+ data_size -= hdr->hdr_len;
+
+ if (ext_sec->off & 0x03) {
+ elog(".BTF.ext %s section is not aligned to 4 bytes\n",
+ ext_sec->desc);
return -EINVAL;
}
- /* The record size needs to meet below minimum standard */
- record_size = *(__u32 *)finfo;
- if (record_size < sizeof(struct bpf_func_info_min) ||
- record_size % sizeof(__u32)) {
- elog("BTF.ext func_info invalid record size");
+ if (data_size < ext_sec->off ||
+ ext_sec->len > data_size - ext_sec->off) {
+ elog("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n",
+ ext_sec->desc, ext_sec->off, ext_sec->len);
return -EINVAL;
}
- sinfo = finfo + sizeof(__u32);
- size_left = size - sizeof(__u32);
+ info = data + ext_sec->off;
+ info_left = ext_sec->len;
- /* If no func_info records, return failure now so .BTF.ext
- * won't be used.
- */
- if (!size_left) {
- elog("BTF.ext no func info records");
+ /* At least a record size */
+ if (info_left < sizeof(__u32)) {
+ elog(".BTF.ext %s record size not found\n", ext_sec->desc);
+ return -EINVAL;
+ }
+
+ /* The record size needs to meet the minimum standard */
+ record_size = *(__u32 *)info;
+ if (record_size < ext_sec->min_rec_size ||
+ record_size & 0x03) {
+ elog("%s section in .BTF.ext has invalid record size %u\n",
+ ext_sec->desc, record_size);
+ return -EINVAL;
+ }
+
+ sinfo = info + sizeof(__u32);
+ info_left -= sizeof(__u32);
+
+ /* If no records, return failure now so .BTF.ext won't be used. */
+ if (!info_left) {
+ elog("%s section in .BTF.ext has no records", ext_sec->desc);
return -EINVAL;
}
- while (size_left) {
- if (size_left < sec_hdrlen) {
- elog("BTF.ext func_info header not found");
+ while (info_left) {
+ unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
+ __u64 total_record_size;
+ __u32 num_records;
+
+ if (info_left < sec_hdrlen) {
+ elog("%s section header is not found in .BTF.ext\n",
+ ext_sec->desc);
return -EINVAL;
}
- num_records = sinfo->num_func_info;
+ num_records = sinfo->num_info;
if (num_records == 0) {
- elog("incorrect BTF.ext num_func_info");
+ elog("%s section has incorrect num_records in .BTF.ext\n",
+ ext_sec->desc);
return -EINVAL;
}
total_record_size = sec_hdrlen +
(__u64)num_records * record_size;
- if (size_left < total_record_size) {
- elog("incorrect BTF.ext num_func_info");
+ if (info_left < total_record_size) {
+ elog("%s section has incorrect num_records in .BTF.ext\n",
+ ext_sec->desc);
return -EINVAL;
}
- size_left -= total_record_size;
+ info_left -= total_record_size;
sinfo = (void *)sinfo + total_record_size;
}
+ ext_info = ext_sec->ext_info;
+ ext_info->len = ext_sec->len - sizeof(__u32);
+ ext_info->rec_size = record_size;
+ ext_info->info = malloc(ext_info->len);
+ if (!ext_info->info)
+ return -ENOMEM;
+ memcpy(ext_info->info, info + sizeof(__u32), ext_info->len);
+
return 0;
}
+static int btf_ext_copy_func_info(struct btf_ext *btf_ext,
+ __u8 *data, __u32 data_size,
+ btf_print_fn_t err_log)
+{
+ const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
+ struct btf_ext_sec_copy_param param = {
+ .off = hdr->func_info_off,
+ .len = hdr->func_info_len,
+ .min_rec_size = sizeof(struct bpf_func_info_min),
+ .ext_info = &btf_ext->func_info,
+ .desc = "func_info"
+ };
+
+ return btf_ext_copy_info(btf_ext, data, data_size, &param, err_log);
+}
+
+static int btf_ext_copy_line_info(struct btf_ext *btf_ext,
+ __u8 *data, __u32 data_size,
+ btf_print_fn_t err_log)
+{
+ const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
+ struct btf_ext_sec_copy_param param = {
+ .off = hdr->line_info_off,
+ .len = hdr->line_info_len,
+ .min_rec_size = sizeof(struct bpf_line_info_min),
+ .ext_info = &btf_ext->line_info,
+ .desc = "line_info",
+ };
+
+ return btf_ext_copy_info(btf_ext, data, data_size, &param, err_log);
+}
+
static int btf_ext_parse_hdr(__u8 *data, __u32 data_size,
btf_print_fn_t err_log)
{
const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
- __u32 meta_left, last_func_info_pos;
- void *finfo;
if (data_size < offsetof(struct btf_ext_header, func_info_off) ||
data_size < hdr->hdr_len) {
@@ -566,34 +667,12 @@ static int btf_ext_parse_hdr(__u8 *data, __u32 data_size,
return -ENOTSUP;
}
- meta_left = data_size - hdr->hdr_len;
- if (!meta_left) {
+ if (data_size == hdr->hdr_len) {
elog("BTF.ext has no data\n");
return -EINVAL;
}
- if (meta_left < hdr->func_info_off) {
- elog("Invalid BTF.ext func_info section offset:%u\n",
- hdr->func_info_off);
- return -EINVAL;
- }
-
- if (hdr->func_info_off & 0x03) {
- elog("BTF.ext func_info section is not aligned to 4 bytes\n");
- return -EINVAL;
- }
-
- last_func_info_pos = hdr->hdr_len + hdr->func_info_off +
- hdr->func_info_len;
- if (last_func_info_pos > data_size) {
- elog("Invalid BTF.ext func_info section size:%u\n",
- hdr->func_info_len);
- return -EINVAL;
- }
-
- finfo = data + hdr->hdr_len + hdr->func_info_off;
- return btf_ext_validate_func_info(finfo, hdr->func_info_len,
- err_log);
+ return 0;
}
void btf_ext__free(struct btf_ext *btf_ext)
@@ -601,16 +680,14 @@ void btf_ext__free(struct btf_ext *btf_ext)
if (!btf_ext)
return;
- free(btf_ext->func_info);
+ free(btf_ext->func_info.info);
+ free(btf_ext->line_info.info);
free(btf_ext);
}
struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log)
{
- const struct btf_ext_header *hdr;
struct btf_ext *btf_ext;
- void *org_fdata, *fdata;
- __u32 hdrlen, size_u32;
int err;
err = btf_ext_parse_hdr(data, size, err_log);
@@ -621,42 +698,38 @@ struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log)
if (!btf_ext)
return ERR_PTR(-ENOMEM);
- hdr = (const struct btf_ext_header *)data;
- hdrlen = hdr->hdr_len;
- size_u32 = sizeof(__u32);
- fdata = malloc(hdr->func_info_len - size_u32);
- if (!fdata) {
- free(btf_ext);
- return ERR_PTR(-ENOMEM);
+ err = btf_ext_copy_func_info(btf_ext, data, size, err_log);
+ if (err) {
+ btf_ext__free(btf_ext);
+ return ERR_PTR(err);
}
- /* remember record size and copy rest of func_info data */
- org_fdata = data + hdrlen + hdr->func_info_off;
- btf_ext->func_info_rec_size = *(__u32 *)org_fdata;
- memcpy(fdata, org_fdata + size_u32, hdr->func_info_len - size_u32);
- btf_ext->func_info = fdata;
- btf_ext->func_info_len = hdr->func_info_len - size_u32;
+ err = btf_ext_copy_line_info(btf_ext, data, size, err_log);
+ if (err) {
+ btf_ext__free(btf_ext);
+ return ERR_PTR(err);
+ }
return btf_ext;
}
-int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext,
- const char *sec_name, void **func_info,
- __u32 *func_info_rec_size, __u32 *func_info_len)
+static int btf_ext_reloc_info(const struct btf *btf,
+ const struct btf_ext_info *ext_info,
+ const char *sec_name, __u32 insns_cnt,
+ void **info, __u32 *cnt)
{
- __u32 sec_hdrlen = sizeof(struct btf_sec_func_info);
- __u32 i, record_size, records_len;
- struct btf_sec_func_info *sinfo;
+ __u32 sec_hdrlen = sizeof(struct btf_ext_info_sec);
+ __u32 i, record_size, existing_len, records_len;
+ struct btf_ext_info_sec *sinfo;
const char *info_sec_name;
- __s64 remain_len;
+ __u64 remain_len;
void *data;
- record_size = btf_ext->func_info_rec_size;
- sinfo = btf_ext->func_info;
- remain_len = btf_ext->func_info_len;
-
+ record_size = ext_info->rec_size;
+ sinfo = ext_info->info;
+ remain_len = ext_info->len;
while (remain_len > 0) {
- records_len = sinfo->num_func_info * record_size;
+ records_len = sinfo->num_info * record_size;
info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off);
if (strcmp(info_sec_name, sec_name)) {
remain_len -= sec_hdrlen + records_len;
@@ -664,79 +737,52 @@ int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext,
continue;
}
- data = malloc(records_len);
+ existing_len = (*cnt) * record_size;
+ data = realloc(*info, existing_len + records_len);
if (!data)
return -ENOMEM;
- memcpy(data, sinfo->data, records_len);
-
- /* adjust the insn_offset, the data in .BTF.ext is
- * the actual byte offset, and the kernel expects
- * the offset in term of bpf_insn.
- *
- * adjust the insn offset only, the rest data will
- * be passed to kernel.
+ memcpy(data + existing_len, sinfo->data, records_len);
+ /* adjust insn_off only, the rest data will be passed
+ * to the kernel.
*/
- for (i = 0; i < sinfo->num_func_info; i++) {
- struct bpf_func_info_min *record;
+ for (i = 0; i < sinfo->num_info; i++) {
+ __u32 *insn_off;
- record = data + i * record_size;
- record->insn_offset /= sizeof(struct bpf_insn);
+ insn_off = data + existing_len + (i * record_size);
+ *insn_off = *insn_off / sizeof(struct bpf_insn) +
+ insns_cnt;
}
-
- *func_info = data;
- *func_info_len = records_len;
- *func_info_rec_size = record_size;
+ *info = data;
+ *cnt += sinfo->num_info;
return 0;
}
- return -EINVAL;
+ return -ENOENT;
}
-int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext,
- const char *sec_name, __u32 insns_cnt,
- void **func_info, __u32 *func_info_len)
+int btf_ext__reloc_func_info(const struct btf *btf, const struct btf_ext *btf_ext,
+ const char *sec_name, __u32 insns_cnt,
+ void **func_info, __u32 *cnt)
{
- __u32 sec_hdrlen = sizeof(struct btf_sec_func_info);
- __u32 i, record_size, existing_flen, records_len;
- struct btf_sec_func_info *sinfo;
- const char *info_sec_name;
- __u64 remain_len;
- void *data;
-
- record_size = btf_ext->func_info_rec_size;
- sinfo = btf_ext->func_info;
- remain_len = btf_ext->func_info_len;
- while (remain_len > 0) {
- records_len = sinfo->num_func_info * record_size;
- info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off);
- if (strcmp(info_sec_name, sec_name)) {
- remain_len -= sec_hdrlen + records_len;
- sinfo = (void *)sinfo + sec_hdrlen + records_len;
- continue;
- }
-
- existing_flen = *func_info_len;
- data = realloc(*func_info, existing_flen + records_len);
- if (!data)
- return -ENOMEM;
+ return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name,
+ insns_cnt, func_info, cnt);
+}
- memcpy(data + existing_flen, sinfo->data, records_len);
- /* adjust insn_offset only, the rest data will be passed
- * to the kernel.
- */
- for (i = 0; i < sinfo->num_func_info; i++) {
- struct bpf_func_info_min *record;
+int btf_ext__reloc_line_info(const struct btf *btf, const struct btf_ext *btf_ext,
+ const char *sec_name, __u32 insns_cnt,
+ void **line_info, __u32 *cnt)
+{
+ return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name,
+ insns_cnt, line_info, cnt);
+}
- record = data + existing_flen + i * record_size;
- record->insn_offset =
- record->insn_offset / sizeof(struct bpf_insn) +
- insns_cnt;
- }
- *func_info = data;
- *func_info_len = existing_flen + records_len;
- return 0;
- }
+__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext)
+{
+ return btf_ext->func_info.rec_size;
+}
- return -EINVAL;
+__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext)
+{
+ return btf_ext->line_info.rec_size;
}
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h
index 5336b2f37293..b0610dcdae6b 100644
--- a/tools/lib/bpf/btf.h
+++ b/tools/lib/bpf/btf.h
@@ -51,13 +51,8 @@ struct btf_ext_header {
/* All offsets are in bytes relative to the end of this header */
__u32 func_info_off;
__u32 func_info_len;
-};
-
-struct btf_sec_func_info {
- __u32 sec_name_off;
- __u32 num_func_info;
- /* Followed by num_func_info number of bpf func_info records */
- __u8 data[0];
+ __u32 line_info_off;
+ __u32 line_info_len;
};
typedef int (*btf_print_fn_t)(const char *, ...)
@@ -77,12 +72,16 @@ LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf);
struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log);
void btf_ext__free(struct btf_ext *btf_ext);
-int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext,
- const char *sec_name, void **func_info,
- __u32 *func_info_rec_size, __u32 *func_info_len);
-int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext,
- const char *sec_name, __u32 insns_cnt, void **func_info,
- __u32 *func_info_len);
+int btf_ext__reloc_func_info(const struct btf *btf,
+ const struct btf_ext *btf_ext,
+ const char *sec_name, __u32 insns_cnt,
+ void **func_info, __u32 *func_info_len);
+int btf_ext__reloc_line_info(const struct btf *btf,
+ const struct btf_ext *btf_ext,
+ const char *sec_name, __u32 insns_cnt,
+ void **line_info, __u32 *cnt);
+__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext);
+__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext);
#ifdef __cplusplus
} /* extern "C" */
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 59b748ebd15f..e2bc75ee1614 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -167,9 +167,13 @@ struct bpf_program {
int btf_fd;
void *func_info;
__u32 func_info_rec_size;
- __u32 func_info_len;
+ __u32 func_info_cnt;
struct bpf_capabilities *caps;
+
+ void *line_info;
+ __u32 line_info_rec_size;
+ __u32 line_info_cnt;
};
struct bpf_map {
@@ -779,6 +783,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
{
Elf *elf = obj->efile.elf;
GElf_Ehdr *ep = &obj->efile.ehdr;
+ Elf_Data *btf_ext_data = NULL;
Elf_Scn *scn = NULL;
int idx = 0, err = 0;
@@ -841,14 +846,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
obj->btf = NULL;
}
} else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
- obj->btf_ext = btf_ext__new(data->d_buf, data->d_size,
- __pr_debug);
- if (IS_ERR(obj->btf_ext)) {
- pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
- BTF_EXT_ELF_SEC,
- PTR_ERR(obj->btf_ext));
- obj->btf_ext = NULL;
- }
+ btf_ext_data = data;
} else if (sh.sh_type == SHT_SYMTAB) {
if (obj->efile.symbols) {
pr_warning("bpf: multiple SYMTAB in %s\n",
@@ -910,6 +908,22 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
pr_warning("Corrupted ELF file: index of strtab invalid\n");
return LIBBPF_ERRNO__FORMAT;
}
+ if (btf_ext_data) {
+ if (!obj->btf) {
+ pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
+ BTF_EXT_ELF_SEC, BTF_ELF_SEC);
+ } else {
+ obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
+ btf_ext_data->d_size,
+ __pr_debug);
+ if (IS_ERR(obj->btf_ext)) {
+ pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
+ BTF_EXT_ELF_SEC,
+ PTR_ERR(obj->btf_ext));
+ obj->btf_ext = NULL;
+ }
+ }
+ }
if (obj->efile.maps_shndx >= 0) {
err = bpf_object__init_maps(obj, flags);
if (err)
@@ -1276,6 +1290,82 @@ bpf_object__create_maps(struct bpf_object *obj)
}
static int
+check_btf_ext_reloc_err(struct bpf_program *prog, int err,
+ void *btf_prog_info, const char *info_name)
+{
+ if (err != -ENOENT) {
+ pr_warning("Error in loading %s for sec %s.\n",
+ info_name, prog->section_name);
+ return err;
+ }
+
+ /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */
+
+ if (btf_prog_info) {
+ /*
+ * Some info has already been found but has problem
+ * in the last btf_ext reloc. Must have to error
+ * out.
+ */
+ pr_warning("Error in relocating %s for sec %s.\n",
+ info_name, prog->section_name);
+ return err;
+ }
+
+ /*
+ * Have problem loading the very first info. Ignore
+ * the rest.
+ */
+ pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n",
+ info_name, prog->section_name, info_name);
+ return 0;
+}
+
+static int
+bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj,
+ const char *section_name, __u32 insn_offset)
+{
+ int err;
+
+ if (!insn_offset || prog->func_info) {
+ /*
+ * !insn_offset => main program
+ *
+ * For sub prog, the main program's func_info has to
+ * be loaded first (i.e. prog->func_info != NULL)
+ */
+ err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext,
+ section_name, insn_offset,
+ &prog->func_info,
+ &prog->func_info_cnt);
+ if (err)
+ return check_btf_ext_reloc_err(prog, err,
+ prog->func_info,
+ "bpf_func_info");
+
+ prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext);
+ }
+
+ if (!insn_offset || prog->line_info) {
+ err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext,
+ section_name, insn_offset,
+ &prog->line_info,
+ &prog->line_info_cnt);
+ if (err)
+ return check_btf_ext_reloc_err(prog, err,
+ prog->line_info,
+ "bpf_line_info");
+
+ prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext);
+ }
+
+ if (!insn_offset)
+ prog->btf_fd = btf__fd(obj->btf);
+
+ return 0;
+}
+
+static int
bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
struct reloc_desc *relo)
{
@@ -1306,17 +1396,12 @@ bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
return -ENOMEM;
}
- if (obj->btf && obj->btf_ext) {
- err = btf_ext__reloc(obj->btf, obj->btf_ext,
- text->section_name,
- prog->insns_cnt,
- &prog->func_info,
- &prog->func_info_len);
- if (err) {
- pr_warning("error in btf_ext__reloc for sec %s\n",
- text->section_name);
+ if (obj->btf_ext) {
+ err = bpf_program_reloc_btf_ext(prog, obj,
+ text->section_name,
+ prog->insns_cnt);
+ if (err)
return err;
- }
}
memcpy(new_insn + prog->insns_cnt, text->insns,
@@ -1341,18 +1426,11 @@ bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
if (!prog)
return 0;
- if (obj->btf && obj->btf_ext) {
- err = btf_ext__reloc_init(obj->btf, obj->btf_ext,
- prog->section_name,
- &prog->func_info,
- &prog->func_info_rec_size,
- &prog->func_info_len);
- if (err) {
- pr_warning("err in btf_ext__reloc_init for sec %s\n",
- prog->section_name);
+ if (obj->btf_ext) {
+ err = bpf_program_reloc_btf_ext(prog, obj,
+ prog->section_name, 0);
+ if (err)
return err;
- }
- prog->btf_fd = btf__fd(obj->btf);
}
if (!prog->reloc_desc)
@@ -1444,8 +1522,7 @@ static int bpf_object__collect_reloc(struct bpf_object *obj)
static int
load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
- char *license, __u32 kern_version, int *pfd,
- __u32 func_info_cnt)
+ char *license, __u32 kern_version, int *pfd)
{
struct bpf_load_program_attr load_attr;
char *cp, errmsg[STRERR_BUFSIZE];
@@ -1465,8 +1542,10 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0;
load_attr.func_info = prog->func_info;
load_attr.func_info_rec_size = prog->func_info_rec_size;
- load_attr.func_info_cnt = func_info_cnt;
-
+ load_attr.func_info_cnt = prog->func_info_cnt;
+ load_attr.line_info = prog->line_info;
+ load_attr.line_info_rec_size = prog->line_info_rec_size;
+ load_attr.line_info_cnt = prog->line_info_cnt;
if (!load_attr.insns || !load_attr.insns_cnt)
return -EINVAL;
@@ -1523,14 +1602,8 @@ int
bpf_program__load(struct bpf_program *prog,
char *license, __u32 kern_version)
{
- __u32 func_info_cnt;
int err = 0, fd, i;
- if (prog->func_info_len == 0)
- func_info_cnt = 0;
- else
- func_info_cnt = prog->func_info_len / prog->func_info_rec_size;
-
if (prog->instances.nr < 0 || !prog->instances.fds) {
if (prog->preprocessor) {
pr_warning("Internal error: can't load program '%s'\n",
@@ -1553,8 +1626,7 @@ bpf_program__load(struct bpf_program *prog,
prog->section_name, prog->instances.nr);
}
err = load_program(prog, prog->insns, prog->insns_cnt,
- license, kern_version, &fd,
- func_info_cnt);
+ license, kern_version, &fd);
if (!err)
prog->instances.fds[0] = fd;
goto out;
@@ -1584,8 +1656,7 @@ bpf_program__load(struct bpf_program *prog,
err = load_program(prog, result.new_insn_ptr,
result.new_insn_cnt,
- license, kern_version, &fd,
- func_info_cnt);
+ license, kern_version, &fd);
if (err) {
pr_warning("Loading the %dth instance of program '%s' failed\n",
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
index f30c3d07bb7d..5f68d7b75215 100644
--- a/tools/lib/bpf/libbpf.h
+++ b/tools/lib/bpf/libbpf.h
@@ -342,6 +342,19 @@ int libbpf_nl_get_qdisc(int sock, unsigned int nl_pid, int ifindex,
int libbpf_nl_get_filter(int sock, unsigned int nl_pid, int ifindex, int handle,
libbpf_dump_nlmsg_t dump_filter_nlmsg, void *cookie);
+struct bpf_prog_linfo;
+struct bpf_prog_info;
+
+LIBBPF_API void bpf_prog_linfo__free(struct bpf_prog_linfo *prog_linfo);
+LIBBPF_API struct bpf_prog_linfo *
+bpf_prog_linfo__new(const struct bpf_prog_info *info);
+LIBBPF_API const struct bpf_line_info *
+bpf_prog_linfo__lfind_addr_func(const struct bpf_prog_linfo *prog_linfo,
+ __u64 addr, __u32 func_idx, __u32 nr_skip);
+LIBBPF_API const struct bpf_line_info *
+bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo,
+ __u32 insn_off, __u32 nr_skip);
+
#ifdef __cplusplus
} /* extern "C" */
#endif
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
index 4fb29f6d7a80..cd02cd4e2cc3 100644
--- a/tools/lib/bpf/libbpf.map
+++ b/tools/lib/bpf/libbpf.map
@@ -65,6 +65,7 @@ LIBBPF_0.0.1 {
bpf_prog_load_xattr;
bpf_prog_query;
bpf_prog_test_run;
+ bpf_prog_test_run_xattr;
bpf_program__fd;
bpf_program__is_kprobe;
bpf_program__is_perf_event;
@@ -98,6 +99,10 @@ LIBBPF_0.0.1 {
bpf_program__unload;
bpf_program__unpin;
bpf_program__unpin_instance;
+ bpf_prog_linfo__free;
+ bpf_prog_linfo__new;
+ bpf_prog_linfo__lfind_addr_func;
+ bpf_prog_linfo__lfind;
bpf_raw_tracepoint_open;
bpf_set_link_xdp_fd;
bpf_task_fd_query;
diff --git a/tools/testing/selftests/bpf/bpf_helpers.h b/tools/testing/selftests/bpf/bpf_helpers.h
index d3eef9d82100..6c77cf7bedce 100644
--- a/tools/testing/selftests/bpf/bpf_helpers.h
+++ b/tools/testing/selftests/bpf/bpf_helpers.h
@@ -170,6 +170,8 @@ static int (*bpf_skb_vlan_push)(void *ctx, __be16 vlan_proto, __u16 vlan_tci) =
(void *) BPF_FUNC_skb_vlan_push;
static int (*bpf_skb_vlan_pop)(void *ctx) =
(void *) BPF_FUNC_skb_vlan_pop;
+static int (*bpf_rc_pointer_rel)(void *ctx, int rel_x, int rel_y) =
+ (void *) BPF_FUNC_rc_pointer_rel;
/* llvm builtin functions that eBPF C program may use to
* emit BPF_LD_ABS and BPF_LD_IND instructions
diff --git a/tools/testing/selftests/bpf/test_align.c b/tools/testing/selftests/bpf/test_align.c
index 5f377ec53f2f..3c789d03b629 100644
--- a/tools/testing/selftests/bpf/test_align.c
+++ b/tools/testing/selftests/bpf/test_align.c
@@ -620,8 +620,8 @@ static int do_test_single(struct bpf_align_test *test)
prog_len = probe_filter_length(prog);
fd_prog = bpf_verify_program(prog_type ? : BPF_PROG_TYPE_SOCKET_FILTER,
- prog, prog_len, 1, "GPL", 0,
- bpf_vlog, sizeof(bpf_vlog), 2);
+ prog, prog_len, BPF_F_STRICT_ALIGNMENT,
+ "GPL", 0, bpf_vlog, sizeof(bpf_vlog), 2);
if (fd_prog < 0 && test->result != REJECT) {
printf("Failed to load program.\n");
printf("%s", bpf_vlog);
diff --git a/tools/testing/selftests/bpf/test_btf.c b/tools/testing/selftests/bpf/test_btf.c
index bded6c2a97d3..f570e0a39959 100644
--- a/tools/testing/selftests/bpf/test_btf.c
+++ b/tools/testing/selftests/bpf/test_btf.c
@@ -6,6 +6,7 @@
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/filter.h>
+#include <linux/unistd.h>
#include <bpf/bpf.h>
#include <sys/resource.h>
#include <libelf.h>
@@ -107,19 +108,20 @@ static int __base_pr(const char *format, ...)
#define BTF_END_RAW 0xdeadbeef
#define NAME_TBD 0xdeadb33f
-#define MAX_NR_RAW_TYPES 1024
+#define MAX_NR_RAW_U32 1024
#define BTF_LOG_BUF_SIZE 65535
static struct args {
unsigned int raw_test_num;
unsigned int file_test_num;
unsigned int get_info_test_num;
+ unsigned int info_raw_test_num;
bool raw_test;
bool file_test;
bool get_info_test;
bool pprint_test;
bool always_log;
- bool func_type_test;
+ bool info_raw_test;
} args;
static char btf_log_buf[BTF_LOG_BUF_SIZE];
@@ -135,7 +137,7 @@ struct btf_raw_test {
const char *str_sec;
const char *map_name;
const char *err_str;
- __u32 raw_types[MAX_NR_RAW_TYPES];
+ __u32 raw_types[MAX_NR_RAW_U32];
__u32 str_sec_size;
enum bpf_map_type map_type;
__u32 key_size;
@@ -154,6 +156,9 @@ struct btf_raw_test {
int str_len_delta;
};
+#define BTF_STR_SEC(str) \
+ .str_sec = str, .str_sec_size = sizeof(str)
+
static struct btf_raw_test raw_tests[] = {
/* enum E {
* E0,
@@ -2217,11 +2222,11 @@ static const char *get_next_str(const char *start, const char *end)
return start < end - 1 ? start + 1 : NULL;
}
-static int get_type_sec_size(const __u32 *raw_types)
+static int get_raw_sec_size(const __u32 *raw_types)
{
int i;
- for (i = MAX_NR_RAW_TYPES - 1;
+ for (i = MAX_NR_RAW_U32 - 1;
i >= 0 && raw_types[i] != BTF_END_RAW;
i--)
;
@@ -2233,7 +2238,8 @@ static void *btf_raw_create(const struct btf_header *hdr,
const __u32 *raw_types,
const char *str,
unsigned int str_sec_size,
- unsigned int *btf_size)
+ unsigned int *btf_size,
+ const char **ret_next_str)
{
const char *next_str = str, *end_str = str + str_sec_size;
unsigned int size_needed, offset;
@@ -2242,7 +2248,7 @@ static void *btf_raw_create(const struct btf_header *hdr,
uint32_t *ret_types;
void *raw_btf;
- type_sec_size = get_type_sec_size(raw_types);
+ type_sec_size = get_raw_sec_size(raw_types);
if (CHECK(type_sec_size < 0, "Cannot get nr_raw_types"))
return NULL;
@@ -2281,6 +2287,8 @@ static void *btf_raw_create(const struct btf_header *hdr,
ret_hdr->str_len = str_sec_size;
*btf_size = size_needed;
+ if (ret_next_str)
+ *ret_next_str = next_str;
return raw_btf;
}
@@ -2300,7 +2308,7 @@ static int do_test_raw(unsigned int test_num)
test->raw_types,
test->str_sec,
test->str_sec_size,
- &raw_btf_size);
+ &raw_btf_size, NULL);
if (!raw_btf)
return -1;
@@ -2377,7 +2385,7 @@ static int test_raw(void)
struct btf_get_info_test {
const char *descr;
const char *str_sec;
- __u32 raw_types[MAX_NR_RAW_TYPES];
+ __u32 raw_types[MAX_NR_RAW_U32];
__u32 str_sec_size;
int btf_size_delta;
int (*special_test)(unsigned int test_num);
@@ -2457,7 +2465,7 @@ static int test_big_btf_info(unsigned int test_num)
test->raw_types,
test->str_sec,
test->str_sec_size,
- &raw_btf_size);
+ &raw_btf_size, NULL);
if (!raw_btf)
return -1;
@@ -2541,7 +2549,7 @@ static int test_btf_id(unsigned int test_num)
test->raw_types,
test->str_sec,
test->str_sec_size,
- &raw_btf_size);
+ &raw_btf_size, NULL);
if (!raw_btf)
return -1;
@@ -2679,7 +2687,7 @@ static int do_test_get_info(unsigned int test_num)
test->raw_types,
test->str_sec,
test->str_sec_size,
- &raw_btf_size);
+ &raw_btf_size, NULL);
if (!raw_btf)
return -1;
@@ -2901,9 +2909,9 @@ static int do_test_file(unsigned int test_num)
err = -1;
goto done;
}
- if (CHECK(info.func_info_cnt != 3,
- "incorrect info.func_info_cnt (1st) %d",
- info.func_info_cnt)) {
+ if (CHECK(info.nr_func_info != 3,
+ "incorrect info.nr_func_info (1st) %d",
+ info.nr_func_info)) {
err = -1;
goto done;
}
@@ -2914,7 +2922,7 @@ static int do_test_file(unsigned int test_num)
goto done;
}
- func_info = malloc(info.func_info_cnt * rec_size);
+ func_info = malloc(info.nr_func_info * rec_size);
if (CHECK(!func_info, "out of memory")) {
err = -1;
goto done;
@@ -2922,7 +2930,7 @@ static int do_test_file(unsigned int test_num)
/* reset info to only retrieve func_info related data */
memset(&info, 0, sizeof(info));
- info.func_info_cnt = 3;
+ info.nr_func_info = 3;
info.func_info_rec_size = rec_size;
info.func_info = ptr_to_u64(func_info);
@@ -2933,9 +2941,9 @@ static int do_test_file(unsigned int test_num)
err = -1;
goto done;
}
- if (CHECK(info.func_info_cnt != 3,
- "incorrect info.func_info_cnt (2nd) %d",
- info.func_info_cnt)) {
+ if (CHECK(info.nr_func_info != 3,
+ "incorrect info.nr_func_info (2nd) %d",
+ info.nr_func_info)) {
err = -1;
goto done;
}
@@ -3208,7 +3216,7 @@ static int do_test_pprint(void)
fprintf(stderr, "%s......", test->descr);
raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types,
test->str_sec, test->str_sec_size,
- &raw_btf_size);
+ &raw_btf_size, NULL);
if (!raw_btf)
return -1;
@@ -3412,18 +3420,25 @@ static int test_pprint(void)
return err;
}
-static struct btf_func_type_test {
+#define BPF_LINE_INFO_ENC(insn_off, file_off, line_off, line_num, line_col) \
+ (insn_off), (file_off), (line_off), ((line_num) << 10 | ((line_col) & 0x3ff))
+
+static struct prog_info_raw_test {
const char *descr;
const char *str_sec;
- __u32 raw_types[MAX_NR_RAW_TYPES];
+ const char *err_str;
+ __u32 raw_types[MAX_NR_RAW_U32];
__u32 str_sec_size;
struct bpf_insn insns[MAX_INSNS];
__u32 prog_type;
__u32 func_info[MAX_SUBPROGS][2];
__u32 func_info_rec_size;
__u32 func_info_cnt;
+ __u32 line_info[MAX_NR_RAW_U32];
+ __u32 line_info_rec_size;
+ __u32 nr_jited_ksyms;
bool expected_prog_load_failure;
-} func_type_test[] = {
+} info_raw_tests[] = {
{
.descr = "func_type (main func + one sub)",
.raw_types = {
@@ -3452,6 +3467,7 @@ static struct btf_func_type_test {
.func_info = { {0, 5}, {3, 6} },
.func_info_rec_size = 8,
.func_info_cnt = 2,
+ .line_info = { BTF_END_RAW },
},
{
@@ -3482,6 +3498,7 @@ static struct btf_func_type_test {
.func_info = { {0, 5}, {3, 6} },
.func_info_rec_size = 4,
.func_info_cnt = 2,
+ .line_info = { BTF_END_RAW },
.expected_prog_load_failure = true,
},
@@ -3513,11 +3530,12 @@ static struct btf_func_type_test {
.func_info = { {0, 5}, {3, 6} },
.func_info_rec_size = 8,
.func_info_cnt = 1,
+ .line_info = { BTF_END_RAW },
.expected_prog_load_failure = true,
},
{
- .descr = "func_type (Incorrect bpf_func_info.insn_offset)",
+ .descr = "func_type (Incorrect bpf_func_info.insn_off)",
.raw_types = {
BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
BTF_TYPE_INT_ENC(NAME_TBD, 0, 0, 32, 4), /* [2] */
@@ -3544,6 +3562,278 @@ static struct btf_func_type_test {
.func_info = { {0, 5}, {2, 6} },
.func_info_rec_size = 8,
.func_info_cnt = 2,
+ .line_info = { BTF_END_RAW },
+ .expected_prog_load_failure = true,
+},
+
+{
+ .descr = "line_info (No subprog)",
+ .raw_types = {
+ BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
+ BTF_END_RAW,
+ },
+ BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"),
+ .insns = {
+ BPF_MOV64_IMM(BPF_REG_0, 1),
+ BPF_MOV64_IMM(BPF_REG_1, 2),
+ BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
+ .func_info_cnt = 0,
+ .line_info = {
+ BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
+ BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9),
+ BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8),
+ BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7),
+ BTF_END_RAW,
+ },
+ .line_info_rec_size = sizeof(struct bpf_line_info),
+ .nr_jited_ksyms = 1,
+},
+
+{
+ .descr = "line_info (No subprog. insn_off >= prog->len)",
+ .raw_types = {
+ BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
+ BTF_END_RAW,
+ },
+ BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"),
+ .insns = {
+ BPF_MOV64_IMM(BPF_REG_0, 1),
+ BPF_MOV64_IMM(BPF_REG_1, 2),
+ BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
+ .func_info_cnt = 0,
+ .line_info = {
+ BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
+ BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9),
+ BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8),
+ BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7),
+ BPF_LINE_INFO_ENC(4, 0, 0, 5, 6),
+ BTF_END_RAW,
+ },
+ .line_info_rec_size = sizeof(struct bpf_line_info),
+ .nr_jited_ksyms = 1,
+ .err_str = "line_info[4].insn_off",
+ .expected_prog_load_failure = true,
+},
+
+{
+ .descr = "line_info (No subprog. zero tailing line_info",
+ .raw_types = {
+ BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
+ BTF_END_RAW,
+ },
+ BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"),
+ .insns = {
+ BPF_MOV64_IMM(BPF_REG_0, 1),
+ BPF_MOV64_IMM(BPF_REG_1, 2),
+ BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
+ .func_info_cnt = 0,
+ .line_info = {
+ BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), 0,
+ BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9), 0,
+ BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8), 0,
+ BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7), 0,
+ BTF_END_RAW,
+ },
+ .line_info_rec_size = sizeof(struct bpf_line_info) + sizeof(__u32),
+ .nr_jited_ksyms = 1,
+},
+
+{
+ .descr = "line_info (No subprog. nonzero tailing line_info)",
+ .raw_types = {
+ BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
+ BTF_END_RAW,
+ },
+ BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"),
+ .insns = {
+ BPF_MOV64_IMM(BPF_REG_0, 1),
+ BPF_MOV64_IMM(BPF_REG_1, 2),
+ BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
+ .func_info_cnt = 0,
+ .line_info = {
+ BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), 0,
+ BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9), 0,
+ BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8), 0,
+ BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7), 1,
+ BTF_END_RAW,
+ },
+ .line_info_rec_size = sizeof(struct bpf_line_info) + sizeof(__u32),
+ .nr_jited_ksyms = 1,
+ .err_str = "nonzero tailing record in line_info",
+ .expected_prog_load_failure = true,
+},
+
+{
+ .descr = "line_info (subprog)",
+ .raw_types = {
+ BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
+ BTF_END_RAW,
+ },
+ BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"),
+ .insns = {
+ BPF_MOV64_IMM(BPF_REG_2, 1),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+ BPF_CALL_REL(1),
+ BPF_EXIT_INSN(),
+ BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
+ .func_info_cnt = 0,
+ .line_info = {
+ BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
+ BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9),
+ BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 3, 8),
+ BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7),
+ BTF_END_RAW,
+ },
+ .line_info_rec_size = sizeof(struct bpf_line_info),
+ .nr_jited_ksyms = 2,
+},
+
+{
+ .descr = "line_info (subprog + func_info)",
+ .raw_types = {
+ BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
+ BTF_FUNC_PROTO_ENC(1, 1), /* [2] */
+ BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1),
+ BTF_FUNC_ENC(NAME_TBD, 2), /* [3] */
+ BTF_FUNC_ENC(NAME_TBD, 2), /* [4] */
+ BTF_END_RAW,
+ },
+ BTF_STR_SEC("\0int\0x\0sub\0main\0int a=1+1;\0return func(a);\0b+=1;\0return b;"),
+ .insns = {
+ BPF_MOV64_IMM(BPF_REG_2, 1),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+ BPF_CALL_REL(1),
+ BPF_EXIT_INSN(),
+ BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
+ .func_info_cnt = 2,
+ .func_info_rec_size = 8,
+ .func_info = { {0, 4}, {5, 3} },
+ .line_info = {
+ BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
+ BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9),
+ BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 3, 8),
+ BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7),
+ BTF_END_RAW,
+ },
+ .line_info_rec_size = sizeof(struct bpf_line_info),
+ .nr_jited_ksyms = 2,
+},
+
+{
+ .descr = "line_info (subprog. missing 1st func line info)",
+ .raw_types = {
+ BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
+ BTF_END_RAW,
+ },
+ BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"),
+ .insns = {
+ BPF_MOV64_IMM(BPF_REG_2, 1),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+ BPF_CALL_REL(1),
+ BPF_EXIT_INSN(),
+ BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
+ .func_info_cnt = 0,
+ .line_info = {
+ BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 1, 10),
+ BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9),
+ BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 3, 8),
+ BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7),
+ BTF_END_RAW,
+ },
+ .line_info_rec_size = sizeof(struct bpf_line_info),
+ .nr_jited_ksyms = 2,
+ .err_str = "missing bpf_line_info for func#0",
+ .expected_prog_load_failure = true,
+},
+
+{
+ .descr = "line_info (subprog. missing 2nd func line info)",
+ .raw_types = {
+ BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
+ BTF_END_RAW,
+ },
+ BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"),
+ .insns = {
+ BPF_MOV64_IMM(BPF_REG_2, 1),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+ BPF_CALL_REL(1),
+ BPF_EXIT_INSN(),
+ BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
+ .func_info_cnt = 0,
+ .line_info = {
+ BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
+ BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9),
+ BPF_LINE_INFO_ENC(6, 0, NAME_TBD, 3, 8),
+ BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7),
+ BTF_END_RAW,
+ },
+ .line_info_rec_size = sizeof(struct bpf_line_info),
+ .nr_jited_ksyms = 2,
+ .err_str = "missing bpf_line_info for func#1",
+ .expected_prog_load_failure = true,
+},
+
+{
+ .descr = "line_info (subprog. unordered insn offset)",
+ .raw_types = {
+ BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
+ BTF_END_RAW,
+ },
+ BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"),
+ .insns = {
+ BPF_MOV64_IMM(BPF_REG_2, 1),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1),
+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_2),
+ BPF_CALL_REL(1),
+ BPF_EXIT_INSN(),
+ BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
+ .func_info_cnt = 0,
+ .line_info = {
+ BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10),
+ BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 2, 9),
+ BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8),
+ BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7),
+ BTF_END_RAW,
+ },
+ .line_info_rec_size = sizeof(struct bpf_line_info),
+ .nr_jited_ksyms = 2,
+ .err_str = "Invalid line_info[2].insn_off",
.expected_prog_load_failure = true,
},
@@ -3559,90 +3849,84 @@ static size_t probe_prog_length(const struct bpf_insn *fp)
return len + 1;
}
-static int do_test_func_type(int test_num)
+static __u32 *patch_name_tbd(const __u32 *raw_u32,
+ const char *str, __u32 str_off,
+ unsigned int str_sec_size,
+ unsigned int *ret_size)
{
- const struct btf_func_type_test *test = &func_type_test[test_num];
- unsigned int raw_btf_size, info_len, rec_size;
- int i, btf_fd = -1, prog_fd = -1, err = 0;
- struct bpf_load_program_attr attr = {};
- void *raw_btf, *func_info = NULL;
- struct bpf_prog_info info = {};
- struct bpf_func_info *finfo;
+ int i, raw_u32_size = get_raw_sec_size(raw_u32);
+ const char *end_str = str + str_sec_size;
+ const char *next_str = str + str_off;
+ __u32 *new_u32 = NULL;
- fprintf(stderr, "%s......", test->descr);
- raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types,
- test->str_sec, test->str_sec_size,
- &raw_btf_size);
+ if (raw_u32_size == -1)
+ return ERR_PTR(-EINVAL);
- if (!raw_btf)
- return -1;
+ if (!raw_u32_size) {
+ *ret_size = 0;
+ return NULL;
+ }
- *btf_log_buf = '\0';
- btf_fd = bpf_load_btf(raw_btf, raw_btf_size,
- btf_log_buf, BTF_LOG_BUF_SIZE,
- args.always_log);
- free(raw_btf);
+ new_u32 = malloc(raw_u32_size);
+ if (!new_u32)
+ return ERR_PTR(-ENOMEM);
- if (CHECK(btf_fd == -1, "invalid btf_fd errno:%d", errno)) {
- err = -1;
- goto done;
+ for (i = 0; i < raw_u32_size / sizeof(raw_u32[0]); i++) {
+ if (raw_u32[i] == NAME_TBD) {
+ next_str = get_next_str(next_str, end_str);
+ if (CHECK(!next_str, "Error in getting next_str\n")) {
+ free(new_u32);
+ return ERR_PTR(-EINVAL);
+ }
+ new_u32[i] = next_str - str;
+ next_str += strlen(next_str);
+ } else {
+ new_u32[i] = raw_u32[i];
+ }
}
- if (*btf_log_buf && args.always_log)
- fprintf(stderr, "\n%s", btf_log_buf);
-
- attr.prog_type = test->prog_type;
- attr.insns = test->insns;
- attr.insns_cnt = probe_prog_length(attr.insns);
- attr.license = "GPL";
- attr.prog_btf_fd = btf_fd;
- attr.func_info_rec_size = test->func_info_rec_size;
- attr.func_info_cnt = test->func_info_cnt;
- attr.func_info = test->func_info;
+ *ret_size = raw_u32_size;
+ return new_u32;
+}
- *btf_log_buf = '\0';
- prog_fd = bpf_load_program_xattr(&attr, btf_log_buf,
- BTF_LOG_BUF_SIZE);
- if (test->expected_prog_load_failure && prog_fd == -1) {
- err = 0;
- goto done;
- }
- if (CHECK(prog_fd == -1, "invalid prog_id errno:%d", errno)) {
- fprintf(stderr, "%s\n", btf_log_buf);
- err = -1;
- goto done;
- }
+static int test_get_finfo(const struct prog_info_raw_test *test,
+ int prog_fd)
+{
+ struct bpf_prog_info info = {};
+ struct bpf_func_info *finfo;
+ __u32 info_len, rec_size, i;
+ void *func_info = NULL;
+ int err;
/* get necessary lens */
info_len = sizeof(struct bpf_prog_info);
err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len);
if (CHECK(err == -1, "invalid get info (1st) errno:%d", errno)) {
fprintf(stderr, "%s\n", btf_log_buf);
- err = -1;
- goto done;
+ return -1;
}
- if (CHECK(info.func_info_cnt != 2,
- "incorrect info.func_info_cnt (1st) %d\n",
- info.func_info_cnt)) {
- err = -1;
- goto done;
+ if (CHECK(info.nr_func_info != test->func_info_cnt,
+ "incorrect info.nr_func_info (1st) %d",
+ info.nr_func_info)) {
+ return -1;
}
+
rec_size = info.func_info_rec_size;
- if (CHECK(rec_size < 4,
- "incorrect info.func_info_rec_size (1st) %d\n", rec_size)) {
- err = -1;
- goto done;
+ if (CHECK(rec_size < 8,
+ "incorrect info.func_info_rec_size (1st) %d", rec_size)) {
+ return -1;
}
- func_info = malloc(info.func_info_cnt * rec_size);
- if (CHECK(!func_info, "out of memory")) {
- err = -1;
- goto done;
- }
+ if (!info.nr_func_info)
+ return 0;
+
+ func_info = malloc(info.nr_func_info * rec_size);
+ if (CHECK(!func_info, "out of memory"))
+ return -1;
/* reset info to only retrieve func_info related data */
memset(&info, 0, sizeof(info));
- info.func_info_cnt = 2;
+ info.nr_func_info = test->func_info_cnt;
info.func_info_rec_size = rec_size;
info.func_info = ptr_to_u64(func_info);
err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len);
@@ -3651,21 +3935,27 @@ static int do_test_func_type(int test_num)
err = -1;
goto done;
}
- if (CHECK(info.func_info_cnt != 2,
- "incorrect info.func_info_cnt (2nd) %d\n",
- info.func_info_cnt)) {
+ if (CHECK(info.nr_func_info != test->func_info_cnt,
+ "incorrect info.nr_func_info (2nd) %d",
+ info.nr_func_info)) {
err = -1;
goto done;
}
- if (CHECK(info.func_info_rec_size != rec_size,
- "incorrect info.func_info_rec_size (2nd) %d\n",
+ if (CHECK(info.func_info_rec_size < 8,
+ "incorrect info.func_info_rec_size (2nd) %d",
info.func_info_rec_size)) {
err = -1;
goto done;
}
+ if (CHECK(!info.func_info,
+ "info.func_info == 0. kernel.kptr_restrict is set?")) {
+ err = -1;
+ goto done;
+ }
+
finfo = func_info;
- for (i = 0; i < 2; i++) {
+ for (i = 0; i < test->func_info_cnt; i++) {
if (CHECK(finfo->type_id != test->func_info[i][1],
"incorrect func_type %u expected %u",
finfo->type_id, test->func_info[i][1])) {
@@ -3675,7 +3965,307 @@ static int do_test_func_type(int test_num)
finfo = (void *)finfo + rec_size;
}
+ err = 0;
+
done:
+ free(func_info);
+ return err;
+}
+
+static int test_get_linfo(const struct prog_info_raw_test *test,
+ const void *patched_linfo,
+ __u32 cnt, int prog_fd)
+{
+ __u32 i, info_len, nr_jited_ksyms, nr_jited_func_lens;
+ __u64 *jited_linfo = NULL, *jited_ksyms = NULL;
+ __u32 rec_size, jited_rec_size, jited_cnt;
+ struct bpf_line_info *linfo = NULL;
+ __u32 cur_func_len, ksyms_found;
+ struct bpf_prog_info info = {};
+ __u32 *jited_func_lens = NULL;
+ __u64 cur_func_ksyms;
+ int err;
+
+ jited_cnt = cnt;
+ rec_size = sizeof(*linfo);
+ jited_rec_size = sizeof(*jited_linfo);
+ if (test->nr_jited_ksyms)
+ nr_jited_ksyms = test->nr_jited_ksyms;
+ else
+ nr_jited_ksyms = test->func_info_cnt;
+ nr_jited_func_lens = nr_jited_ksyms;
+
+ info_len = sizeof(struct bpf_prog_info);
+ err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len);
+ if (CHECK(err == -1, "err:%d errno:%d", err, errno)) {
+ err = -1;
+ goto done;
+ }
+
+ if (!info.jited_prog_len) {
+ /* prog is not jited */
+ jited_cnt = 0;
+ nr_jited_ksyms = 1;
+ nr_jited_func_lens = 1;
+ }
+
+ if (CHECK(info.nr_line_info != cnt ||
+ info.nr_jited_line_info != jited_cnt ||
+ info.nr_jited_ksyms != nr_jited_ksyms ||
+ info.nr_jited_func_lens != nr_jited_func_lens ||
+ (!info.nr_line_info && info.nr_jited_line_info),
+ "info: nr_line_info:%u(expected:%u) nr_jited_line_info:%u(expected:%u) nr_jited_ksyms:%u(expected:%u) nr_jited_func_lens:%u(expected:%u)",
+ info.nr_line_info, cnt,
+ info.nr_jited_line_info, jited_cnt,
+ info.nr_jited_ksyms, nr_jited_ksyms,
+ info.nr_jited_func_lens, nr_jited_func_lens)) {
+ err = -1;
+ goto done;
+ }
+
+ if (CHECK(info.line_info_rec_size < 16 ||
+ info.jited_line_info_rec_size < 8,
+ "info: line_info_rec_size:%u(userspace expected:%u) jited_line_info_rec_size:%u(userspace expected:%u)",
+ info.line_info_rec_size, rec_size,
+ info.jited_line_info_rec_size, jited_rec_size)) {
+ err = -1;
+ goto done;
+ }
+
+ if (!cnt)
+ return 0;
+
+ rec_size = info.line_info_rec_size;
+ jited_rec_size = info.jited_line_info_rec_size;
+
+ memset(&info, 0, sizeof(info));
+
+ linfo = calloc(cnt, rec_size);
+ if (CHECK(!linfo, "!linfo")) {
+ err = -1;
+ goto done;
+ }
+ info.nr_line_info = cnt;
+ info.line_info_rec_size = rec_size;
+ info.line_info = ptr_to_u64(linfo);
+
+ if (jited_cnt) {
+ jited_linfo = calloc(jited_cnt, jited_rec_size);
+ jited_ksyms = calloc(nr_jited_ksyms, sizeof(*jited_ksyms));
+ jited_func_lens = calloc(nr_jited_func_lens,
+ sizeof(*jited_func_lens));
+ if (CHECK(!jited_linfo || !jited_ksyms || !jited_func_lens,
+ "jited_linfo:%p jited_ksyms:%p jited_func_lens:%p",
+ jited_linfo, jited_ksyms, jited_func_lens)) {
+ err = -1;
+ goto done;
+ }
+
+ info.nr_jited_line_info = jited_cnt;
+ info.jited_line_info_rec_size = jited_rec_size;
+ info.jited_line_info = ptr_to_u64(jited_linfo);
+ info.nr_jited_ksyms = nr_jited_ksyms;
+ info.jited_ksyms = ptr_to_u64(jited_ksyms);
+ info.nr_jited_func_lens = nr_jited_func_lens;
+ info.jited_func_lens = ptr_to_u64(jited_func_lens);
+ }
+
+ err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len);
+
+ /*
+ * Only recheck the info.*line_info* fields.
+ * Other fields are not the concern of this test.
+ */
+ if (CHECK(err == -1 ||
+ !info.line_info ||
+ info.nr_line_info != cnt ||
+ (jited_cnt && !info.jited_line_info) ||
+ info.nr_jited_line_info != jited_cnt ||
+ info.line_info_rec_size != rec_size ||
+ info.jited_line_info_rec_size != jited_rec_size,
+ "err:%d errno:%d info: nr_line_info:%u(expected:%u) nr_jited_line_info:%u(expected:%u) line_info_rec_size:%u(expected:%u) jited_linfo_rec_size:%u(expected:%u) line_info:%p jited_line_info:%p",
+ err, errno,
+ info.nr_line_info, cnt,
+ info.nr_jited_line_info, jited_cnt,
+ info.line_info_rec_size, rec_size,
+ info.jited_line_info_rec_size, jited_rec_size,
+ (void *)(long)info.line_info,
+ (void *)(long)info.jited_line_info)) {
+ err = -1;
+ goto done;
+ }
+
+ CHECK(linfo[0].insn_off, "linfo[0].insn_off:%u",
+ linfo[0].insn_off);
+ for (i = 1; i < cnt; i++) {
+ const struct bpf_line_info *expected_linfo;
+
+ expected_linfo = patched_linfo + (i * test->line_info_rec_size);
+ if (CHECK(linfo[i].insn_off <= linfo[i - 1].insn_off,
+ "linfo[%u].insn_off:%u <= linfo[%u].insn_off:%u",
+ i, linfo[i].insn_off,
+ i - 1, linfo[i - 1].insn_off)) {
+ err = -1;
+ goto done;
+ }
+ if (CHECK(linfo[i].file_name_off != expected_linfo->file_name_off ||
+ linfo[i].line_off != expected_linfo->line_off ||
+ linfo[i].line_col != expected_linfo->line_col,
+ "linfo[%u] (%u, %u, %u) != (%u, %u, %u)", i,
+ linfo[i].file_name_off,
+ linfo[i].line_off,
+ linfo[i].line_col,
+ expected_linfo->file_name_off,
+ expected_linfo->line_off,
+ expected_linfo->line_col)) {
+ err = -1;
+ goto done;
+ }
+ }
+
+ if (!jited_cnt) {
+ fprintf(stderr, "not jited. skipping jited_line_info check. ");
+ err = 0;
+ goto done;
+ }
+
+ if (CHECK(jited_linfo[0] != jited_ksyms[0],
+ "jited_linfo[0]:%lx != jited_ksyms[0]:%lx",
+ (long)(jited_linfo[0]), (long)(jited_ksyms[0]))) {
+ err = -1;
+ goto done;
+ }
+
+ ksyms_found = 1;
+ cur_func_len = jited_func_lens[0];
+ cur_func_ksyms = jited_ksyms[0];
+ for (i = 1; i < jited_cnt; i++) {
+ if (ksyms_found < nr_jited_ksyms &&
+ jited_linfo[i] == jited_ksyms[ksyms_found]) {
+ cur_func_ksyms = jited_ksyms[ksyms_found];
+ cur_func_len = jited_ksyms[ksyms_found];
+ ksyms_found++;
+ continue;
+ }
+
+ if (CHECK(jited_linfo[i] <= jited_linfo[i - 1],
+ "jited_linfo[%u]:%lx <= jited_linfo[%u]:%lx",
+ i, (long)jited_linfo[i],
+ i - 1, (long)(jited_linfo[i - 1]))) {
+ err = -1;
+ goto done;
+ }
+
+ if (CHECK(jited_linfo[i] - cur_func_ksyms > cur_func_len,
+ "jited_linfo[%u]:%lx - %lx > %u",
+ i, (long)jited_linfo[i], (long)cur_func_ksyms,
+ cur_func_len)) {
+ err = -1;
+ goto done;
+ }
+ }
+
+ if (CHECK(ksyms_found != nr_jited_ksyms,
+ "ksyms_found:%u != nr_jited_ksyms:%u",
+ ksyms_found, nr_jited_ksyms)) {
+ err = -1;
+ goto done;
+ }
+
+ err = 0;
+
+done:
+ free(linfo);
+ free(jited_linfo);
+ free(jited_ksyms);
+ free(jited_func_lens);
+ return err;
+}
+
+static int do_test_info_raw(unsigned int test_num)
+{
+ const struct prog_info_raw_test *test = &info_raw_tests[test_num - 1];
+ unsigned int raw_btf_size, linfo_str_off, linfo_size;
+ int btf_fd = -1, prog_fd = -1, err = 0;
+ void *raw_btf, *patched_linfo = NULL;
+ const char *ret_next_str;
+ union bpf_attr attr = {};
+
+ fprintf(stderr, "BTF prog info raw test[%u] (%s): ", test_num, test->descr);
+ raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types,
+ test->str_sec, test->str_sec_size,
+ &raw_btf_size, &ret_next_str);
+
+ if (!raw_btf)
+ return -1;
+
+ *btf_log_buf = '\0';
+ btf_fd = bpf_load_btf(raw_btf, raw_btf_size,
+ btf_log_buf, BTF_LOG_BUF_SIZE,
+ args.always_log);
+ free(raw_btf);
+
+ if (CHECK(btf_fd == -1, "invalid btf_fd errno:%d", errno)) {
+ err = -1;
+ goto done;
+ }
+
+ if (*btf_log_buf && args.always_log)
+ fprintf(stderr, "\n%s", btf_log_buf);
+ *btf_log_buf = '\0';
+
+ linfo_str_off = ret_next_str - test->str_sec;
+ patched_linfo = patch_name_tbd(test->line_info,
+ test->str_sec, linfo_str_off,
+ test->str_sec_size, &linfo_size);
+ if (IS_ERR(patched_linfo)) {
+ fprintf(stderr, "error in creating raw bpf_line_info");
+ err = -1;
+ goto done;
+ }
+
+ attr.prog_type = test->prog_type;
+ attr.insns = ptr_to_u64(test->insns);
+ attr.insn_cnt = probe_prog_length(test->insns);
+ attr.license = ptr_to_u64("GPL");
+ attr.prog_btf_fd = btf_fd;
+ attr.func_info_rec_size = test->func_info_rec_size;
+ attr.func_info_cnt = test->func_info_cnt;
+ attr.func_info = ptr_to_u64(test->func_info);
+ attr.log_buf = ptr_to_u64(btf_log_buf);
+ attr.log_size = BTF_LOG_BUF_SIZE;
+ attr.log_level = 1;
+ if (linfo_size) {
+ attr.line_info_rec_size = test->line_info_rec_size;
+ attr.line_info = ptr_to_u64(patched_linfo);
+ attr.line_info_cnt = linfo_size / attr.line_info_rec_size;
+ }
+
+ prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr));
+ err = ((prog_fd == -1) != test->expected_prog_load_failure);
+ if (CHECK(err, "prog_fd:%d expected_prog_load_failure:%u errno:%d",
+ prog_fd, test->expected_prog_load_failure, errno) ||
+ CHECK(test->err_str && !strstr(btf_log_buf, test->err_str),
+ "expected err_str:%s", test->err_str)) {
+ err = -1;
+ goto done;
+ }
+
+ if (prog_fd == -1)
+ goto done;
+
+ err = test_get_finfo(test, prog_fd);
+ if (err)
+ goto done;
+
+ err = test_get_linfo(test, patched_linfo, attr.line_info_cnt, prog_fd);
+ if (err)
+ goto done;
+
+done:
+ if (!err)
+ fprintf(stderr, "OK");
+
if (*btf_log_buf && (err || args.always_log))
fprintf(stderr, "\n%s", btf_log_buf);
@@ -3683,33 +4273,41 @@ done:
close(btf_fd);
if (prog_fd != -1)
close(prog_fd);
- free(func_info);
+
+ if (!IS_ERR(patched_linfo))
+ free(patched_linfo);
+
return err;
}
-static int test_func_type(void)
+static int test_info_raw(void)
{
unsigned int i;
int err = 0;
- for (i = 0; i < ARRAY_SIZE(func_type_test); i++)
- err |= count_result(do_test_func_type(i));
+ if (args.info_raw_test_num)
+ return count_result(do_test_info_raw(args.info_raw_test_num));
+
+ for (i = 1; i <= ARRAY_SIZE(info_raw_tests); i++)
+ err |= count_result(do_test_info_raw(i));
return err;
}
static void usage(const char *cmd)
{
- fprintf(stderr, "Usage: %s [-l] [[-r test_num (1 - %zu)] |"
- " [-g test_num (1 - %zu)] |"
- " [-f test_num (1 - %zu)] | [-p] | [-k] ]\n",
+ fprintf(stderr, "Usage: %s [-l] [[-r btf_raw_test_num (1 - %zu)] |\n"
+ "\t[-g btf_get_info_test_num (1 - %zu)] |\n"
+ "\t[-f btf_file_test_num (1 - %zu)] |\n"
+ "\t[-k btf_prog_info_raw_test_num (1 - %zu)] |\n"
+ "\t[-p (pretty print test)]]\n",
cmd, ARRAY_SIZE(raw_tests), ARRAY_SIZE(get_info_tests),
- ARRAY_SIZE(file_tests));
+ ARRAY_SIZE(file_tests), ARRAY_SIZE(info_raw_tests));
}
static int parse_args(int argc, char **argv)
{
- const char *optstr = "lpkf:r:g:";
+ const char *optstr = "lpk:f:r:g:";
int opt;
while ((opt = getopt(argc, argv, optstr)) != -1) {
@@ -3733,7 +4331,8 @@ static int parse_args(int argc, char **argv)
args.pprint_test = true;
break;
case 'k':
- args.func_type_test = true;
+ args.info_raw_test_num = atoi(optarg);
+ args.info_raw_test = true;
break;
case 'h':
usage(argv[0]);
@@ -3768,6 +4367,14 @@ static int parse_args(int argc, char **argv)
return -1;
}
+ if (args.info_raw_test_num &&
+ (args.info_raw_test_num < 1 ||
+ args.info_raw_test_num > ARRAY_SIZE(info_raw_tests))) {
+ fprintf(stderr, "BTF prog info raw test number must be [1 - %zu]\n",
+ ARRAY_SIZE(info_raw_tests));
+ return -1;
+ }
+
return 0;
}
@@ -3800,16 +4407,17 @@ int main(int argc, char **argv)
if (args.pprint_test)
err |= test_pprint();
- if (args.func_type_test)
- err |= test_func_type();
+ if (args.info_raw_test)
+ err |= test_info_raw();
if (args.raw_test || args.get_info_test || args.file_test ||
- args.pprint_test || args.func_type_test)
+ args.pprint_test || args.info_raw_test)
goto done;
err |= test_raw();
err |= test_get_info();
err |= test_file();
+ err |= test_info_raw();
done:
print_summary();
diff --git a/tools/testing/selftests/bpf/test_lirc_mode2.sh b/tools/testing/selftests/bpf/test_lirc_mode2.sh
index 677686198df3..ec4e15948e40 100755
--- a/tools/testing/selftests/bpf/test_lirc_mode2.sh
+++ b/tools/testing/selftests/bpf/test_lirc_mode2.sh
@@ -21,13 +21,14 @@ do
if grep -q DRV_NAME=rc-loopback $i/uevent
then
LIRCDEV=$(grep DEVNAME= $i/lirc*/uevent | sed sQDEVNAME=Q/dev/Q)
+ INPUTDEV=$(grep DEVNAME= $i/input*/event*/uevent | sed sQDEVNAME=Q/dev/Q)
fi
done
if [ -n $LIRCDEV ];
then
TYPE=lirc_mode2
- ./test_lirc_mode2_user $LIRCDEV
+ ./test_lirc_mode2_user $LIRCDEV $INPUTDEV
ret=$?
if [ $ret -ne 0 ]; then
echo -e ${RED}"FAIL: $TYPE"${NC}
diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_kern.c b/tools/testing/selftests/bpf/test_lirc_mode2_kern.c
index ba26855563a5..4147130cc3b7 100644
--- a/tools/testing/selftests/bpf/test_lirc_mode2_kern.c
+++ b/tools/testing/selftests/bpf/test_lirc_mode2_kern.c
@@ -15,6 +15,9 @@ int bpf_decoder(unsigned int *sample)
if (duration & 0x10000)
bpf_rc_keydown(sample, 0x40, duration & 0xffff, 0);
+ if (duration & 0x20000)
+ bpf_rc_pointer_rel(sample, (duration >> 8) & 0xff,
+ duration & 0xff);
}
return 0;
diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_user.c b/tools/testing/selftests/bpf/test_lirc_mode2_user.c
index d470d63c33db..fb5fd6841ef3 100644
--- a/tools/testing/selftests/bpf/test_lirc_mode2_user.c
+++ b/tools/testing/selftests/bpf/test_lirc_mode2_user.c
@@ -29,6 +29,7 @@
#include <linux/bpf.h>
#include <linux/lirc.h>
+#include <linux/input.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
@@ -47,12 +48,13 @@
int main(int argc, char **argv)
{
struct bpf_object *obj;
- int ret, lircfd, progfd, mode;
- int testir = 0x1dead;
+ int ret, lircfd, progfd, inputfd;
+ int testir1 = 0x1dead;
+ int testir2 = 0x20101;
u32 prog_ids[10], prog_flags[10], prog_cnt;
- if (argc != 2) {
- printf("Usage: %s /dev/lircN\n", argv[0]);
+ if (argc != 3) {
+ printf("Usage: %s /dev/lircN /dev/input/eventM\n", argv[0]);
return 2;
}
@@ -76,9 +78,9 @@ int main(int argc, char **argv)
return 1;
}
- mode = LIRC_MODE_SCANCODE;
- if (ioctl(lircfd, LIRC_SET_REC_MODE, &mode)) {
- printf("failed to set rec mode: %m\n");
+ inputfd = open(argv[2], O_RDONLY | O_NONBLOCK);
+ if (inputfd == -1) {
+ printf("failed to open input device %s: %m\n", argv[1]);
return 1;
}
@@ -102,29 +104,54 @@ int main(int argc, char **argv)
}
/* Write raw IR */
- ret = write(lircfd, &testir, sizeof(testir));
- if (ret != sizeof(testir)) {
+ ret = write(lircfd, &testir1, sizeof(testir1));
+ if (ret != sizeof(testir1)) {
printf("Failed to send test IR message: %m\n");
return 1;
}
- struct pollfd pfd = { .fd = lircfd, .events = POLLIN };
- struct lirc_scancode lsc;
+ struct pollfd pfd = { .fd = inputfd, .events = POLLIN };
+ struct input_event event;
- poll(&pfd, 1, 100);
+ for (;;) {
+ poll(&pfd, 1, 100);
- /* Read decoded IR */
- ret = read(lircfd, &lsc, sizeof(lsc));
- if (ret != sizeof(lsc)) {
- printf("Failed to read decoded IR: %m\n");
- return 1;
+ /* Read decoded IR */
+ ret = read(inputfd, &event, sizeof(event));
+ if (ret != sizeof(event)) {
+ printf("Failed to read decoded IR: %m\n");
+ return 1;
+ }
+
+ if (event.type == EV_MSC && event.code == MSC_SCAN &&
+ event.value == 0xdead) {
+ break;
+ }
}
- if (lsc.scancode != 0xdead || lsc.rc_proto != 64) {
- printf("Incorrect scancode decoded\n");
+ /* Write raw IR */
+ ret = write(lircfd, &testir2, sizeof(testir2));
+ if (ret != sizeof(testir2)) {
+ printf("Failed to send test IR message: %m\n");
return 1;
}
+ for (;;) {
+ poll(&pfd, 1, 100);
+
+ /* Read decoded IR */
+ ret = read(inputfd, &event, sizeof(event));
+ if (ret != sizeof(event)) {
+ printf("Failed to read decoded IR: %m\n");
+ return 1;
+ }
+
+ if (event.type == EV_REL && event.code == REL_Y &&
+ event.value == 1 ) {
+ break;
+ }
+ }
+
prog_cnt = 10;
ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids,
&prog_cnt);
diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c
index 1c57abbe0945..26f1fdf3e2bf 100644
--- a/tools/testing/selftests/bpf/test_progs.c
+++ b/tools/testing/selftests/bpf/test_progs.c
@@ -70,7 +70,7 @@ static struct {
.tcp.urg_ptr = 123,
};
-#define CHECK(condition, tag, format...) ({ \
+#define _CHECK(condition, tag, duration, format...) ({ \
int __ret = !!(condition); \
if (__ret) { \
error_cnt++; \
@@ -83,6 +83,11 @@ static struct {
__ret; \
})
+#define CHECK(condition, tag, format...) \
+ _CHECK(condition, tag, duration, format)
+#define CHECK_ATTR(condition, tag, format...) \
+ _CHECK(condition, tag, tattr.duration, format)
+
static int bpf_find_map(const char *test, struct bpf_object *obj,
const char *name)
{
@@ -124,6 +129,53 @@ static void test_pkt_access(void)
bpf_object__close(obj);
}
+static void test_prog_run_xattr(void)
+{
+ const char *file = "./test_pkt_access.o";
+ struct bpf_object *obj;
+ char buf[10];
+ int err;
+ struct bpf_prog_test_run_attr tattr = {
+ .repeat = 1,
+ .data_in = &pkt_v4,
+ .data_size_in = sizeof(pkt_v4),
+ .data_out = buf,
+ .data_size_out = 5,
+ };
+
+ err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj,
+ &tattr.prog_fd);
+ if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno))
+ return;
+
+ memset(buf, 0, sizeof(buf));
+
+ err = bpf_prog_test_run_xattr(&tattr);
+ CHECK_ATTR(err != -1 || errno != ENOSPC || tattr.retval, "run",
+ "err %d errno %d retval %d\n", err, errno, tattr.retval);
+
+ CHECK_ATTR(tattr.data_size_out != sizeof(pkt_v4), "data_size_out",
+ "incorrect output size, want %lu have %u\n",
+ sizeof(pkt_v4), tattr.data_size_out);
+
+ CHECK_ATTR(buf[5] != 0, "overflow",
+ "BPF_PROG_TEST_RUN ignored size hint\n");
+
+ tattr.data_out = NULL;
+ tattr.data_size_out = 0;
+ errno = 0;
+
+ err = bpf_prog_test_run_xattr(&tattr);
+ CHECK_ATTR(err || errno || tattr.retval, "run_no_output",
+ "err %d errno %d retval %d\n", err, errno, tattr.retval);
+
+ tattr.data_size_out = 1;
+ err = bpf_prog_test_run_xattr(&tattr);
+ CHECK_ATTR(err != -EINVAL, "run_wrong_size_out", "err %d\n", err);
+
+ bpf_object__close(obj);
+}
+
static void test_xdp(void)
{
struct vip key4 = {.protocol = 6, .family = AF_INET};
@@ -1837,6 +1889,7 @@ int main(void)
jit_enabled = is_jit_enabled();
test_pkt_access();
+ test_prog_run_xattr();
test_xdp();
test_xdp_adjust_tail();
test_l4lb_all();
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index 1b7760d174d7..a08c67c8767e 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -721,8 +721,18 @@ static struct bpf_test tests[] = {
BPF_ALU32_IMM(BPF_ARSH, BPF_REG_0, 5),
BPF_EXIT_INSN(),
},
- .result = REJECT,
- .errstr = "unknown opcode c4",
+ .result = ACCEPT,
+ .retval = 0,
+ },
+ {
+ "arsh32 on imm 2",
+ .insns = {
+ BPF_LD_IMM64(BPF_REG_0, 0x1122334485667788),
+ BPF_ALU32_IMM(BPF_ARSH, BPF_REG_0, 7),
+ BPF_EXIT_INSN(),
+ },
+ .result = ACCEPT,
+ .retval = -16069393,
},
{
"arsh32 on reg",
@@ -732,8 +742,19 @@ static struct bpf_test tests[] = {
BPF_ALU32_REG(BPF_ARSH, BPF_REG_0, BPF_REG_1),
BPF_EXIT_INSN(),
},
- .result = REJECT,
- .errstr = "unknown opcode cc",
+ .result = ACCEPT,
+ .retval = 0,
+ },
+ {
+ "arsh32 on reg 2",
+ .insns = {
+ BPF_LD_IMM64(BPF_REG_0, 0xffff55667788),
+ BPF_MOV64_IMM(BPF_REG_1, 15),
+ BPF_ALU32_REG(BPF_ARSH, BPF_REG_0, BPF_REG_1),
+ BPF_EXIT_INSN(),
+ },
+ .result = ACCEPT,
+ .retval = 43724,
},
{
"arsh64 on imm",
@@ -1823,6 +1844,7 @@ static struct bpf_test tests[] = {
.errstr = "invalid bpf_context access",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_SK_MSG,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"direct packet read for SK_MSG",
@@ -2215,6 +2237,8 @@ static struct bpf_test tests[] = {
},
.errstr = "invalid bpf_context access",
.result = REJECT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"check cb access: half, wrong type",
@@ -2936,6 +2960,19 @@ static struct bpf_test tests[] = {
.result = ACCEPT,
},
{
+ "alu32: mov u32 const",
+ .insns = {
+ BPF_MOV32_IMM(BPF_REG_7, 0),
+ BPF_ALU32_IMM(BPF_AND, BPF_REG_7, 1),
+ BPF_MOV32_REG(BPF_REG_0, BPF_REG_7),
+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1),
+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
+ BPF_EXIT_INSN(),
+ },
+ .result = ACCEPT,
+ .retval = 0,
+ },
+ {
"unpriv: partial copy of pointer",
.insns = {
BPF_MOV32_REG(BPF_REG_1, BPF_REG_10),
@@ -3281,6 +3318,7 @@ static struct bpf_test tests[] = {
.result = REJECT,
.errstr = "R0 invalid mem access 'inv'",
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"raw_stack: skb_load_bytes, spilled regs corruption 2",
@@ -3311,6 +3349,7 @@ static struct bpf_test tests[] = {
.result = REJECT,
.errstr = "R3 invalid mem access 'inv'",
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"raw_stack: skb_load_bytes, spilled regs + data",
@@ -3810,6 +3849,7 @@ static struct bpf_test tests[] = {
.errstr = "R2 invalid mem access 'inv'",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"direct packet access: test16 (arith on data_end)",
@@ -3912,6 +3952,7 @@ static struct bpf_test tests[] = {
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = ACCEPT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"direct packet access: test21 (x += pkt_ptr, 2)",
@@ -3937,6 +3978,7 @@ static struct bpf_test tests[] = {
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = ACCEPT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"direct packet access: test22 (x += pkt_ptr, 3)",
@@ -3967,6 +4009,7 @@ static struct bpf_test tests[] = {
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = ACCEPT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"direct packet access: test23 (x += pkt_ptr, 4)",
@@ -3993,6 +4036,7 @@ static struct bpf_test tests[] = {
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = REJECT,
.errstr = "invalid access to packet, off=0 size=8, R5(id=1,off=0,r=0)",
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"direct packet access: test24 (x += pkt_ptr, 5)",
@@ -4018,6 +4062,7 @@ static struct bpf_test tests[] = {
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = ACCEPT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"direct packet access: test25 (marking on <, good access)",
@@ -5149,6 +5194,7 @@ static struct bpf_test tests[] = {
.result = REJECT,
.errstr = "invalid access to map value, value_size=64 off=-2 size=4",
.prog_type = BPF_PROG_TYPE_CGROUP_SKB,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"invalid cgroup storage access 5",
@@ -5265,6 +5311,7 @@ static struct bpf_test tests[] = {
.result = REJECT,
.errstr = "invalid access to map value, value_size=64 off=-2 size=4",
.prog_type = BPF_PROG_TYPE_CGROUP_SKB,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"invalid per-cpu cgroup storage access 5",
@@ -7206,6 +7253,7 @@ static struct bpf_test tests[] = {
.errstr = "invalid mem access 'inv'",
.result = REJECT,
.result_unpriv = REJECT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"map element value illegal alu op, 5",
@@ -7228,6 +7276,7 @@ static struct bpf_test tests[] = {
.fixup_map_hash_48b = { 3 },
.errstr = "R0 invalid mem access 'inv'",
.result = REJECT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"map element value is preserved across register spilling",
@@ -7721,6 +7770,7 @@ static struct bpf_test tests[] = {
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.retval = 0 /* csum_diff of 64-byte packet */,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"helper access to variable memory: size = 0 not allowed on NULL (!ARG_PTR_TO_MEM_OR_NULL)",
@@ -9683,6 +9733,7 @@ static struct bpf_test tests[] = {
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data' > pkt_end, bad access 1",
@@ -9720,6 +9771,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_end > pkt_data', good access",
@@ -9758,6 +9810,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_end > pkt_data', bad access 2",
@@ -9776,6 +9829,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data' < pkt_end, good access",
@@ -9814,6 +9868,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data' < pkt_end, bad access 2",
@@ -9832,6 +9887,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_end < pkt_data', good access",
@@ -9849,6 +9905,7 @@ static struct bpf_test tests[] = {
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_end < pkt_data', bad access 1",
@@ -9886,6 +9943,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data' >= pkt_end, good access",
@@ -9922,6 +9980,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data' >= pkt_end, bad access 2",
@@ -9959,6 +10018,7 @@ static struct bpf_test tests[] = {
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_end >= pkt_data', bad access 1",
@@ -9997,6 +10057,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data' <= pkt_end, good access",
@@ -10015,6 +10076,7 @@ static struct bpf_test tests[] = {
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data' <= pkt_end, bad access 1",
@@ -10053,6 +10115,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_end <= pkt_data', good access",
@@ -10089,6 +10152,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_end <= pkt_data', bad access 2",
@@ -10125,6 +10189,7 @@ static struct bpf_test tests[] = {
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_meta' > pkt_data, bad access 1",
@@ -10162,6 +10227,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data > pkt_meta', good access",
@@ -10200,6 +10266,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data > pkt_meta', bad access 2",
@@ -10218,6 +10285,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_meta' < pkt_data, good access",
@@ -10256,6 +10324,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_meta' < pkt_data, bad access 2",
@@ -10274,6 +10343,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data < pkt_meta', good access",
@@ -10291,6 +10361,7 @@ static struct bpf_test tests[] = {
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data < pkt_meta', bad access 1",
@@ -10328,6 +10399,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_meta' >= pkt_data, good access",
@@ -10364,6 +10436,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_meta' >= pkt_data, bad access 2",
@@ -10401,6 +10474,7 @@ static struct bpf_test tests[] = {
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data >= pkt_meta', bad access 1",
@@ -10439,6 +10513,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_meta' <= pkt_data, good access",
@@ -10457,6 +10532,7 @@ static struct bpf_test tests[] = {
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_meta' <= pkt_data, bad access 1",
@@ -10495,6 +10571,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data <= pkt_meta', good access",
@@ -10531,6 +10608,7 @@ static struct bpf_test tests[] = {
.errstr = "R1 offset is outside of the packet",
.result = REJECT,
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"XDP pkt read, pkt_data <= pkt_meta', bad access 2",
@@ -10635,6 +10713,7 @@ static struct bpf_test tests[] = {
},
.result = REJECT,
.errstr = "dereference of modified ctx ptr",
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"check deducing bounds from const, 8",
@@ -10648,6 +10727,7 @@ static struct bpf_test tests[] = {
},
.result = REJECT,
.errstr = "dereference of modified ctx ptr",
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"check deducing bounds from const, 9",
@@ -11122,6 +11202,7 @@ static struct bpf_test tests[] = {
.result = REJECT,
.errstr = "R6 invalid mem access 'inv'",
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: two calls with args",
@@ -11987,6 +12068,7 @@ static struct bpf_test tests[] = {
.fixup_map_hash_8b = { 12, 22 },
.result = REJECT,
.errstr = "invalid access to map value, value_size=8 off=2 size=8",
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: two calls that receive map_value via arg=ptr_stack_of_caller. test2",
@@ -12130,6 +12212,7 @@ static struct bpf_test tests[] = {
.fixup_map_hash_8b = { 12, 22 },
.result = REJECT,
.errstr = "invalid access to map value, value_size=8 off=2 size=8",
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: two calls that receive map_value_ptr_or_null via arg. test1",
@@ -12301,6 +12384,7 @@ static struct bpf_test tests[] = {
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.retval = POINTER_VALUE,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: pkt_ptr spill into caller stack 2",
@@ -12332,6 +12416,7 @@ static struct bpf_test tests[] = {
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.errstr = "invalid access to packet",
.result = REJECT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: pkt_ptr spill into caller stack 3",
@@ -12367,6 +12452,7 @@ static struct bpf_test tests[] = {
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = ACCEPT,
.retval = 1,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: pkt_ptr spill into caller stack 4",
@@ -12401,6 +12487,7 @@ static struct bpf_test tests[] = {
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = ACCEPT,
.retval = 1,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: pkt_ptr spill into caller stack 5",
@@ -12434,6 +12521,7 @@ static struct bpf_test tests[] = {
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.errstr = "same insn cannot be used with different",
.result = REJECT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: pkt_ptr spill into caller stack 6",
@@ -12469,6 +12557,7 @@ static struct bpf_test tests[] = {
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.errstr = "R4 invalid mem access",
.result = REJECT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: pkt_ptr spill into caller stack 7",
@@ -12503,6 +12592,7 @@ static struct bpf_test tests[] = {
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.errstr = "R4 invalid mem access",
.result = REJECT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: pkt_ptr spill into caller stack 8",
@@ -12543,6 +12633,7 @@ static struct bpf_test tests[] = {
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = ACCEPT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: pkt_ptr spill into caller stack 9",
@@ -12584,6 +12675,7 @@ static struct bpf_test tests[] = {
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.errstr = "invalid access to packet",
.result = REJECT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"calls: caller stack init to zero or map_value_or_null",
@@ -12949,6 +13041,7 @@ static struct bpf_test tests[] = {
.result = REJECT,
.errstr = "BPF_XADD stores into R2 pkt is not allowed",
.prog_type = BPF_PROG_TYPE_XDP,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"xadd/w check whether src/dst got mangled, 1",
@@ -13435,6 +13528,7 @@ static struct bpf_test tests[] = {
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.errstr = "Unreleased reference",
.result = REJECT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"reference tracking: alloc, check, free in both subbranches",
@@ -13463,6 +13557,7 @@ static struct bpf_test tests[] = {
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = ACCEPT,
+ .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
},
{
"reference tracking in call: free reference in subprog",
@@ -13972,6 +14067,38 @@ static struct bpf_test tests[] = {
.result_unpriv = REJECT,
.result = ACCEPT,
},
+ {
+ "check wire_len is not readable by sockets",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
+ offsetof(struct __sk_buff, wire_len)),
+ BPF_EXIT_INSN(),
+ },
+ .errstr = "invalid bpf_context access",
+ .result = REJECT,
+ },
+ {
+ "check wire_len is readable by tc classifier",
+ .insns = {
+ BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
+ offsetof(struct __sk_buff, wire_len)),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_SCHED_CLS,
+ .result = ACCEPT,
+ },
+ {
+ "check wire_len is not writable by tc classifier",
+ .insns = {
+ BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_1,
+ offsetof(struct __sk_buff, wire_len)),
+ BPF_EXIT_INSN(),
+ },
+ .prog_type = BPF_PROG_TYPE_SCHED_CLS,
+ .errstr = "invalid bpf_context access",
+ .errstr_unpriv = "R1 leaks addr",
+ .result = REJECT,
+ },
};
static int probe_filter_length(const struct bpf_insn *fp)
@@ -14257,13 +14384,14 @@ out:
static void do_test_single(struct bpf_test *test, bool unpriv,
int *passes, int *errors)
{
- int fd_prog, expected_ret, reject_from_alignment;
+ int fd_prog, expected_ret, alignment_prevented_execution;
int prog_len, prog_type = test->prog_type;
struct bpf_insn *prog = test->insns;
int map_fds[MAX_NR_MAPS];
const char *expected_err;
uint32_t expected_val;
uint32_t retval;
+ __u32 pflags;
int i, err;
for (i = 0; i < MAX_NR_MAPS; i++)
@@ -14274,8 +14402,12 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
do_test_fixup(test, prog_type, prog, map_fds);
prog_len = probe_filter_length(prog);
- fd_prog = bpf_verify_program(prog_type, prog, prog_len,
- test->flags & F_LOAD_WITH_STRICT_ALIGNMENT,
+ pflags = 0;
+ if (test->flags & F_LOAD_WITH_STRICT_ALIGNMENT)
+ pflags |= BPF_F_STRICT_ALIGNMENT;
+ if (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS)
+ pflags |= BPF_F_ANY_ALIGNMENT;
+ fd_prog = bpf_verify_program(prog_type, prog, prog_len, pflags,
"GPL", 0, bpf_vlog, sizeof(bpf_vlog), 1);
expected_ret = unpriv && test->result_unpriv != UNDEF ?
@@ -14285,28 +14417,27 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
expected_val = unpriv && test->retval_unpriv ?
test->retval_unpriv : test->retval;
- reject_from_alignment = fd_prog < 0 &&
- (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS) &&
- strstr(bpf_vlog, "misaligned");
-#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
- if (reject_from_alignment) {
- printf("FAIL\nFailed due to alignment despite having efficient unaligned access: '%s'!\n",
- strerror(errno));
- goto fail_log;
- }
-#endif
+ alignment_prevented_execution = 0;
+
if (expected_ret == ACCEPT) {
- if (fd_prog < 0 && !reject_from_alignment) {
+ if (fd_prog < 0) {
printf("FAIL\nFailed to load prog '%s'!\n",
strerror(errno));
goto fail_log;
}
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+ if (fd_prog >= 0 &&
+ (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS)) {
+ alignment_prevented_execution = 1;
+ goto test_ok;
+ }
+#endif
} else {
if (fd_prog >= 0) {
printf("FAIL\nUnexpected success to load!\n");
goto fail_log;
}
- if (!strstr(bpf_vlog, expected_err) && !reject_from_alignment) {
+ if (!strstr(bpf_vlog, expected_err)) {
printf("FAIL\nUnexpected error message!\n\tEXP: %s\n\tRES: %s\n",
expected_err, bpf_vlog);
goto fail_log;
@@ -14334,9 +14465,12 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
goto fail_log;
}
}
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+test_ok:
+#endif
(*passes)++;
- printf("OK%s\n", reject_from_alignment ?
- " (NOTE: reject due to unknown alignment)" : "");
+ printf("OK%s\n", alignment_prevented_execution ?
+ " (NOTE: not executed due to unknown alignment)" : "");
close_fds:
close(fd_prog);
for (i = 0; i < MAX_NR_MAPS; i++)