diff options
Diffstat (limited to 'pkgs/patches-linux-5.15/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch')
| -rw-r--r-- | pkgs/patches-linux-5.15/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch | 1005 |
1 files changed, 1005 insertions, 0 deletions
diff --git a/pkgs/patches-linux-5.15/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch b/pkgs/patches-linux-5.15/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch new file mode 100644 index 0000000..38b46c0 --- /dev/null +++ b/pkgs/patches-linux-5.15/050-v5.16-03-mips-bpf-Add-new-eBPF-JIT-for-64-bit-MIPS.patch @@ -0,0 +1,1005 @@ +From: Johan Almbladh <johan.almbladh@anyfinetworks.com> +Date: Tue, 5 Oct 2021 18:54:05 +0200 +Subject: [PATCH] mips: bpf: Add new eBPF JIT for 64-bit MIPS + +This is an implementation on of an eBPF JIT for 64-bit MIPS III-V and +MIPS64r1-r6. It uses the same framework introduced by the 32-bit JIT. + +Signed-off-by: Johan Almbladh <johan.almbladh@anyfinetworks.com> +--- + create mode 100644 arch/mips/net/bpf_jit_comp64.c + +--- /dev/null ++++ b/arch/mips/net/bpf_jit_comp64.c +@@ -0,0 +1,991 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Just-In-Time compiler for eBPF bytecode on MIPS. ++ * Implementation of JIT functions for 64-bit CPUs. ++ * ++ * Copyright (c) 2021 Anyfi Networks AB. ++ * Author: Johan Almbladh <johan.almbladh@gmail.com> ++ * ++ * Based on code and ideas from ++ * Copyright (c) 2017 Cavium, Inc. ++ * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com> ++ * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com> ++ */ ++ ++#include <linux/errno.h> ++#include <linux/filter.h> ++#include <linux/bpf.h> ++#include <asm/cpu-features.h> ++#include <asm/isa-rev.h> ++#include <asm/uasm.h> ++ ++#include "bpf_jit_comp.h" ++ ++/* MIPS t0-t3 are not available in the n64 ABI */ ++#undef MIPS_R_T0 ++#undef MIPS_R_T1 ++#undef MIPS_R_T2 ++#undef MIPS_R_T3 ++ ++/* Stack is 16-byte aligned in n64 ABI */ ++#define MIPS_STACK_ALIGNMENT 16 ++ ++/* Extra 64-bit eBPF registers used by JIT */ ++#define JIT_REG_TC (MAX_BPF_JIT_REG + 0) ++#define JIT_REG_ZX (MAX_BPF_JIT_REG + 1) ++ ++/* Number of prologue bytes to skip when doing a tail call */ ++#define JIT_TCALL_SKIP 4 ++ ++/* Callee-saved CPU registers that the JIT must preserve */ ++#define JIT_CALLEE_REGS \ ++ (BIT(MIPS_R_S0) | \ ++ BIT(MIPS_R_S1) | \ ++ BIT(MIPS_R_S2) | \ ++ BIT(MIPS_R_S3) | \ ++ BIT(MIPS_R_S4) | \ ++ BIT(MIPS_R_S5) | \ ++ BIT(MIPS_R_S6) | \ ++ BIT(MIPS_R_S7) | \ ++ BIT(MIPS_R_GP) | \ ++ BIT(MIPS_R_FP) | \ ++ BIT(MIPS_R_RA)) ++ ++/* Caller-saved CPU registers available for JIT use */ ++#define JIT_CALLER_REGS \ ++ (BIT(MIPS_R_A5) | \ ++ BIT(MIPS_R_A6) | \ ++ BIT(MIPS_R_A7)) ++/* ++ * Mapping of 64-bit eBPF registers to 64-bit native MIPS registers. ++ * MIPS registers t4 - t7 may be used by the JIT as temporary registers. ++ * MIPS registers t8 - t9 are reserved for single-register common functions. ++ */ ++static const u8 bpf2mips64[] = { ++ /* Return value from in-kernel function, and exit value from eBPF */ ++ [BPF_REG_0] = MIPS_R_V0, ++ /* Arguments from eBPF program to in-kernel function */ ++ [BPF_REG_1] = MIPS_R_A0, ++ [BPF_REG_2] = MIPS_R_A1, ++ [BPF_REG_3] = MIPS_R_A2, ++ [BPF_REG_4] = MIPS_R_A3, ++ [BPF_REG_5] = MIPS_R_A4, ++ /* Callee-saved registers that in-kernel function will preserve */ ++ [BPF_REG_6] = MIPS_R_S0, ++ [BPF_REG_7] = MIPS_R_S1, ++ [BPF_REG_8] = MIPS_R_S2, ++ [BPF_REG_9] = MIPS_R_S3, ++ /* Read-only frame pointer to access the eBPF stack */ ++ [BPF_REG_FP] = MIPS_R_FP, ++ /* Temporary register for blinding constants */ ++ [BPF_REG_AX] = MIPS_R_AT, ++ /* Tail call count register, caller-saved */ ++ [JIT_REG_TC] = MIPS_R_A5, ++ /* Constant for register zero-extension */ ++ [JIT_REG_ZX] = MIPS_R_V1, ++}; ++ ++/* ++ * MIPS 32-bit operations on 64-bit registers generate a sign-extended ++ * result. However, the eBPF ISA mandates zero-extension, so we rely on the ++ * verifier to add that for us (emit_zext_ver). In addition, ALU arithmetic ++ * operations, right shift and byte swap require properly sign-extended ++ * operands or the result is unpredictable. We emit explicit sign-extensions ++ * in those cases. ++ */ ++ ++/* Sign extension */ ++static void emit_sext(struct jit_context *ctx, u8 dst, u8 src) ++{ ++ emit(ctx, sll, dst, src, 0); ++ clobber_reg(ctx, dst); ++} ++ ++/* Zero extension */ ++static void emit_zext(struct jit_context *ctx, u8 dst) ++{ ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) { ++ emit(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); ++ } else { ++ emit(ctx, and, dst, dst, bpf2mips64[JIT_REG_ZX]); ++ access_reg(ctx, JIT_REG_ZX); /* We need the ZX register */ ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Zero extension, if verifier does not do it for us */ ++static void emit_zext_ver(struct jit_context *ctx, u8 dst) ++{ ++ if (!ctx->program->aux->verifier_zext) ++ emit_zext(ctx, dst); ++} ++ ++/* dst = imm (64-bit) */ ++static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64) ++{ ++ if (imm64 >= 0xffffffffffff8000ULL || imm64 < 0x8000ULL) { ++ emit(ctx, daddiu, dst, MIPS_R_ZERO, (s16)imm64); ++ } else if (imm64 >= 0xffffffff80000000ULL || ++ (imm64 < 0x80000000 && imm64 > 0xffff)) { ++ emit(ctx, lui, dst, (s16)(imm64 >> 16)); ++ emit(ctx, ori, dst, dst, (u16)imm64 & 0xffff); ++ } else { ++ u8 acc = MIPS_R_ZERO; ++ int k; ++ ++ for (k = 0; k < 4; k++) { ++ u16 half = imm64 >> (48 - 16 * k); ++ ++ if (acc == dst) ++ emit(ctx, dsll, dst, dst, 16); ++ ++ if (half) { ++ emit(ctx, ori, dst, acc, half); ++ acc = dst; ++ } ++ } ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU immediate operation (64-bit) */ ++static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst | imm */ ++ case BPF_OR: ++ emit(ctx, ori, dst, dst, (u16)imm); ++ break; ++ /* dst = dst ^ imm */ ++ case BPF_XOR: ++ emit(ctx, xori, dst, dst, (u16)imm); ++ break; ++ /* dst = -dst */ ++ case BPF_NEG: ++ emit(ctx, dsubu, dst, MIPS_R_ZERO, dst); ++ break; ++ /* dst = dst << imm */ ++ case BPF_LSH: ++ emit(ctx, dsll_safe, dst, dst, imm); ++ break; ++ /* dst = dst >> imm */ ++ case BPF_RSH: ++ emit(ctx, dsrl_safe, dst, dst, imm); ++ break; ++ /* dst = dst >> imm (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, dsra_safe, dst, dst, imm); ++ break; ++ /* dst = dst + imm */ ++ case BPF_ADD: ++ emit(ctx, daddiu, dst, dst, imm); ++ break; ++ /* dst = dst - imm */ ++ case BPF_SUB: ++ emit(ctx, daddiu, dst, dst, -imm); ++ break; ++ default: ++ /* Width-generic operations */ ++ emit_alu_i(ctx, dst, imm, op); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* ALU register operation (64-bit) */ ++static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) ++{ ++ switch (BPF_OP(op)) { ++ /* dst = dst << src */ ++ case BPF_LSH: ++ emit(ctx, dsllv, dst, dst, src); ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ emit(ctx, dsrlv, dst, dst, src); ++ break; ++ /* dst = dst >> src (arithmetic) */ ++ case BPF_ARSH: ++ emit(ctx, dsrav, dst, dst, src); ++ break; ++ /* dst = dst + src */ ++ case BPF_ADD: ++ emit(ctx, daddu, dst, dst, src); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit(ctx, dsubu, dst, dst, src); ++ break; ++ /* dst = dst * src */ ++ case BPF_MUL: ++ if (cpu_has_mips64r6) { ++ emit(ctx, dmulu, dst, dst, src); ++ } else { ++ emit(ctx, dmultu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst / src */ ++ case BPF_DIV: ++ if (cpu_has_mips64r6) { ++ emit(ctx, ddivu_r6, dst, dst, src); ++ } else { ++ emit(ctx, ddivu, dst, src); ++ emit(ctx, mflo, dst); ++ } ++ break; ++ /* dst = dst % src */ ++ case BPF_MOD: ++ if (cpu_has_mips64r6) { ++ emit(ctx, dmodu, dst, dst, src); ++ } else { ++ emit(ctx, ddivu, dst, src); ++ emit(ctx, mfhi, dst); ++ } ++ break; ++ default: ++ /* Width-generic operations */ ++ emit_alu_r(ctx, dst, src, op); ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Swap sub words in a register double word */ ++static void emit_swap_r64(struct jit_context *ctx, u8 dst, u8 mask, u32 bits) ++{ ++ u8 tmp = MIPS_R_T9; ++ ++ emit(ctx, and, tmp, dst, mask); /* tmp = dst & mask */ ++ emit(ctx, dsll, tmp, tmp, bits); /* tmp = tmp << bits */ ++ emit(ctx, dsrl, dst, dst, bits); /* dst = dst >> bits */ ++ emit(ctx, and, dst, dst, mask); /* dst = dst & mask */ ++ emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ ++} ++ ++/* Swap bytes and truncate a register double word, word or half word */ ++static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ switch (width) { ++ /* Swap bytes in a double word */ ++ case 64: ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) { ++ emit(ctx, dsbh, dst, dst); ++ emit(ctx, dshd, dst, dst); ++ } else { ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ emit(ctx, dsll32, t2, dst, 0); /* t2 = dst << 32 */ ++ emit(ctx, dsrl32, dst, dst, 0); /* dst = dst >> 32 */ ++ emit(ctx, or, dst, dst, t2); /* dst = dst | t2 */ ++ ++ emit(ctx, ori, t2, MIPS_R_ZERO, 0xffff); ++ emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ ++ emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ ++ emit_swap_r64(ctx, dst, t1, 16);/* dst = swap16(dst) */ ++ ++ emit(ctx, lui, t2, 0xff); /* t2 = 0x00ff0000 */ ++ emit(ctx, ori, t2, t2, 0xff); /* t2 = t2 | 0x00ff */ ++ emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ ++ emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ ++ emit_swap_r64(ctx, dst, t1, 8); /* dst = swap8(dst) */ ++ } ++ break; ++ /* Swap bytes in a half word */ ++ /* Swap bytes in a word */ ++ case 32: ++ case 16: ++ emit_sext(ctx, dst, dst); ++ emit_bswap_r(ctx, dst, width); ++ if (cpu_has_mips64r2 || cpu_has_mips64r6) ++ emit_zext(ctx, dst); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Truncate a register double word, word or half word */ ++static void emit_trunc_r64(struct jit_context *ctx, u8 dst, u32 width) ++{ ++ switch (width) { ++ case 64: ++ break; ++ /* Zero-extend a word */ ++ case 32: ++ emit_zext(ctx, dst); ++ break; ++ /* Zero-extend a half word */ ++ case 16: ++ emit(ctx, andi, dst, dst, 0xffff); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Load operation: dst = *(size*)(src + off) */ ++static void emit_ldx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Load a byte */ ++ case BPF_B: ++ emit(ctx, lbu, dst, off, src); ++ break; ++ /* Load a half word */ ++ case BPF_H: ++ emit(ctx, lhu, dst, off, src); ++ break; ++ /* Load a word */ ++ case BPF_W: ++ emit(ctx, lwu, dst, off, src); ++ break; ++ /* Load a double word */ ++ case BPF_DW: ++ emit(ctx, ld, dst, off, src); ++ break; ++ } ++ clobber_reg(ctx, dst); ++} ++ ++/* Store operation: *(size *)(dst + off) = src */ ++static void emit_stx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) ++{ ++ switch (size) { ++ /* Store a byte */ ++ case BPF_B: ++ emit(ctx, sb, src, off, dst); ++ break; ++ /* Store a half word */ ++ case BPF_H: ++ emit(ctx, sh, src, off, dst); ++ break; ++ /* Store a word */ ++ case BPF_W: ++ emit(ctx, sw, src, off, dst); ++ break; ++ /* Store a double word */ ++ case BPF_DW: ++ emit(ctx, sd, src, off, dst); ++ break; ++ } ++} ++ ++/* Atomic read-modify-write */ ++static void emit_atomic_r64(struct jit_context *ctx, ++ u8 dst, u8 src, s16 off, u8 code) ++{ ++ u8 t1 = MIPS_R_T6; ++ u8 t2 = MIPS_R_T7; ++ ++ emit(ctx, lld, t1, off, dst); ++ switch (code) { ++ case BPF_ADD: ++ emit(ctx, daddu, t2, t1, src); ++ break; ++ case BPF_AND: ++ emit(ctx, and, t2, t1, src); ++ break; ++ case BPF_OR: ++ emit(ctx, or, t2, t1, src); ++ break; ++ case BPF_XOR: ++ emit(ctx, xor, t2, t1, src); ++ break; ++ } ++ emit(ctx, scd, t2, off, dst); ++ emit(ctx, beqz, t2, -16); ++ emit(ctx, nop); /* Delay slot */ ++} ++ ++/* Function call */ ++static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) ++{ ++ u8 zx = bpf2mips64[JIT_REG_ZX]; ++ u8 tmp = MIPS_R_T6; ++ bool fixed; ++ u64 addr; ++ ++ /* Decode the call address */ ++ if (bpf_jit_get_func_addr(ctx->program, insn, false, ++ &addr, &fixed) < 0) ++ return -1; ++ if (!fixed) ++ return -1; ++ ++ /* Push caller-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); ++ ++ /* Emit function call */ ++ emit_mov_i64(ctx, tmp, addr); ++ emit(ctx, jalr, MIPS_R_RA, tmp); ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* Restore caller-saved registers */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); ++ ++ /* Re-initialize the JIT zero-extension register if accessed */ ++ if (ctx->accessed & BIT(JIT_REG_ZX)) { ++ emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); ++ emit(ctx, dsrl32, zx, zx, 0); ++ } ++ ++ clobber_reg(ctx, MIPS_R_RA); ++ clobber_reg(ctx, MIPS_R_V0); ++ clobber_reg(ctx, MIPS_R_V1); ++ return 0; ++} ++ ++/* Function tail call */ ++static int emit_tail_call(struct jit_context *ctx) ++{ ++ u8 ary = bpf2mips64[BPF_REG_2]; ++ u8 ind = bpf2mips64[BPF_REG_3]; ++ u8 tcc = bpf2mips64[JIT_REG_TC]; ++ u8 tmp = MIPS_R_T6; ++ int off; ++ ++ /* ++ * Tail call: ++ * eBPF R1 - function argument (context ptr), passed in a0-a1 ++ * eBPF R2 - ptr to object with array of function entry points ++ * eBPF R3 - array index of function to be called ++ */ ++ ++ /* if (ind >= ary->map.max_entries) goto out */ ++ off = offsetof(struct bpf_array, map.max_entries); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, lwu, tmp, off, ary); /* tmp = ary->map.max_entrs*/ ++ emit(ctx, sltu, tmp, ind, tmp); /* tmp = ind < t1 */ ++ emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ ++ ++ /* if (--TCC < 0) goto out */ ++ emit(ctx, daddiu, tcc, tcc, -1); /* tcc-- (delay slot) */ ++ emit(ctx, bltz, tcc, get_offset(ctx, 1)); /* PC += off(1) if tcc < 0 */ ++ /* (next insn delay slot) */ ++ /* prog = ary->ptrs[ind] */ ++ off = offsetof(struct bpf_array, ptrs); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, dsll, tmp, ind, 3); /* tmp = ind << 3 */ ++ emit(ctx, daddu, tmp, tmp, ary); /* tmp += ary */ ++ emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ ++ ++ /* if (prog == 0) goto out */ ++ emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ ++ emit(ctx, nop); /* Delay slot */ ++ ++ /* func = prog->bpf_func + 8 (prologue skip offset) */ ++ off = offsetof(struct bpf_prog, bpf_func); ++ if (off > 0x7fff) ++ return -1; ++ emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ ++ emit(ctx, daddiu, tmp, tmp, JIT_TCALL_SKIP); /* tmp += skip (4) */ ++ ++ /* goto func */ ++ build_epilogue(ctx, tmp); ++ access_reg(ctx, JIT_REG_TC); ++ return 0; ++} ++ ++/* ++ * Stack frame layout for a JITed program (stack grows down). ++ * ++ * Higher address : Previous stack frame : ++ * +===========================+ <--- MIPS sp before call ++ * | Callee-saved registers, | ++ * | including RA and FP | ++ * +---------------------------+ <--- eBPF FP (MIPS fp) ++ * | Local eBPF variables | ++ * | allocated by program | ++ * +---------------------------+ ++ * | Reserved for caller-saved | ++ * | registers | ++ * Lower address +===========================+ <--- MIPS sp ++ */ ++ ++/* Build program prologue to set up the stack and registers */ ++void build_prologue(struct jit_context *ctx) ++{ ++ u8 fp = bpf2mips64[BPF_REG_FP]; ++ u8 tc = bpf2mips64[JIT_REG_TC]; ++ u8 zx = bpf2mips64[JIT_REG_ZX]; ++ int stack, saved, locals, reserved; ++ ++ /* ++ * The first instruction initializes the tail call count register. ++ * On a tail call, the calling function jumps into the prologue ++ * after this instruction. ++ */ ++ emit(ctx, addiu, tc, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT + 1, 0xffff)); ++ ++ /* === Entry-point for tail calls === */ ++ ++ /* ++ * If the eBPF frame pointer and tail call count registers were ++ * accessed they must be preserved. Mark them as clobbered here ++ * to save and restore them on the stack as needed. ++ */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ clobber_reg(ctx, fp); ++ if (ctx->accessed & BIT(JIT_REG_TC)) ++ clobber_reg(ctx, tc); ++ if (ctx->accessed & BIT(JIT_REG_ZX)) ++ clobber_reg(ctx, zx); ++ ++ /* Compute the stack space needed for callee-saved registers */ ++ saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u64); ++ saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); ++ ++ /* Stack space used by eBPF program local data */ ++ locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); ++ ++ /* ++ * If we are emitting function calls, reserve extra stack space for ++ * caller-saved registers needed by the JIT. The required space is ++ * computed automatically during resource usage discovery (pass 1). ++ */ ++ reserved = ctx->stack_used; ++ ++ /* Allocate the stack frame */ ++ stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); ++ if (stack) ++ emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack); ++ ++ /* Store callee-saved registers on stack */ ++ push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); ++ ++ /* Initialize the eBPF frame pointer if accessed */ ++ if (ctx->accessed & BIT(BPF_REG_FP)) ++ emit(ctx, daddiu, fp, MIPS_R_SP, stack - saved); ++ ++ /* Initialize the ePF JIT zero-extension register if accessed */ ++ if (ctx->accessed & BIT(JIT_REG_ZX)) { ++ emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); ++ emit(ctx, dsrl32, zx, zx, 0); ++ } ++ ++ ctx->saved_size = saved; ++ ctx->stack_size = stack; ++} ++ ++/* Build the program epilogue to restore the stack and registers */ ++void build_epilogue(struct jit_context *ctx, int dest_reg) ++{ ++ /* Restore callee-saved registers from stack */ ++ pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, ++ ctx->stack_size - ctx->saved_size); ++ ++ /* Release the stack frame */ ++ if (ctx->stack_size) ++ emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); ++ ++ /* Jump to return address and sign-extend the 32-bit return value */ ++ emit(ctx, jr, dest_reg); ++ emit(ctx, sll, MIPS_R_V0, MIPS_R_V0, 0); /* Delay slot */ ++} ++ ++/* Build one eBPF instruction */ ++int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) ++{ ++ u8 dst = bpf2mips64[insn->dst_reg]; ++ u8 src = bpf2mips64[insn->src_reg]; ++ u8 code = insn->code; ++ s16 off = insn->off; ++ s32 imm = insn->imm; ++ s32 val, rel; ++ u8 alu, jmp; ++ ++ switch (code) { ++ /* ALU operations */ ++ /* dst = imm */ ++ case BPF_ALU | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, dst, imm); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = src */ ++ case BPF_ALU | BPF_MOV | BPF_X: ++ if (imm == 1) { ++ /* Special mov32 for zext */ ++ emit_zext(ctx, dst); ++ } else { ++ emit_mov_r(ctx, dst, src); ++ emit_zext_ver(ctx, dst); ++ } ++ break; ++ /* dst = -dst */ ++ case BPF_ALU | BPF_NEG: ++ emit_sext(ctx, dst, dst); ++ emit_alu_i(ctx, dst, 0, BPF_NEG); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & imm */ ++ /* dst = dst | imm */ ++ /* dst = dst ^ imm */ ++ /* dst = dst << imm */ ++ case BPF_ALU | BPF_OR | BPF_K: ++ case BPF_ALU | BPF_AND | BPF_K: ++ case BPF_ALU | BPF_XOR | BPF_K: ++ case BPF_ALU | BPF_LSH | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i(ctx, dst, val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst >> imm */ ++ /* dst = dst >> imm (arithmetic) */ ++ /* dst = dst + imm */ ++ /* dst = dst - imm */ ++ /* dst = dst * imm */ ++ /* dst = dst / imm */ ++ /* dst = dst % imm */ ++ case BPF_ALU | BPF_RSH | BPF_K: ++ case BPF_ALU | BPF_ARSH | BPF_K: ++ case BPF_ALU | BPF_ADD | BPF_K: ++ case BPF_ALU | BPF_SUB | BPF_K: ++ case BPF_ALU | BPF_MUL | BPF_K: ++ case BPF_ALU | BPF_DIV | BPF_K: ++ case BPF_ALU | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_sext(ctx, dst, dst); ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_sext(ctx, dst, dst); ++ emit_alu_i(ctx, dst, val, alu); ++ } ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst & src */ ++ /* dst = dst | src */ ++ /* dst = dst ^ src */ ++ /* dst = dst << src */ ++ case BPF_ALU | BPF_AND | BPF_X: ++ case BPF_ALU | BPF_OR | BPF_X: ++ case BPF_ALU | BPF_XOR | BPF_X: ++ case BPF_ALU | BPF_LSH | BPF_X: ++ emit_alu_r(ctx, dst, src, BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = dst >> src */ ++ /* dst = dst >> src (arithmetic) */ ++ /* dst = dst + src */ ++ /* dst = dst - src */ ++ /* dst = dst * src */ ++ /* dst = dst / src */ ++ /* dst = dst % src */ ++ case BPF_ALU | BPF_RSH | BPF_X: ++ case BPF_ALU | BPF_ARSH | BPF_X: ++ case BPF_ALU | BPF_ADD | BPF_X: ++ case BPF_ALU | BPF_SUB | BPF_X: ++ case BPF_ALU | BPF_MUL | BPF_X: ++ case BPF_ALU | BPF_DIV | BPF_X: ++ case BPF_ALU | BPF_MOD | BPF_X: ++ emit_sext(ctx, dst, dst); ++ emit_sext(ctx, MIPS_R_T4, src); ++ emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ emit_zext_ver(ctx, dst); ++ break; ++ /* dst = imm (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_K: ++ emit_mov_i(ctx, dst, imm); ++ break; ++ /* dst = src (64-bit) */ ++ case BPF_ALU64 | BPF_MOV | BPF_X: ++ emit_mov_r(ctx, dst, src); ++ break; ++ /* dst = -dst (64-bit) */ ++ case BPF_ALU64 | BPF_NEG: ++ emit_alu_i64(ctx, dst, 0, BPF_NEG); ++ break; ++ /* dst = dst & imm (64-bit) */ ++ /* dst = dst | imm (64-bit) */ ++ /* dst = dst ^ imm (64-bit) */ ++ /* dst = dst << imm (64-bit) */ ++ /* dst = dst >> imm (64-bit) */ ++ /* dst = dst >> imm ((64-bit, arithmetic) */ ++ /* dst = dst + imm (64-bit) */ ++ /* dst = dst - imm (64-bit) */ ++ /* dst = dst * imm (64-bit) */ ++ /* dst = dst / imm (64-bit) */ ++ /* dst = dst % imm (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_K: ++ case BPF_ALU64 | BPF_OR | BPF_K: ++ case BPF_ALU64 | BPF_XOR | BPF_K: ++ case BPF_ALU64 | BPF_LSH | BPF_K: ++ case BPF_ALU64 | BPF_RSH | BPF_K: ++ case BPF_ALU64 | BPF_ARSH | BPF_K: ++ case BPF_ALU64 | BPF_ADD | BPF_K: ++ case BPF_ALU64 | BPF_SUB | BPF_K: ++ case BPF_ALU64 | BPF_MUL | BPF_K: ++ case BPF_ALU64 | BPF_DIV | BPF_K: ++ case BPF_ALU64 | BPF_MOD | BPF_K: ++ if (!valid_alu_i(BPF_OP(code), imm)) { ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code)); ++ } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { ++ emit_alu_i64(ctx, dst, val, alu); ++ } ++ break; ++ /* dst = dst & src (64-bit) */ ++ /* dst = dst | src (64-bit) */ ++ /* dst = dst ^ src (64-bit) */ ++ /* dst = dst << src (64-bit) */ ++ /* dst = dst >> src (64-bit) */ ++ /* dst = dst >> src (64-bit, arithmetic) */ ++ /* dst = dst + src (64-bit) */ ++ /* dst = dst - src (64-bit) */ ++ /* dst = dst * src (64-bit) */ ++ /* dst = dst / src (64-bit) */ ++ /* dst = dst % src (64-bit) */ ++ case BPF_ALU64 | BPF_AND | BPF_X: ++ case BPF_ALU64 | BPF_OR | BPF_X: ++ case BPF_ALU64 | BPF_XOR | BPF_X: ++ case BPF_ALU64 | BPF_LSH | BPF_X: ++ case BPF_ALU64 | BPF_RSH | BPF_X: ++ case BPF_ALU64 | BPF_ARSH | BPF_X: ++ case BPF_ALU64 | BPF_ADD | BPF_X: ++ case BPF_ALU64 | BPF_SUB | BPF_X: ++ case BPF_ALU64 | BPF_MUL | BPF_X: ++ case BPF_ALU64 | BPF_DIV | BPF_X: ++ case BPF_ALU64 | BPF_MOD | BPF_X: ++ emit_alu_r64(ctx, dst, src, BPF_OP(code)); ++ break; ++ /* dst = htole(dst) */ ++ /* dst = htobe(dst) */ ++ case BPF_ALU | BPF_END | BPF_FROM_LE: ++ case BPF_ALU | BPF_END | BPF_FROM_BE: ++ if (BPF_SRC(code) == ++#ifdef __BIG_ENDIAN ++ BPF_FROM_LE ++#else ++ BPF_FROM_BE ++#endif ++ ) ++ emit_bswap_r64(ctx, dst, imm); ++ else ++ emit_trunc_r64(ctx, dst, imm); ++ break; ++ /* dst = imm64 */ ++ case BPF_LD | BPF_IMM | BPF_DW: ++ emit_mov_i64(ctx, dst, (u32)imm | ((u64)insn[1].imm << 32)); ++ return 1; ++ /* LDX: dst = *(size *)(src + off) */ ++ case BPF_LDX | BPF_MEM | BPF_W: ++ case BPF_LDX | BPF_MEM | BPF_H: ++ case BPF_LDX | BPF_MEM | BPF_B: ++ case BPF_LDX | BPF_MEM | BPF_DW: ++ emit_ldx(ctx, dst, src, off, BPF_SIZE(code)); ++ break; ++ /* ST: *(size *)(dst + off) = imm */ ++ case BPF_ST | BPF_MEM | BPF_W: ++ case BPF_ST | BPF_MEM | BPF_H: ++ case BPF_ST | BPF_MEM | BPF_B: ++ case BPF_ST | BPF_MEM | BPF_DW: ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_stx(ctx, dst, MIPS_R_T4, off, BPF_SIZE(code)); ++ break; ++ /* STX: *(size *)(dst + off) = src */ ++ case BPF_STX | BPF_MEM | BPF_W: ++ case BPF_STX | BPF_MEM | BPF_H: ++ case BPF_STX | BPF_MEM | BPF_B: ++ case BPF_STX | BPF_MEM | BPF_DW: ++ emit_stx(ctx, dst, src, off, BPF_SIZE(code)); ++ break; ++ /* Speculation barrier */ ++ case BPF_ST | BPF_NOSPEC: ++ break; ++ /* Atomics */ ++ case BPF_STX | BPF_XADD | BPF_W: ++ case BPF_STX | BPF_XADD | BPF_DW: ++ switch (imm) { ++ case BPF_ADD: ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ if (BPF_SIZE(code) == BPF_DW) { ++ emit_atomic_r64(ctx, dst, src, off, imm); ++ } else { /* 32-bit, no fetch */ ++ emit_sext(ctx, MIPS_R_T4, src); ++ emit_atomic_r(ctx, dst, MIPS_R_T4, off, imm); ++ } ++ break; ++ default: ++ goto notyet; ++ } ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_X: ++ case BPF_JMP32 | BPF_JNE | BPF_X: ++ case BPF_JMP32 | BPF_JSET | BPF_X: ++ case BPF_JMP32 | BPF_JGT | BPF_X: ++ case BPF_JMP32 | BPF_JGE | BPF_X: ++ case BPF_JMP32 | BPF_JLT | BPF_X: ++ case BPF_JMP32 | BPF_JLE | BPF_X: ++ case BPF_JMP32 | BPF_JSGT | BPF_X: ++ case BPF_JMP32 | BPF_JSGE | BPF_X: ++ case BPF_JMP32 | BPF_JSLT | BPF_X: ++ case BPF_JMP32 | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ ++ emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */ ++ emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP32 | BPF_JEQ | BPF_K: ++ case BPF_JMP32 | BPF_JNE | BPF_K: ++ case BPF_JMP32 | BPF_JSET | BPF_K: ++ case BPF_JMP32 | BPF_JGT | BPF_K: ++ case BPF_JMP32 | BPF_JGE | BPF_K: ++ case BPF_JMP32 | BPF_JLT | BPF_K: ++ case BPF_JMP32 | BPF_JLE | BPF_K: ++ case BPF_JMP32 | BPF_JSGT | BPF_K: ++ case BPF_JMP32 | BPF_JSGE | BPF_K: ++ case BPF_JMP32 | BPF_JSLT | BPF_K: ++ case BPF_JMP32 | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); ++ emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, MIPS_R_T4, imm, rel, jmp); ++ } else { ++ /* Move large immediate to register, sign-extended */ ++ emit_mov_i(ctx, MIPS_R_T5, imm); ++ emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst & src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_X: ++ case BPF_JMP | BPF_JNE | BPF_X: ++ case BPF_JMP | BPF_JSET | BPF_X: ++ case BPF_JMP | BPF_JGT | BPF_X: ++ case BPF_JMP | BPF_JGE | BPF_X: ++ case BPF_JMP | BPF_JLT | BPF_X: ++ case BPF_JMP | BPF_JLE | BPF_X: ++ case BPF_JMP | BPF_JSGT | BPF_X: ++ case BPF_JMP | BPF_JSGE | BPF_X: ++ case BPF_JMP | BPF_JSLT | BPF_X: ++ case BPF_JMP | BPF_JSLE | BPF_X: ++ if (off == 0) ++ break; ++ setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); ++ emit_jmp_r(ctx, dst, src, rel, jmp); ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst & imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ case BPF_JMP | BPF_JEQ | BPF_K: ++ case BPF_JMP | BPF_JNE | BPF_K: ++ case BPF_JMP | BPF_JSET | BPF_K: ++ case BPF_JMP | BPF_JGT | BPF_K: ++ case BPF_JMP | BPF_JGE | BPF_K: ++ case BPF_JMP | BPF_JLT | BPF_K: ++ case BPF_JMP | BPF_JLE | BPF_K: ++ case BPF_JMP | BPF_JSGT | BPF_K: ++ case BPF_JMP | BPF_JSGE | BPF_K: ++ case BPF_JMP | BPF_JSLT | BPF_K: ++ case BPF_JMP | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); ++ if (valid_jmp_i(jmp, imm)) { ++ emit_jmp_i(ctx, dst, imm, rel, jmp); ++ } else { ++ /* Move large immediate to register */ ++ emit_mov_i(ctx, MIPS_R_T4, imm); ++ emit_jmp_r(ctx, dst, MIPS_R_T4, rel, jmp); ++ } ++ if (finish_jmp(ctx, jmp, off) < 0) ++ goto toofar; ++ break; ++ /* PC += off */ ++ case BPF_JMP | BPF_JA: ++ if (off == 0) ++ break; ++ if (emit_ja(ctx, off) < 0) ++ goto toofar; ++ break; ++ /* Tail call */ ++ case BPF_JMP | BPF_TAIL_CALL: ++ if (emit_tail_call(ctx) < 0) ++ goto invalid; ++ break; ++ /* Function call */ ++ case BPF_JMP | BPF_CALL: ++ if (emit_call(ctx, insn) < 0) ++ goto invalid; ++ break; ++ /* Function return */ ++ case BPF_JMP | BPF_EXIT: ++ /* ++ * Optimization: when last instruction is EXIT ++ * simply continue to epilogue. ++ */ ++ if (ctx->bpf_index == ctx->program->len - 1) ++ break; ++ if (emit_exit(ctx) < 0) ++ goto toofar; ++ break; ++ ++ default: ++invalid: ++ pr_err_once("unknown opcode %02x\n", code); ++ return -EINVAL; ++notyet: ++ pr_info_once("*** NOT YET: opcode %02x ***\n", code); ++ return -EFAULT; ++toofar: ++ pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", ++ ctx->bpf_index, code); ++ return -E2BIG; ++ } ++ return 0; ++} |