diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 3198acb2aad8c5c858b6b6cda28f2e7d8e8121d2..270110eee28cc8b922dbdfa2d3770eb2bd5c7f03 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -177,6 +177,20 @@ msr spsr_el2, x0 .endm +.macro __init_el2_mpam + /* Memory Partioning And Monitoring: disable EL2 traps */ + mrs x1, id_aa64pfr0_el1 + ubfx x0, x1, #ID_AA64PFR0_MPAM_SHIFT, #4 + cbz x0, .Lskip_mpam_\@ // skip if no MPAM + msr_s SYS_MPAM0_EL1, xzr // use the default partition.. + msr_s SYS_MPAM2_EL2, xzr // ..and disable lower traps + msr_s SYS_MPAM1_EL1, xzr + mrs_s x0, SYS_MPAMIDR_EL1 + tbz x0, #17, .Lskip_mpam_\@ // skip if no MPAMHCR reg + msr_s SYS_MPAMHCR_EL2, xzr // clear TRAP_MPAMIDR_EL1 -> EL2 +.Lskip_mpam_\@: +.endm + /** * Initialize EL2 registers to sane values. This should be called early on all * cores that were booted in EL2. Note that everything gets initialised as @@ -193,6 +207,7 @@ __init_el2_stage2 __init_el2_gicv3 __init_el2_hstr + __init_el2_mpam __init_el2_nvhe_idregs __init_el2_nvhe_cptr __init_el2_nvhe_sve diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index b268082d67edddd4afac9115fd6768c020b59ea5..8b1555512589e370b90ab1a1dd12fc5b56db7644 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -397,7 +397,10 @@ #define SYS_LOREA_EL1 sys_reg(3, 0, 10, 4, 1) #define SYS_LORN_EL1 sys_reg(3, 0, 10, 4, 2) #define SYS_LORC_EL1 sys_reg(3, 0, 10, 4, 3) +#define SYS_MPAMIDR_EL1 sys_reg(3, 0, 10, 4, 4) #define SYS_LORID_EL1 sys_reg(3, 0, 10, 4, 7) +#define SYS_MPAM1_EL1 sys_reg(3, 0, 10, 5, 0) +#define SYS_MPAM0_EL1 sys_reg(3, 0, 10, 5, 1) #define SYS_VBAR_EL1 sys_reg(3, 0, 12, 0, 0) #define SYS_DISR_EL1 sys_reg(3, 0, 12, 1, 1) @@ -545,6 +548,10 @@ #define SYS_TFSR_EL2 sys_reg(3, 4, 5, 6, 0) #define SYS_FAR_EL2 sys_reg(3, 4, 6, 0, 0) +#define SYS_MPAMHCR_EL2 sys_reg(3, 4, 10, 4, 0) +#define SYS_MPAMVPMV_EL2 sys_reg(3, 4, 10, 4, 1) +#define SYS_MPAM2_EL2 sys_reg(3, 4, 10, 5, 0) + #define SYS_VDISR_EL2 sys_reg(3, 4, 12, 1, 1) #define __SYS__AP0Rx_EL2(x) sys_reg(3, 4, 12, 8, x) #define SYS_ICH_AP0R0_EL2 __SYS__AP0Rx_EL2(0) diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig index 32fb9e5b6195e41d5c5d591b12bdab2280477812..5955e80a67fe4e22e96d6688e20aa42dd834ff8f 100644 --- a/drivers/android/Kconfig +++ b/drivers/android/Kconfig @@ -63,6 +63,13 @@ config ANDROID_VENDOR_HOOKS Allow vendor modules to attach to tracepoint "hooks" defined via DECLARE_HOOK or DECLARE_RESTRICTED_HOOK. +config ANDROID_MPAM_ARCH + tristate "MPAM module based architecture driver" + +config ANDROID_MPAM_POLICY + tristate "MPAM policy module" + depends on ANDROID_MPAM_ARCH && ANDROID_VENDOR_HOOKS + endif # if ANDROID endmenu diff --git a/drivers/android/Makefile b/drivers/android/Makefile index d488047415a0aa7cc88dea531b9edc27b357ad4d..dd3beb69e4f15e3923e629df9431de28528ac8a8 100644 --- a/drivers/android/Makefile +++ b/drivers/android/Makefile @@ -5,3 +5,4 @@ obj-$(CONFIG_ANDROID_BINDERFS) += binderfs.o obj-$(CONFIG_ANDROID_BINDER_IPC) += binder.o binder_alloc.o obj-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o obj-$(CONFIG_ANDROID_VENDOR_HOOKS) += vendor_hooks.o +obj-$(CONFIG_ANDROID_MPAM_ARCH) += mpam_arch.o diff --git a/drivers/android/mpam_arch.c b/drivers/android/mpam_arch.c new file mode 100644 index 0000000000000000000000000000000000000000..6178a1fc9a52740fca4f8102737a96eaa1a25428 --- /dev/null +++ b/drivers/android/mpam_arch.c @@ -0,0 +1,694 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Module-based hack to drive MPAM functionality + * + * NOTICE: This circumvents existing infrastructure to discover and enable CPU + * features and attempts to contain everything within a loadable module. This is + * *not* the right way to do things, but it is one way to start testing MPAM on + * real hardware. + * + * Copyright (C) 2022 Arm Ltd. + */ + +#define DEBUG + +#define pr_fmt(fmt) "MPAM_arch: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "mpam_arch.h" +#include "mpam_arch_internal.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Valentin Schneider "); + +#define FIELD_SET(reg, field, val) (reg = (reg & ~field) | FIELD_PREP(field, val)) + +struct msc_part_kobj { + struct mpam_msc *msc; + unsigned int partid; + struct kobject kobj; +}; + +struct mpam_msc { + struct platform_device *pdev; + + void __iomem *base; + spinlock_t lock; + + unsigned int partid_count; + unsigned int cpbm_nbits; + unsigned int cmax_nbits; + unsigned int cmax_shift; + + int has_ris; + union { + struct { + bool has_cpor; + bool has_ccap; + }; + u8 part_feats; + }; + + struct kobject ko_root; + struct kobject ko_part_dir; + struct msc_part_kobj *ko_parts; +}; + +static __read_mostly unsigned int mpam_partid_count = UINT_MAX; + +/* The MPAM0_EL1.PARTID_D in use by a given CPU */ +static DEFINE_PER_CPU(unsigned int, mpam_local_partid) = MPAM_PARTID_DEFAULT; + +unsigned int mpam_get_partid_count(void) +{ + /* + * XXX: this should check the driver has probed all matching devices + * first + */ + return mpam_partid_count; +} +EXPORT_SYMBOL_GPL(mpam_get_partid_count); + +static void mpam_set_el0_partid(unsigned int inst_id, unsigned int data_id) +{ + u64 reg; + + cant_migrate(); + + reg = read_sysreg_s(SYS_MPAM0_EL1); + + FIELD_SET(reg, MPAM0_EL1_PARTID_I, inst_id); + FIELD_SET(reg, MPAM0_EL1_PARTID_D, data_id); + + write_sysreg_s(reg, SYS_MPAM0_EL1); + /* + * Note: if the scope is limited to userspace, we'll get an EL switch + * before getting back to US which will be our context synchronization + * event, so this won't be necessary. + */ + isb(); +} + +/* + * Write the PARTID to use on the local CPU. + */ +void mpam_write_partid(unsigned int partid) +{ + WARN_ON_ONCE(preemptible()); + WARN_ON_ONCE(partid >= mpam_partid_count); + + if (partid == this_cpu_read(mpam_local_partid)) + return; + + this_cpu_write(mpam_local_partid, partid); + mpam_set_el0_partid(partid, partid); +} +EXPORT_SYMBOL_GPL(mpam_write_partid); + +static void mpam_msc_sel_partid(struct mpam_msc *msc, unsigned int id) +{ + u32 reg; + + lockdep_assert_held(&msc->lock); + + reg = readl_relaxed(msc->base + MPAMCFG_PART_SEL); + + FIELD_SET(reg, MPAMCFG_PART_SEL_PARTID_SEL, id); + if (msc->has_ris) + FIELD_SET(reg, MPAMCFG_PART_SEL_RIS, 0); + + writel_relaxed(reg, msc->base + MPAMCFG_PART_SEL); + pr_debug("PART_SEL: 0x%x\n", reg); +} + +static unsigned int mpam_msc_get_partid_max(struct mpam_msc *msc) +{ + lockdep_assert_held(&msc->lock); + + return FIELD_GET(MPAMF_IDR_PARTID_MAX, readq_relaxed(msc->base + MPAMF_IDR)); +} + +static void mpam_msc_set_cpbm(struct mpam_msc *msc, + unsigned int id, + const unsigned long *bitmap) +{ + void __iomem *addr = msc->base + MPAMCFG_CPBM_n; + unsigned int bit = 0, n = 0; + u32 acc = 0; + + lockdep_assert_held(&msc->lock); + + mpam_msc_sel_partid(msc, id); + + /* Single write every reg boundary */ + while (n++ < BITS_TO_U32(msc->cpbm_nbits)) { + for_each_set_bit(bit, bitmap, min_t(unsigned int, + (n * BITS_PER_TYPE(u32)), + msc->cpbm_nbits)) + acc |= 1 << bit % BITS_PER_TYPE(u32); + + writel_relaxed(acc, addr); + pr_debug("CPBM: 0x%x @%px\n", acc, addr); + addr += sizeof(acc); + bit = n*BITS_PER_TYPE(u32); + acc = 0; + } +} + +static void mpam_msc_get_cpbm(struct mpam_msc *msc, + unsigned int id, + unsigned long *bitmap) +{ + void __iomem *addr = msc->base + MPAMCFG_CPBM_n; + size_t regsize = BITS_PER_TYPE(u32); + unsigned int bit; + int n; + + lockdep_assert_held(&msc->lock); + + mpam_msc_sel_partid(msc, id); + + for (n = 0; (n * regsize) < msc->cpbm_nbits; n++) { + unsigned long tmp = readl_relaxed(addr); + + for_each_set_bit(bit, &tmp, min(regsize, msc->cpbm_nbits - (n * regsize))) + bitmap_set(bitmap, bit + (n * regsize), 1); + + addr += regsize; + } +} + +static u16 mpam_msc_get_cmax(struct mpam_msc *msc, unsigned int id) +{ + u32 reg; + u16 res; + + lockdep_assert_held(&msc->lock); + + mpam_msc_sel_partid(msc, id); + + reg = readl_relaxed(msc->base + MPAMCFG_CMAX); + res = FIELD_GET(MPAMCFG_CMAX_CMAX, reg); + return res << msc->cmax_shift; +} + +static void mpam_msc_set_cmax(struct mpam_msc *msc, unsigned int id, u16 val) +{ + lockdep_assert_held(&msc->lock); + + mpam_msc_sel_partid(msc, id); + writel_relaxed(FIELD_PREP(MPAMCFG_CMAX_CMAX, val >> msc->cmax_shift), + msc->base + MPAMCFG_CMAX); +} + +struct mpam_validation_masks { + cpumask_var_t visited_cpus; + cpumask_var_t supported_cpus; + spinlock_t lock; +}; + +static void mpam_validate_cpu(void *info) +{ + struct mpam_validation_masks *masks = (struct mpam_validation_masks *)info; + unsigned int partid_count; + bool valid = true; + + if (!FIELD_GET(ID_AA64PFR0_MPAM, read_sysreg_s(SYS_ID_AA64PFR0_EL1))) { + valid = false; + goto out; + } + + if (!FIELD_GET(MPAM1_EL1_MPAMEN, read_sysreg_s(SYS_MPAM1_EL1))) { + valid = false; + goto out; + } + + partid_count = FIELD_GET(MPAMIDR_EL1_PARTID_MAX, read_sysreg_s(SYS_MPAMIDR_EL1)) + 1; + + spin_lock(&masks->lock); + mpam_partid_count = min(partid_count, mpam_partid_count); + spin_unlock(&masks->lock); +out: + cpumask_set_cpu(smp_processor_id(), masks->visited_cpus); + if (valid) + cpumask_set_cpu(smp_processor_id(), masks->supported_cpus); +} + +/* + * Does the system support MPAM, and if so is it actually usable? + */ +static int mpam_validate_sys(void) +{ + struct mpam_validation_masks masks; + int ret = 0; + + if (!zalloc_cpumask_var(&masks.visited_cpus, GFP_KERNEL)) + return -ENOMEM; + if (!zalloc_cpumask_var(&masks.supported_cpus, GFP_KERNEL)) { + ret = -ENOMEM; + goto out_free_visited; + } + spin_lock_init(&masks.lock); + + on_each_cpu_cond_mask(NULL, mpam_validate_cpu, &masks, true, cpu_present_mask); + + if (!cpumask_equal(masks.visited_cpus, cpu_present_mask)) { + pr_warn("Could not check all CPUs for MPAM settings (visited %*pbl)\n", + cpumask_pr_args(masks.visited_cpus)); + ret = -ENODATA; + goto out; + } + + if (!cpumask_equal(masks.visited_cpus, masks.supported_cpus)) { + pr_warn("MPAM only supported on CPUs [%*pbl]\n", + cpumask_pr_args(masks.supported_cpus)); + ret = -EOPNOTSUPP; + } +out: + free_cpumask_var(masks.supported_cpus); +out_free_visited: + free_cpumask_var(masks.visited_cpus); + + return ret; +} + +static ssize_t mpam_msc_cpbm_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct msc_part_kobj *mpk = container_of(kobj, struct msc_part_kobj, kobj); + unsigned long *bitmap; + unsigned long flags; + size_t size; + + bitmap = bitmap_zalloc(mpk->msc->cpbm_nbits, GFP_KERNEL); + if (!bitmap) + return -ENOMEM; + + spin_lock_irqsave(&mpk->msc->lock, flags); + mpam_msc_get_cpbm(mpk->msc, mpk->partid, bitmap); + spin_unlock_irqrestore(&mpk->msc->lock, flags); + + size = bitmap_print_to_pagebuf(true, buf, bitmap, mpk->msc->cpbm_nbits); + + bitmap_free(bitmap); + return size; +} + +static ssize_t mpam_msc_cpbm_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t size) +{ + struct msc_part_kobj *mpk = container_of(kobj, struct msc_part_kobj, kobj); + unsigned long *bitmap; + unsigned long flags; + int ret; + + bitmap = bitmap_zalloc(mpk->msc->cpbm_nbits, GFP_KERNEL); + if (!bitmap) + return -ENOMEM; + + ret = bitmap_parselist(buf, bitmap, mpk->msc->cpbm_nbits); + if (ret) + goto out_free; + + spin_lock_irqsave(&mpk->msc->lock, flags); + mpam_msc_set_cpbm(mpk->msc, mpk->partid, bitmap); + spin_unlock_irqrestore(&mpk->msc->lock, flags); +out_free: + bitmap_free(bitmap); + return ret ?: size; +} + +static ssize_t mpam_msc_cmax_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct msc_part_kobj *mpk = container_of(kobj, struct msc_part_kobj, kobj); + unsigned long flags; + u16 val; + + spin_lock_irqsave(&mpk->msc->lock, flags); + val = mpam_msc_get_cmax(mpk->msc, mpk->partid); + spin_unlock_irqrestore(&mpk->msc->lock, flags); + + return sprintf(buf, "0x%04x\n", val); +} + +static ssize_t mpam_msc_cmax_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t size) +{ + struct msc_part_kobj *mpk = container_of(kobj, struct msc_part_kobj, kobj); + unsigned long flags; + u16 val; + int ret; + + ret = kstrtou16(buf, 0, &val); + if (ret) + return ret; + + spin_lock_irqsave(&mpk->msc->lock, flags); + mpam_msc_set_cmax(mpk->msc, mpk->partid, val); + spin_unlock_irqrestore(&mpk->msc->lock, flags); + + return size; +} + +static struct kobj_attribute mpam_msc_cpbm_attr = + __ATTR(cpbm, 0644, mpam_msc_cpbm_show, mpam_msc_cpbm_store); + +static struct kobj_attribute mpam_msc_cmax_attr = + __ATTR(cmax, 0644, mpam_msc_cmax_show, mpam_msc_cmax_store); + +static struct attribute *mpam_msc_ctrl_attrs[] = { + &mpam_msc_cpbm_attr.attr, + &mpam_msc_cmax_attr.attr, + NULL, +}; + +static umode_t mpam_msc_ctrl_attr_visible(struct kobject *kobj, + struct attribute *attr, + int n) +{ + struct msc_part_kobj *mpk; + + mpk = container_of(kobj, struct msc_part_kobj, kobj); + + if (attr == &mpam_msc_cpbm_attr.attr && + mpk->msc->has_cpor) + goto visible; + + if (attr == &mpam_msc_cmax_attr.attr && + mpk->msc->has_ccap) + goto visible; + + return 0; + +visible: + return attr->mode; +} + +static struct attribute_group mpam_msc_ctrl_attr_group = { + .attrs = mpam_msc_ctrl_attrs, + .is_visible = mpam_msc_ctrl_attr_visible, +}; + + +static ssize_t mpam_msc_cpbm_nbits_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct mpam_msc *msc = container_of(kobj, struct mpam_msc, ko_root); + + return sprintf(buf, "%u\n", msc->cpbm_nbits); +} + +static ssize_t mpam_msc_cmax_nbits_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct mpam_msc *msc = container_of(kobj, struct mpam_msc, ko_root); + + return sprintf(buf, "%u\n", msc->cmax_nbits); +} + +static struct kobj_attribute mpam_msc_cpbm_nbits_attr = + __ATTR(cpbm_nbits, 0444, mpam_msc_cpbm_nbits_show, NULL); +static struct kobj_attribute mpam_msc_cmax_nbits_attr = + __ATTR(cmax_nbits, 0444, mpam_msc_cmax_nbits_show, NULL); + +static struct attribute *mpam_msc_info_attrs[] = { + &mpam_msc_cpbm_nbits_attr.attr, + &mpam_msc_cmax_nbits_attr.attr, + NULL, +}; + +static umode_t mpam_msc_info_attr_visible(struct kobject *kobj, + struct attribute *attr, + int n) +{ + struct mpam_msc *msc = container_of(kobj, struct mpam_msc, ko_root); + + if (attr == &mpam_msc_cpbm_nbits_attr.attr && + msc->has_cpor) + goto visible; + + if (attr == &mpam_msc_cmax_nbits_attr.attr && + msc->has_ccap) + goto visible; + + return 0; + +visible: + return attr->mode; +} + +static struct attribute_group mpam_msc_info_attr_group = { + .attrs = mpam_msc_info_attrs, + .is_visible = mpam_msc_info_attr_visible, +}; + +static struct kobj_type mpam_kobj_ktype = { + .sysfs_ops = &kobj_sysfs_ops, +}; + +/* + * msc-foo/ + * mpam/ + * cpbm_nbits + * partitions/ + * 0/cpbm + * 1/cpbm + * ... + */ +static int mpam_msc_create_sysfs(struct mpam_msc *msc) +{ + unsigned int partid_count = min(mpam_partid_count, msc->partid_count); + unsigned int part, tmp; + int ret; + + kobject_init(&msc->ko_root, &mpam_kobj_ktype); + ret = kobject_add(&msc->ko_root, &msc->pdev->dev.kobj, "mpam"); + if (ret) + goto err_root; + + kobject_init(&msc->ko_part_dir, &mpam_kobj_ktype); + ret = kobject_add(&msc->ko_part_dir, &msc->ko_root, "partitions"); + if (ret) + goto err_part_dir; + + msc->ko_parts = devm_kzalloc(&msc->pdev->dev, + sizeof(*msc->ko_parts) * partid_count, + GFP_KERNEL); + if (!msc->ko_parts) { + ret = -ENOMEM; + goto err_part_dir; + } + + ret = sysfs_create_group(&msc->ko_root, &mpam_msc_info_attr_group); + if (ret) + goto err_info_grp; + + for (part = 0; part < partid_count; part++) { + kobject_init(&msc->ko_parts[part].kobj, &mpam_kobj_ktype); + msc->ko_parts[part].msc = msc; + msc->ko_parts[part].partid = part; + + ret = kobject_add(&msc->ko_parts[part].kobj, &msc->ko_part_dir, "%d", part); + if (ret) + goto err_parts_add; + } + + for (part = 0; part < partid_count; part++) { + ret = sysfs_create_group(&msc->ko_parts[part].kobj, &mpam_msc_ctrl_attr_group); + if (ret) + goto err_parts_grp; + } + return 0; + +err_parts_grp: + for (tmp = 0; tmp < part; tmp++) + sysfs_remove_group(&msc->ko_parts[part].kobj, &mpam_msc_ctrl_attr_group); + part = partid_count - 1; + +err_parts_add: + for (tmp = 0; tmp < part; tmp++) + kobject_put(&msc->ko_parts[tmp].kobj); + + sysfs_remove_group(&msc->ko_root, &mpam_msc_info_attr_group); + +err_info_grp: + devm_kfree(&msc->pdev->dev, msc->ko_parts); +err_part_dir: + kobject_put(&msc->ko_part_dir); +err_root: + kobject_put(&msc->ko_root); + return ret; +} + +static void mpam_msc_cmax_shift_set(struct mpam_msc *msc) +{ + u16 val; + /* + * Note: The TRM says the implemented bits are the most significant ones, + * but the model doesn't seem to agree with it... + * Handle that in the background, dropping a warning case needed + */ + lockdep_assert_held(&msc->lock); + + if (!(msc->cmax_nbits < 16)) + return; + /* + * Unimplemented bits within the field are RAZ/WI + * At this point the MPAM_CMAX.CMAX will not be adjusted with the shift + * so this operates on an unmodified reg content. + * Also, the default value for CMAX will be set further down the init + * so there is no need for reset here. + */ + mpam_msc_set_cmax(msc, MPAM_PARTID_DEFAULT, GENMASK(15, 0)); + val = mpam_msc_get_cmax(msc, MPAM_PARTID_DEFAULT); + + if (val & GENMASK(15 - msc->cmax_nbits, 0)) { + msc->cmax_shift = 16 - msc->cmax_nbits; + pr_warn("MPAM_CMAX: implemented bits are the least-significant ones!"); + } +} + +static int mpam_msc_initialize(struct mpam_msc *msc) +{ + static unsigned long *bitmap; + int partid; + u64 reg; + + /* + * We're using helpers that expect the lock to be held, but we're + * setting things up and there is no interface yet, so nothing can + * race with us. Make lockdep happy, and save ourselves from a couple + * of lock/unlock. + */ + spin_acquire(&msc->lock.dep_map, 0, 0, _THIS_IP_); + + reg = readq_relaxed(msc->base + MPAMF_IDR); + + msc->has_cpor = FIELD_GET(MPAMF_IDR_HAS_CPOR_PART, reg); + msc->has_ccap = FIELD_GET(MPAMF_IDR_HAS_CCAP_PART, reg); + /* Detect more features here */ + + if (!msc->part_feats) { + pr_err("MSC does not support any recognized partitionning feature\n"); + return -EOPNOTSUPP; + } + + /* Check for features that aren't supported, disable those we can */ + if (FIELD_GET(MPAMF_IDR_HAS_PRI_PART, reg)) { + pr_err("Priority partitionning present but not supported\n"); + return -EOPNOTSUPP; + } + + msc->has_ris = FIELD_GET(MPAMF_IDR_HAS_RIS, reg); + if (msc->has_ris) + pr_warn("RIS present but not supported, only instance 0 will be used\n"); + + /* Error interrupts aren't handled */ + reg = readl_relaxed(msc->base + MPAMF_ECR); + FIELD_SET(reg, MPAMF_ECR_INTEN, 0); + writel_relaxed(reg, msc->base + MPAMF_ECR); + + msc->partid_count = mpam_msc_get_partid_max(msc) + 1; + pr_debug("%d partitions supported\n", msc->partid_count); + if (msc->partid_count > mpam_partid_count) + pr_debug("System limited to %d partitions\n", mpam_partid_count); + + reg = readl_relaxed(msc->base + MPAMF_CPOR_IDR); + msc->cpbm_nbits = FIELD_GET(MPAMF_CPOR_IDR_CPBM_WD, reg); + pr_debug("%d portions supported\n", msc->cpbm_nbits); + + reg = readl_relaxed(msc->base + MPAMF_CCAP_IDR); + msc->cmax_nbits = FIELD_GET(MPAMF_CCAP_IDR_CMAX_WD, reg); + mpam_msc_cmax_shift_set(msc); + + bitmap = bitmap_alloc(mpam_partid_count, GFP_KERNEL); + if (!bitmap) + return -ENOMEM; + + /* + * Make all partitions have a sane default setting. The reference manual + * "suggests" sane defaults, be paranoid. + */ + bitmap_fill(bitmap, mpam_partid_count); + for (partid = 0; partid < mpam_partid_count; partid++) { + mpam_msc_set_cpbm(msc, partid, bitmap); + mpam_msc_set_cmax(msc, partid, + GENMASK(15, 15 - (msc->cmax_nbits -1))); + } + bitmap_free(bitmap); + + spin_release(&msc->lock.dep_map, _THIS_IP_); + + return mpam_msc_create_sysfs(msc); +} + +static int mpam_probe(struct platform_device *pdev) +{ + struct mpam_msc *msc; + struct resource *mem; + void __iomem *base; + int ret; + + msc = devm_kzalloc(&pdev->dev, sizeof(*msc), GFP_KERNEL); + if (!msc) + return -ENOMEM; + + msc->pdev = pdev; + + base = devm_platform_get_and_ioremap_resource(pdev, 0, &mem); + if (IS_ERR(base)) { + devm_kfree(&pdev->dev, msc); + return PTR_ERR(base); + } + + msc->base = base; + spin_lock_init(&msc->lock); + platform_set_drvdata(pdev, msc); + + ret = mpam_msc_initialize(msc); + + return ret; +} + +static const struct of_device_id of_mpam_match[] = { + { + .compatible = "arm,mpam-msc" + }, + { /* end */ }, +}; + +static struct platform_driver mpam_arch_driver = { + .probe = mpam_probe, + .driver = { + .name = "mpam", + .of_match_table = of_mpam_match + }, +}; + +static int __init mpam_arch_driver_init(void) +{ + int ret; + + /* Does the system support MPAM at all? */ + ret = mpam_validate_sys(); + if (ret) + return -EOPNOTSUPP; + + return platform_driver_register(&mpam_arch_driver); +} + +module_init(mpam_arch_driver_init); diff --git a/drivers/android/mpam_arch.h b/drivers/android/mpam_arch.h new file mode 100644 index 0000000000000000000000000000000000000000..1d620c812ac1703cdbed20dc205791dc55fc6041 --- /dev/null +++ b/drivers/android/mpam_arch.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0+ +#ifndef MPAM_ARCH_H +#define MPAM_ARCH_H + +#define MPAM_PARTID_DEFAULT 0 + +extern void mpam_write_partid(unsigned int partid); +extern unsigned int mpam_get_partid_count(void); + +#endif diff --git a/drivers/android/mpam_arch_internal.h b/drivers/android/mpam_arch_internal.h new file mode 100644 index 0000000000000000000000000000000000000000..860f7f9fd470cbfc07226c78b631dfcf2239040e --- /dev/null +++ b/drivers/android/mpam_arch_internal.h @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0+ +#ifndef MPAM_ARCH_INTERNAL_H +#define MPAM_ARCH_INTERNAL_H + +#define ID_AA64PFR0_MPAM BIT(40) + +#define MPAMF_IDR 0x0000 +#define MPAMF_CPOR_IDR 0x0030 +#define MPAMF_CCAP_IDR 0x0038 +#define MPAMF_ECR 0x00F0 + +#define MPAMCFG_PART_SEL 0x0100 +#define MPAMCFG_CMAX 0x0108 +#define MPAMCFG_CPBM_n 0x1000 + +/* Sysregs */ + +/* MPAM0_EL1 */ +#define MPAM0_EL1_PARTID_I GENMASK(15, 0) +#define MPAM0_EL1_PARTID_D GENMASK(31, 16) + +/* MPAM1_EL1 */ +#define MPAM1_EL1_PARTID_I GENMASK(15, 0) +#define MPAM1_EL1_PARTID_D GENMASK(31, 16) +#define MPAM1_EL1_MPAMEN BIT(63) + +/* MPAMIDR_EL1 */ +#define MPAMIDR_EL1_PARTID_MAX GENMASK(15, 0) + +/* MSC MMRs */ + +/* MPAMF_IDR */ +#define MPAMF_IDR_PARTID_MAX GENMASK(15, 0) +#define MPAMF_IDR_HAS_CCAP_PART BIT(24) +#define MPAMF_IDR_HAS_CPOR_PART BIT(25) +#define MPAMF_IDR_HAS_PRI_PART BIT(27) +#define MPAMF_IDR_HAS_RIS GENMASK(32, 31) + +/* MPAMF_CPOR_IDR */ +#define MPAMF_CPOR_IDR_CPBM_WD GENMASK(15, 0) + +/* MPMAMF_CCAP_IDR */ +#define MPAMF_CCAP_IDR_CMAX_WD GENMASK(5, 0) + +/* MPAMF_ECR */ +#define MPAMF_ECR_INTEN BIT(0) + +/* MPAMCFG_PART_SEL */ +#define MPAMCFG_PART_SEL_PARTID_SEL GENMASK(15, 0) +#define MPAMCFG_PART_SEL_RIS GENMASK(27, 24) + +#define MPAMCFG_CMAX_CMAX GENMASK(15, 0) + +#endif diff --git a/include/linux/sched.h b/include/linux/sched.h index d48dd4df44fadeeebe49fbc68a38c3bdd2f67559..c402c97dc585984084b57993c7735a9937c0028c 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1481,6 +1481,7 @@ struct task_struct { #endif ANDROID_VENDOR_DATA_ARRAY(1, 64); ANDROID_OEM_DATA_ARRAY(1, 6); + ANDROID_VENDOR_DATA(2); #ifdef CONFIG_KRETPROBES struct llist_head kretprobe_instances; diff --git a/include/uapi/linux/magic.h b/include/uapi/linux/magic.h index 35687dcb1a42952ab6ecebee1f1b620f21dd510e..aafa6b527378856413400cbddc4761fa299109f4 100644 --- a/include/uapi/linux/magic.h +++ b/include/uapi/linux/magic.h @@ -63,6 +63,7 @@ #define CGROUP2_SUPER_MAGIC 0x63677270 #define RDTGROUP_SUPER_MAGIC 0x7655821 +#define MPAM_POLICY_SUPER_MAGIC 0xBADC0FFEE #define STACK_END_MAGIC 0x57AC6E9D diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 978fcfca5871d76524efb5268288f59dd5e20f71..b88a204b227aae3b0119246096e49b26c9ecd6fc 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -37,3 +37,4 @@ obj-$(CONFIG_MEMBARRIER) += membarrier.o obj-$(CONFIG_CPU_ISOLATION) += isolation.o obj-$(CONFIG_PSI) += psi.o obj-$(CONFIG_SCHED_CORE) += core_sched.o +obj-$(CONFIG_ANDROID_MPAM_POLICY) += mpam_policy.o diff --git a/kernel/sched/mpam_policy.c b/kernel/sched/mpam_policy.c new file mode 100644 index 0000000000000000000000000000000000000000..c46a0bf1a66e277672a4bae5b9cd1dc699be0a17 --- /dev/null +++ b/kernel/sched/mpam_policy.c @@ -0,0 +1,430 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Module interface to map PIDs to MPAM PARTIDs + * + * Copyright (C) 2021 Arm Ltd. + */ + +#define DEBUG + +#define pr_fmt(fmt) "MPAM_policy: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "sched.h" +#include "../../drivers/android/mpam_arch.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Valentin Schneider "); +MODULE_SOFTDEP("mpam_policy pre: mpam_arch"); + +/* + * This presents a virtual fs that looks something like: + * + * /sys/fs/mpam + * partitions/ + * 0/ + * tasks + * 1/ + * tasks + * ... + * $(PARTID_COUNT-1)/ + * tasks + */ + +struct mpam_partition { + unsigned int partid; +}; + +static struct { + struct mpam_partition *partitions; + struct mutex lock; + unsigned int partitions_count; +} mpam_fs = { + .lock = __MUTEX_INITIALIZER(mpam_fs.lock), +}; + +static int mpam_init_fs_context(struct fs_context *fc); +static void mpam_kill_sb(struct super_block *sb); +static int mpam_get_tree(struct fs_context *fc); + +static int mpam_policy_tasks_open(struct inode *inode, struct file *file); +static ssize_t mpam_policy_tasks_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos); + +static void mpam_kick_task(struct task_struct *p); + +static const struct fs_context_operations mpam_fs_context_ops = { + .get_tree = mpam_get_tree, +}; + +static struct file_system_type mpam_fs_type = { + .owner = THIS_MODULE, + .name = "mpam", + .init_fs_context = mpam_init_fs_context, + .kill_sb = mpam_kill_sb, +}; + +static const struct file_operations mpam_fs_tasks_ops = { + .owner = THIS_MODULE, + .open = mpam_policy_tasks_open, + .write = mpam_policy_tasks_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static inline unsigned int mpam_get_task_partid(struct task_struct *p) +{ + return READ_ONCE(p->android_vendor_data2); +} + +static inline void mpam_set_task_partid(struct task_struct *p, unsigned int partid) +{ + WRITE_ONCE(p->android_vendor_data2, partid); + mpam_kick_task(p); +} + +static int mpam_init_fs_context(struct fs_context *fc) +{ + fc->ops = &mpam_fs_context_ops; + put_user_ns(fc->user_ns); + fc->user_ns = get_user_ns(&init_user_ns); + fc->global = true; + return 0; +} + +static void mpam_kill_sb(struct super_block *sb) +{ + kfree(mpam_fs.partitions); + kill_litter_super(sb); +} + +static ssize_t mpam_policy_tasks_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + struct mpam_partition *part; + struct task_struct *p; + struct inode *inode; + /* PID limit is the millions, 7 chars + newline + \0 */ + char buf[9]; + int ret = 0; + pid_t pid; + + if (cnt >= ARRAY_SIZE(buf)) + cnt = ARRAY_SIZE(buf) - 1; + + if (copy_from_user(&buf, ubuf, cnt)) + return -EFAULT; + + buf[cnt] = 0; + + if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0) + return -EINVAL; + + inode = file_inode(filp); + part = (struct mpam_partition *)inode->i_private; + + rcu_read_lock(); + p = find_task_by_vpid(pid); + if (!p) { + ret = -EINVAL; + rcu_read_unlock(); + goto out; + } + + get_task_struct(p); + rcu_read_unlock(); + + mutex_lock(&mpam_fs.lock); + mpam_set_task_partid(p, part->partid); + mutex_unlock(&mpam_fs.lock); + + put_task_struct(p); + + *ppos += cnt; +out: + return ret ?: cnt; +} + +static int mpam_policy_tasks_show(struct seq_file *s, void *v) +{ + struct mpam_partition *part = s->private; + struct task_struct *p, *t; + + mutex_lock(&mpam_fs.lock); + rcu_read_lock(); + for_each_process_thread(p, t) { + if (mpam_get_task_partid(t) == part->partid) + seq_printf(s, "%d\n", t->pid); + } + rcu_read_unlock(); + mutex_unlock(&mpam_fs.lock); + + return 0; +} + +static int mpam_policy_tasks_open(struct inode *inode, struct file *file) +{ + return single_open(file, mpam_policy_tasks_show, inode->i_private); +} + +static const struct super_operations mpam_fs_super_ops = { + .statfs = simple_statfs, +}; + +static struct inode *mpam_fs_create_inode(struct super_block *sb, int mode) +{ + struct inode *ret = new_inode(sb); + + if (ret) { + ret->i_ino = get_next_ino(); + ret->i_mode = mode; + ret->i_atime = ret->i_mtime = ret->i_ctime = current_time(ret); + } + return ret; +} + +static struct dentry *mpam_fs_create_dir(struct dentry *parent, const char *name) +{ + struct dentry *dentry; + struct inode *inode; + + dentry = d_alloc_name(parent, name); + if (!dentry) + return ERR_PTR(-ENOMEM); + + inode = mpam_fs_create_inode(parent->d_sb, S_IFDIR | 0444); + if (!inode) { + dput(dentry); + return ERR_PTR(-ENOMEM); + } + + inode->i_op = &simple_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + + d_add(dentry, inode); + return dentry; +} + +static struct dentry *mpam_fs_create_file(struct dentry *parent, + const char *name, + const struct file_operations *fops, + void *data, + int mode) +{ + struct dentry *dentry; + struct inode *inode; + + dentry = d_alloc_name(parent, name); + if (!dentry) + return ERR_PTR(-ENOMEM); + + inode = mpam_fs_create_inode(parent->d_sb, S_IFREG | mode); + if (!inode) { + dput(dentry); + return ERR_PTR(-ENOMEM); + } + + inode->i_fop = fops; + inode->i_private = data; + + d_add(dentry, inode); + return dentry; +} + +static int mpam_fs_create_files(struct super_block *sb) +{ + struct dentry *parts_dir; + /* Max PARTID is 16 bits aka 65535 */ + char dirname[6]; + int i; + + mpam_fs.partitions_count = mpam_get_partid_count(); + + mpam_fs.partitions = kmalloc_array( + mpam_fs.partitions_count, sizeof(*mpam_fs.partitions), GFP_KERNEL); + if (!mpam_fs.partitions) + return -ENOMEM; + + parts_dir = mpam_fs_create_dir(sb->s_root, "partitions"); + if (IS_ERR(parts_dir)) + return PTR_ERR(parts_dir); + + for (i = 0; i < mpam_get_partid_count(); i++) { + struct mpam_partition *part = &mpam_fs.partitions[i]; + struct dentry *dir; + struct dentry *file; + + part->partid = i; + + snprintf(dirname, sizeof(dirname), "%d", i); + dir = mpam_fs_create_dir(parts_dir, dirname); + if (IS_ERR(dir)) + return PTR_ERR(dir); + file = mpam_fs_create_file(dir, "tasks", &mpam_fs_tasks_ops, part, 0644); + if (IS_ERR(file)) + return PTR_ERR(file); + } + return 0; +} + +static int mpam_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct inode *root; + + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_magic = MPAM_POLICY_SUPER_MAGIC; + sb->s_op = &mpam_fs_super_ops; + + root = mpam_fs_create_inode(sb, S_IFDIR | 0444); + if (!root) + return -ENOMEM; + + root->i_op = &simple_dir_inode_operations; + root->i_fop = &simple_dir_operations; + + sb->s_root = d_make_root(root); + if (!sb->s_root) + return -ENOMEM; + + return mpam_fs_create_files(sb); +} + +static int mpam_get_tree(struct fs_context *fc) +{ + return get_tree_single(fc, mpam_fill_super); +} + +static int __init mpam_policy_fs_init(void) +{ + int ret = 0; + + ret = sysfs_create_mount_point(fs_kobj, "mpam"); + if (ret) + goto err; + + ret = register_filesystem(&mpam_fs_type); + if (ret) + goto err_mount; + + return ret; + +err_mount: + sysfs_remove_mount_point(fs_kobj, "mpam"); +err: + return ret; +} + +static void mpam_policy_fs_exit(void) +{ + unregister_filesystem(&mpam_fs_type); + sysfs_remove_mount_point(fs_kobj, "mpam"); +} + +/* + * Sync @p's associated PARTID with this CPU's register. + */ +static void mpam_sync_task(struct task_struct *p) +{ + mpam_write_partid(mpam_get_task_partid(p)); +} + +/* + * Same as mpam_sync_task(), with a pre-filter for the current task. + */ +static void mpam_sync_current(void *task) +{ + if (task && task != current) + return; + + mpam_sync_task(current); +} + +static bool __task_curr(struct task_struct *p) +{ + return cpu_curr(task_cpu(p)) == p; +} + +static void mpam_kick_task(struct task_struct *p) +{ + /* + * If @p is no longer on the task_cpu(p) we see here when the smp_call + * actually runs, then it had a context switch, so it doesn't need the + * explicit update - no need to chase after it. + */ + if (__task_curr(p)) + smp_call_function_single(task_cpu(p), mpam_sync_current, p, 1); +} + +static void mpam_hook_fork(void __always_unused *data, + struct task_struct *p) +{ + /* + * The task isn't supposed to be runnable yet, so we don't have to issue + * an mpam_sync_task() here. + */ + if (p->sched_reset_on_fork) + mpam_set_task_partid(p, MPAM_PARTID_DEFAULT); +} + +static void mpam_hook_switch(void __always_unused *data, + struct task_struct *prev, struct task_struct *next) +{ + mpam_sync_task(next); +} + +/* + * Default-0 is a sensible thing, and it avoids us having to do anything + * to setup the task_struct vendor data field that serves as partid. + * If it becomes different than zero, we need the following after registering + * the sched_switch hook: + * - a for_each_process_thread() loop, to initialize existing tasks + * - a trace_task_newtask hook, to initialize tasks that are being + * forked and may not be covered by the above loop + */ +static_assert(MPAM_PARTID_DEFAULT == 0); +static int mpam_policy_hooks_init(void) +{ + int ret = 0; + + ret = register_trace_android_vh_is_fpsimd_save(mpam_hook_switch, NULL); + if (ret) + return ret; + + ret = register_trace_android_rvh_sched_fork(mpam_hook_fork, NULL); + if (ret) + unregister_trace_android_vh_is_fpsimd_save(mpam_hook_switch, NULL); + + return ret; +} + +static int __init mpam_policy_init(void) +{ + int ret; + + ret = mpam_policy_fs_init(); + if (ret) + return ret; + + ret = mpam_policy_hooks_init(); + if (ret) + mpam_policy_fs_exit(); + + return ret; +} + +module_init(mpam_policy_init);