// SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ #include #include #include #include "cxlmem.h" #include "core.h" /** * DOC: cxl core hdm * * Compute Express Link Host Managed Device Memory, starting with the * CXL 2.0 specification, is managed by an array of HDM Decoder register * instances per CXL port and per CXL endpoint. Define common helpers * for enumerating these registers and capabilities. */ DECLARE_RWSEM(cxl_dpa_rwsem); static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, int *target_map) { int rc; rc = cxl_decoder_add_locked(cxld, target_map); if (rc) { put_device(&cxld->dev); dev_err(&port->dev, "Failed to add decoder\n"); return rc; } rc = cxl_decoder_autoremove(&port->dev, cxld); if (rc) return rc; dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev)); return 0; } /* * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure) * single ported host-bridges need not publish a decoder capability when a * passthrough decode can be assumed, i.e. all transactions that the uport sees * are claimed and passed to the single dport. Disable the range until the first * CXL region is enumerated / activated. */ int devm_cxl_add_passthrough_decoder(struct cxl_port *port) { struct cxl_switch_decoder *cxlsd; struct cxl_dport *dport = NULL; int single_port_map[1]; unsigned long index; struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); /* * Capability checks are moot for passthrough decoders, support * any and all possibilities. */ cxlhdm->interleave_mask = ~0U; cxlhdm->iw_cap_mask = ~0UL; cxlsd = cxl_switch_decoder_alloc(port, 1); if (IS_ERR(cxlsd)) return PTR_ERR(cxlsd); device_lock_assert(&port->dev); xa_for_each(&port->dports, index, dport) break; single_port_map[0] = dport->port_id; return add_hdm_decoder(port, &cxlsd->cxld, single_port_map); } EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL); static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm) { u32 hdm_cap; hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET); cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap); cxlhdm->target_count = FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap); if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap)) cxlhdm->interleave_mask |= GENMASK(11, 8); if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap)) cxlhdm->interleave_mask |= GENMASK(14, 12); cxlhdm->iw_cap_mask = BIT(1) | BIT(2) | BIT(4) | BIT(8); if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY, hdm_cap)) cxlhdm->iw_cap_mask |= BIT(3) | BIT(6) | BIT(12); if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_16_WAY, hdm_cap)) cxlhdm->iw_cap_mask |= BIT(16); } static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) { struct cxl_hdm *cxlhdm; void __iomem *hdm; u32 ctrl; int i; if (!info) return false; cxlhdm = dev_get_drvdata(&info->port->dev); hdm = cxlhdm->regs.hdm_decoder; if (!hdm) return true; /* * If HDM decoders are present and the driver is in control of * Mem_Enable skip DVSEC based emulation */ if (!info->mem_enabled) return false; /* * If any decoders are committed already, there should not be any * emulated DVSEC decoders. */ for (i = 0; i < cxlhdm->decoder_count; i++) { ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); dev_dbg(&info->port->dev, "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n", info->port->id, i, FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl), readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)), readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)), readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)), readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i))); if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) return false; } return true; } /** * devm_cxl_setup_hdm - map HDM decoder component registers * @port: cxl_port to map * @info: cached DVSEC range register info */ struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, struct cxl_endpoint_dvsec_info *info) { struct cxl_register_map *reg_map = &port->reg_map; struct device *dev = &port->dev; struct cxl_hdm *cxlhdm; int rc; cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL); if (!cxlhdm) return ERR_PTR(-ENOMEM); cxlhdm->port = port; dev_set_drvdata(dev, cxlhdm); /* Memory devices can configure device HDM using DVSEC range regs. */ if (reg_map->resource == CXL_RESOURCE_NONE) { if (!info || !info->mem_enabled) { dev_err(dev, "No component registers mapped\n"); return ERR_PTR(-ENXIO); } cxlhdm->decoder_count = info->ranges; return cxlhdm; } if (!reg_map->component_map.hdm_decoder.valid) { dev_dbg(&port->dev, "HDM decoder registers not implemented\n"); /* unique error code to indicate no HDM decoder capability */ return ERR_PTR(-ENODEV); } rc = cxl_map_component_regs(reg_map, &cxlhdm->regs, BIT(CXL_CM_CAP_CAP_ID_HDM)); if (rc) { dev_err(dev, "Failed to map HDM capability.\n"); return ERR_PTR(rc); } parse_hdm_decoder_caps(cxlhdm); if (cxlhdm->decoder_count == 0) { dev_err(dev, "Spec violation. Caps invalid\n"); return ERR_PTR(-ENXIO); } /* * Now that the hdm capability is parsed, decide if range * register emulation is needed and fixup cxlhdm accordingly. */ if (should_emulate_decoders(info)) { dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges, info->ranges > 1 ? "s" : ""); cxlhdm->decoder_count = info->ranges; } return cxlhdm; } EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL); static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth) { unsigned long long start = r->start, end = r->end; seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end, r->name); } void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds) { struct resource *p1, *p2; down_read(&cxl_dpa_rwsem); for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) { __cxl_dpa_debug(file, p1, 0); for (p2 = p1->child; p2; p2 = p2->sibling) __cxl_dpa_debug(file, p2, 1); } up_read(&cxl_dpa_rwsem); } EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL); /* * Must be called in a context that synchronizes against this decoder's * port ->remove() callback (like an endpoint decoder sysfs attribute) */ static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled) { struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); struct cxl_port *port = cxled_to_port(cxled); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct resource *res = cxled->dpa_res; resource_size_t skip_start; lockdep_assert_held_write(&cxl_dpa_rwsem); /* save @skip_start, before @res is released */ skip_start = res->start - cxled->skip; __release_region(&cxlds->dpa_res, res->start, resource_size(res)); if (cxled->skip) __release_region(&cxlds->dpa_res, skip_start, cxled->skip); cxled->skip = 0; cxled->dpa_res = NULL; put_device(&cxled->cxld.dev); port->hdm_end--; } static void cxl_dpa_release(void *cxled) { down_write(&cxl_dpa_rwsem); __cxl_dpa_release(cxled); up_write(&cxl_dpa_rwsem); } /* * Must be called from context that will not race port device * unregistration, like decoder sysfs attribute methods */ static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled) { struct cxl_port *port = cxled_to_port(cxled); lockdep_assert_held_write(&cxl_dpa_rwsem); devm_remove_action(&port->dev, cxl_dpa_release, cxled); __cxl_dpa_release(cxled); } static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, resource_size_t base, resource_size_t len, resource_size_t skipped) { struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); struct cxl_port *port = cxled_to_port(cxled); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct device *dev = &port->dev; struct resource *res; lockdep_assert_held_write(&cxl_dpa_rwsem); if (!len) { dev_warn(dev, "decoder%d.%d: empty reservation attempted\n", port->id, cxled->cxld.id); return -EINVAL; } if (cxled->dpa_res) { dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n", port->id, cxled->cxld.id, cxled->dpa_res); return -EBUSY; } if (port->hdm_end + 1 != cxled->cxld.id) { /* * Assumes alloc and commit order is always in hardware instance * order per expectations from 8.2.5.12.20 Committing Decoder * Programming that enforce decoder[m] committed before * decoder[m+1] commit start. */ dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id, cxled->cxld.id, port->id, port->hdm_end + 1); return -EBUSY; } if (skipped) { res = __request_region(&cxlds->dpa_res, base - skipped, skipped, dev_name(&cxled->cxld.dev), 0); if (!res) { dev_dbg(dev, "decoder%d.%d: failed to reserve skipped space\n", port->id, cxled->cxld.id); return -EBUSY; } } res = __request_region(&cxlds->dpa_res, base, len, dev_name(&cxled->cxld.dev), 0); if (!res) { dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n", port->id, cxled->cxld.id); if (skipped) __release_region(&cxlds->dpa_res, base - skipped, skipped); return -EBUSY; } cxled->dpa_res = res; cxled->skip = skipped; if (resource_contains(&cxlds->pmem_res, res)) cxled->mode = CXL_DECODER_PMEM; else if (resource_contains(&cxlds->ram_res, res)) cxled->mode = CXL_DECODER_RAM; else { dev_warn(dev, "decoder%d.%d: %pr mixed mode not supported\n", port->id, cxled->cxld.id, cxled->dpa_res); cxled->mode = CXL_DECODER_MIXED; } port->hdm_end++; get_device(&cxled->cxld.dev); return 0; } int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, resource_size_t base, resource_size_t len, resource_size_t skipped) { struct cxl_port *port = cxled_to_port(cxled); int rc; down_write(&cxl_dpa_rwsem); rc = __cxl_dpa_reserve(cxled, base, len, skipped); up_write(&cxl_dpa_rwsem); if (rc) return rc; return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); } EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, CXL); resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled) { resource_size_t size = 0; down_read(&cxl_dpa_rwsem); if (cxled->dpa_res) size = resource_size(cxled->dpa_res); up_read(&cxl_dpa_rwsem); return size; } resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled) { resource_size_t base = -1; lockdep_assert_held(&cxl_dpa_rwsem); if (cxled->dpa_res) base = cxled->dpa_res->start; return base; } int cxl_dpa_free(struct cxl_endpoint_decoder *cxled) { struct cxl_port *port = cxled_to_port(cxled); struct device *dev = &cxled->cxld.dev; int rc; down_write(&cxl_dpa_rwsem); if (!cxled->dpa_res) { rc = 0; goto out; } if (cxled->cxld.region) { dev_dbg(dev, "decoder assigned to: %s\n", dev_name(&cxled->cxld.region->dev)); rc = -EBUSY; goto out; } if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { dev_dbg(dev, "decoder enabled\n"); rc = -EBUSY; goto out; } if (cxled->cxld.id != port->hdm_end) { dev_dbg(dev, "expected decoder%d.%d\n", port->id, port->hdm_end); rc = -EBUSY; goto out; } devm_cxl_dpa_release(cxled); rc = 0; out: up_write(&cxl_dpa_rwsem); return rc; } int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled, enum cxl_decoder_mode mode) { struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct device *dev = &cxled->cxld.dev; int rc; switch (mode) { case CXL_DECODER_RAM: case CXL_DECODER_PMEM: break; default: dev_dbg(dev, "unsupported mode: %d\n", mode); return -EINVAL; } down_write(&cxl_dpa_rwsem); if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { rc = -EBUSY; goto out; } /* * Only allow modes that are supported by the current partition * configuration */ if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) { dev_dbg(dev, "no available pmem capacity\n"); rc = -ENXIO; goto out; } if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) { dev_dbg(dev, "no available ram capacity\n"); rc = -ENXIO; goto out; } cxled->mode = mode; rc = 0; out: up_write(&cxl_dpa_rwsem); return rc; } int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size) { struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); resource_size_t free_ram_start, free_pmem_start; struct cxl_port *port = cxled_to_port(cxled); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct device *dev = &cxled->cxld.dev; resource_size_t start, avail, skip; struct resource *p, *last; int rc; down_write(&cxl_dpa_rwsem); if (cxled->cxld.region) { dev_dbg(dev, "decoder attached to %s\n", dev_name(&cxled->cxld.region->dev)); rc = -EBUSY; goto out; } if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { dev_dbg(dev, "decoder enabled\n"); rc = -EBUSY; goto out; } for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling) last = p; if (last) free_ram_start = last->end + 1; else free_ram_start = cxlds->ram_res.start; for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling) last = p; if (last) free_pmem_start = last->end + 1; else free_pmem_start = cxlds->pmem_res.start; if (cxled->mode == CXL_DECODER_RAM) { start = free_ram_start; avail = cxlds->ram_res.end - start + 1; skip = 0; } else if (cxled->mode == CXL_DECODER_PMEM) { resource_size_t skip_start, skip_end; start = free_pmem_start; avail = cxlds->pmem_res.end - start + 1; skip_start = free_ram_start; /* * If some pmem is already allocated, then that allocation * already handled the skip. */ if (cxlds->pmem_res.child && skip_start == cxlds->pmem_res.child->start) skip_end = skip_start - 1; else skip_end = start - 1; skip = skip_end - skip_start + 1; } else { dev_dbg(dev, "mode not set\n"); rc = -EINVAL; goto out; } if (size > avail) { dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size, cxl_decoder_mode_name(cxled->mode), &avail); rc = -ENOSPC; goto out; } rc = __cxl_dpa_reserve(cxled, start, size, skip); out: up_write(&cxl_dpa_rwsem); if (rc) return rc; return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); } static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl) { u16 eig; u8 eiw; /* * Input validation ensures these warns never fire, but otherwise * suppress unititalized variable usage warnings. */ if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw), "invalid interleave_ways: %d\n", cxld->interleave_ways)) return; if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig), "invalid interleave_granularity: %d\n", cxld->interleave_granularity)) return; u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK); u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK); *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT; } static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl) { u32p_replace_bits(ctrl, !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM), CXL_HDM_DECODER0_CTRL_HOSTONLY); } static void cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt) { struct cxl_dport **t = &cxlsd->target[0]; int ways = cxlsd->cxld.interleave_ways; *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id); if (ways > 1) *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id); if (ways > 2) *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id); if (ways > 3) *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id); if (ways > 4) *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id); if (ways > 5) *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id); if (ways > 6) *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id); if (ways > 7) *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id); } /* * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set * committed or error within 10ms, but just be generous with 20ms to account for * clock skew and other marginal behavior */ #define COMMIT_TIMEOUT_MS 20 static int cxld_await_commit(void __iomem *hdm, int id) { u32 ctrl; int i; for (i = 0; i < COMMIT_TIMEOUT_MS; i++) { ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) { ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT; writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); return -EIO; } if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) return 0; fsleep(1000); } return -ETIMEDOUT; } static int cxl_decoder_commit(struct cxl_decoder *cxld) { struct cxl_port *port = to_cxl_port(cxld->dev.parent); struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); void __iomem *hdm = cxlhdm->regs.hdm_decoder; int id = cxld->id, rc; u64 base, size; u32 ctrl; if (cxld->flags & CXL_DECODER_F_ENABLE) return 0; if (cxl_num_decoders_committed(port) != id) { dev_dbg(&port->dev, "%s: out of order commit, expected decoder%d.%d\n", dev_name(&cxld->dev), port->id, cxl_num_decoders_committed(port)); return -EBUSY; } /* * For endpoint decoders hosted on CXL memory devices that * support the sanitize operation, make sure sanitize is not in-flight. */ if (is_endpoint_decoder(&cxld->dev)) { struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(&cxld->dev); struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); if (mds && mds->security.sanitize_active) { dev_dbg(&cxlmd->dev, "attempted to commit %s during sanitize\n", dev_name(&cxld->dev)); return -EBUSY; } } down_read(&cxl_dpa_rwsem); /* common decoder settings */ ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id)); cxld_set_interleave(cxld, &ctrl); cxld_set_type(cxld, &ctrl); base = cxld->hpa_range.start; size = range_len(&cxld->hpa_range); writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id)); writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id)); writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id)); writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id)); if (is_switch_decoder(&cxld->dev)) { struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(&cxld->dev); void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id); void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id); u64 targets; cxlsd_set_targets(cxlsd, &targets); writel(upper_32_bits(targets), tl_hi); writel(lower_32_bits(targets), tl_lo); } else { struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(&cxld->dev); void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id); void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id); writel(upper_32_bits(cxled->skip), sk_hi); writel(lower_32_bits(cxled->skip), sk_lo); } writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); up_read(&cxl_dpa_rwsem); port->commit_end++; rc = cxld_await_commit(hdm, cxld->id); if (rc) { dev_dbg(&port->dev, "%s: error %d committing decoder\n", dev_name(&cxld->dev), rc); cxld->reset(cxld); return rc; } cxld->flags |= CXL_DECODER_F_ENABLE; return 0; } static int cxl_decoder_reset(struct cxl_decoder *cxld) { struct cxl_port *port = to_cxl_port(cxld->dev.parent); struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); void __iomem *hdm = cxlhdm->regs.hdm_decoder; int id = cxld->id; u32 ctrl; if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0) return 0; if (port->commit_end != id) { dev_dbg(&port->dev, "%s: out of order reset, expected decoder%d.%d\n", dev_name(&cxld->dev), port->id, port->commit_end); return -EBUSY; } down_read(&cxl_dpa_rwsem); ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT; writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id)); writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id)); writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id)); writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id)); up_read(&cxl_dpa_rwsem); port->commit_end--; cxld->flags &= ~CXL_DECODER_F_ENABLE; /* Userspace is now responsible for reconfiguring this decoder */ if (is_endpoint_decoder(&cxld->dev)) { struct cxl_endpoint_decoder *cxled; cxled = to_cxl_endpoint_decoder(&cxld->dev); cxled->state = CXL_DECODER_STATE_MANUAL; } return 0; } static int cxl_setup_hdm_decoder_from_dvsec( struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base, int which, struct cxl_endpoint_dvsec_info *info) { struct cxl_endpoint_decoder *cxled; u64 len; int rc; if (!is_cxl_endpoint(port)) return -EOPNOTSUPP; cxled = to_cxl_endpoint_decoder(&cxld->dev); len = range_len(&info->dvsec_range[which]); if (!len) return -ENOENT; cxld->target_type = CXL_DECODER_HOSTONLYMEM; cxld->commit = NULL; cxld->reset = NULL; cxld->hpa_range = info->dvsec_range[which]; /* * Set the emulated decoder as locked pending additional support to * change the range registers at run time. */ cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK; port->commit_end = cxld->id; rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0); if (rc) { dev_err(&port->dev, "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc); return rc; } *dpa_base += len; cxled->state = CXL_DECODER_STATE_AUTO; return 0; } static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, int *target_map, void __iomem *hdm, int which, u64 *dpa_base, struct cxl_endpoint_dvsec_info *info) { struct cxl_endpoint_decoder *cxled = NULL; u64 size, base, skip, dpa_size, lo, hi; bool committed; u32 remainder; int i, rc; u32 ctrl; union { u64 value; unsigned char target_id[8]; } target_list; if (should_emulate_decoders(info)) return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base, which, info); ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which)); lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which)); hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which)); base = (hi << 32) + lo; lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which)); hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which)); size = (hi << 32) + lo; committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED); cxld->commit = cxl_decoder_commit; cxld->reset = cxl_decoder_reset; if (!committed) size = 0; if (base == U64_MAX || size == U64_MAX) { dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n", port->id, cxld->id); return -ENXIO; } if (info) cxled = to_cxl_endpoint_decoder(&cxld->dev); cxld->hpa_range = (struct range) { .start = base, .end = base + size - 1, }; /* decoders are enabled if committed */ if (committed) { cxld->flags |= CXL_DECODER_F_ENABLE; if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK) cxld->flags |= CXL_DECODER_F_LOCK; if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl)) cxld->target_type = CXL_DECODER_HOSTONLYMEM; else cxld->target_type = CXL_DECODER_DEVMEM; guard(rwsem_write)(&cxl_region_rwsem); if (cxld->id != cxl_num_decoders_committed(port)) { dev_warn(&port->dev, "decoder%d.%d: Committed out of order\n", port->id, cxld->id); return -ENXIO; } if (size == 0) { dev_warn(&port->dev, "decoder%d.%d: Committed with zero size\n", port->id, cxld->id); return -ENXIO; } port->commit_end = cxld->id; } else { if (cxled) { struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); struct cxl_dev_state *cxlds = cxlmd->cxlds; /* * Default by devtype until a device arrives that needs * more precision. */ if (cxlds->type == CXL_DEVTYPE_CLASSMEM) cxld->target_type = CXL_DECODER_HOSTONLYMEM; else cxld->target_type = CXL_DECODER_DEVMEM; } else { /* To be overridden by region type at commit time */ cxld->target_type = CXL_DECODER_HOSTONLYMEM; } if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) && cxld->target_type == CXL_DECODER_HOSTONLYMEM) { ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY; writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which)); } } rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl), &cxld->interleave_ways); if (rc) { dev_warn(&port->dev, "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n", port->id, cxld->id, ctrl); return rc; } rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl), &cxld->interleave_granularity); if (rc) { dev_warn(&port->dev, "decoder%d.%d: Invalid interleave granularity (ctrl: %#x)\n", port->id, cxld->id, ctrl); return rc; } dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n", port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end, cxld->interleave_ways, cxld->interleave_granularity); if (!cxled) { lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which)); hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which)); target_list.value = (hi << 32) + lo; for (i = 0; i < cxld->interleave_ways; i++) target_map[i] = target_list.target_id[i]; return 0; } if (!committed) return 0; dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder); if (remainder) { dev_err(&port->dev, "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n", port->id, cxld->id, size, cxld->interleave_ways); return -ENXIO; } lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which)); hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which)); skip = (hi << 32) + lo; rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip); if (rc) { dev_err(&port->dev, "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", port->id, cxld->id, *dpa_base, *dpa_base + dpa_size + skip - 1, rc); return rc; } *dpa_base += dpa_size + skip; cxled->state = CXL_DECODER_STATE_AUTO; return 0; } static void cxl_settle_decoders(struct cxl_hdm *cxlhdm) { void __iomem *hdm = cxlhdm->regs.hdm_decoder; int committed, i; u32 ctrl; if (!hdm) return; /* * Since the register resource was recently claimed via request_region() * be careful about trusting the "not-committed" status until the commit * timeout has elapsed. The commit timeout is 10ms (CXL 2.0 * 8.2.5.12.20), but double it to be tolerant of any clock skew between * host and target. */ for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) { ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) committed++; } /* ensure that future checks of committed can be trusted */ if (committed != cxlhdm->decoder_count) msleep(20); } /** * devm_cxl_enumerate_decoders - add decoder objects per HDM register set * @cxlhdm: Structure to populate with HDM capabilities * @info: cached DVSEC range register info */ int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm, struct cxl_endpoint_dvsec_info *info) { void __iomem *hdm = cxlhdm->regs.hdm_decoder; struct cxl_port *port = cxlhdm->port; int i; u64 dpa_base = 0; cxl_settle_decoders(cxlhdm); for (i = 0; i < cxlhdm->decoder_count; i++) { int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 }; int rc, target_count = cxlhdm->target_count; struct cxl_decoder *cxld; if (is_cxl_endpoint(port)) { struct cxl_endpoint_decoder *cxled; cxled = cxl_endpoint_decoder_alloc(port); if (IS_ERR(cxled)) { dev_warn(&port->dev, "Failed to allocate decoder%d.%d\n", port->id, i); return PTR_ERR(cxled); } cxld = &cxled->cxld; } else { struct cxl_switch_decoder *cxlsd; cxlsd = cxl_switch_decoder_alloc(port, target_count); if (IS_ERR(cxlsd)) { dev_warn(&port->dev, "Failed to allocate decoder%d.%d\n", port->id, i); return PTR_ERR(cxlsd); } cxld = &cxlsd->cxld; } rc = init_hdm_decoder(port, cxld, target_map, hdm, i, &dpa_base, info); if (rc) { dev_warn(&port->dev, "Failed to initialize decoder%d.%d\n", port->id, i); put_device(&cxld->dev); return rc; } rc = add_hdm_decoder(port, cxld, target_map); if (rc) { dev_warn(&port->dev, "Failed to add decoder%d.%d\n", port->id, i); return rc; } } return 0; } EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);