/* $NetBSD: bus_space.c,v 1.47 2022/07/17 08:33:48 riastradh Exp $ */ /*- * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace * Simulation Facility, NASA Ames Research Center. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include __KERNEL_RCSID(0, "$NetBSD: bus_space.c,v 1.47 2022/07/17 08:33:48 riastradh Exp $"); #include #include #include #include #include #include #include #include #include #include #ifdef XEN #include #endif /* * Macros for sanity-checking the aligned-ness of pointers passed to * bus space ops. These are not strictly necessary on the x86, but * could lead to performance improvements, and help catch problems * with drivers that would creep up on other architectures. */ #ifdef BUS_SPACE_DEBUG #define BUS_SPACE_ALIGNED_ADDRESS(p, t) \ ((((u_long)(p)) & (sizeof(t)-1)) == 0) #define BUS_SPACE_ADDRESS_SANITY(p, t, d) \ ({ \ if (BUS_SPACE_ALIGNED_ADDRESS((p), t) == 0) { \ printf("%s 0x%lx not aligned to %zu bytes %s:%d\n", \ d, (u_long)(p), sizeof(t), __FILE__, __LINE__); \ } \ (void) 0; \ }) #else #define BUS_SPACE_ADDRESS_SANITY(p,t,d) (void) 0 #endif /* BUS_SPACE_DEBUG */ /* * Extent maps to manage I/O and memory space. Allocate * storage for 8 regions in each, initially. Later, ioport_malloc_safe * will indicate that it's safe to use malloc() to dynamically allocate * region descriptors. * * N.B. At least two regions are _always_ allocated from the iomem * extent map; (0 -> ISA hole) and (end of ISA hole -> end of RAM). * * The extent maps are not static! Machine-dependent ISA and EISA * routines need access to them for bus address space allocation. */ static long ioport_ex_storage[EXTENT_FIXED_STORAGE_SIZE(16) / sizeof(long)]; static long iomem_ex_storage[EXTENT_FIXED_STORAGE_SIZE(64) / sizeof(long)]; struct extent *ioport_ex; struct extent *iomem_ex; static int ioport_malloc_safe; static struct bus_space_tag x86_io = { .bst_type = X86_BUS_SPACE_IO }; static struct bus_space_tag x86_mem = { .bst_type = X86_BUS_SPACE_MEM }; bus_space_tag_t x86_bus_space_io = &x86_io; bus_space_tag_t x86_bus_space_mem = &x86_mem; int x86_mem_add_mapping(bus_addr_t, bus_size_t, int, bus_space_handle_t *); static inline bool x86_bus_space_is_io(bus_space_tag_t t) { return t->bst_type == X86_BUS_SPACE_IO; } static inline bool x86_bus_space_is_mem(bus_space_tag_t t) { return t->bst_type == X86_BUS_SPACE_MEM; } void x86_bus_space_init(void) { /* * Initialize the I/O port and I/O mem extent maps. * Note: we don't have to check the return value since * creation of a fixed extent map will never fail (since * descriptor storage has already been allocated). * * N.B. The iomem extent manages _all_ physical addresses * on the machine. When the amount of RAM is found, the two * extents of RAM are allocated from the map (0 -> ISA hole * and end of ISA hole -> end of RAM). */ ioport_ex = extent_create("ioport", 0x0, 0xffff, (void *)ioport_ex_storage, sizeof(ioport_ex_storage), EX_NOCOALESCE|EX_NOWAIT); iomem_ex = extent_create("iomem", 0x0, MAXIOMEM, (void *)iomem_ex_storage, sizeof(iomem_ex_storage), EX_NOCOALESCE|EX_NOWAIT); #ifdef XENPV /* We are privileged guest os - should have IO privileges. */ if (xendomain_is_privileged()) { struct physdev_set_iopl set_iopl; memset(&set_iopl, 0, sizeof(set_iopl)); set_iopl.iopl = 1; if (HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl) != 0) panic("Unable to obtain IOPL, " "despite being SIF_PRIVILEGED"); } #endif /* XENPV */ } void x86_bus_space_mallocok(void) { ioport_malloc_safe = 1; } int bus_space_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size, int flags, bus_space_handle_t *bshp) { bus_space_reservation_t bsr; bus_space_tag_t it; int error; if ((t->bst_exists & BUS_SPACE_OVERRIDE_MAP) == 0) ; /* skip override */ else for (it = t; it != NULL; it = it->bst_super) { if ((it->bst_present & BUS_SPACE_OVERRIDE_MAP) == 0) continue; return (*it->bst_ov->ov_space_map)(it->bst_ctx, t, bpa, size, flags, bshp); } error = bus_space_reserve(t, bpa, size, flags, &bsr); if (error != 0) return error; error = bus_space_reservation_map(t, &bsr, flags, bshp); if (error != 0) bus_space_release(t, &bsr); return error; } int bus_space_reservation_map(bus_space_tag_t t, bus_space_reservation_t *bsr, int flags, bus_space_handle_t *bshp) { bus_addr_t bpa; bus_size_t size; bus_space_tag_t it; if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVATION_MAP) == 0) ; /* skip override */ else for (it = t; it != NULL; it = it->bst_super) { if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVATION_MAP) == 0) continue; return (*it->bst_ov->ov_space_reservation_map)(it->bst_ctx, t, bsr, flags, bshp); } bpa = bus_space_reservation_addr(bsr); size = bus_space_reservation_size(bsr); /* * For I/O space, that's all she wrote. */ if (x86_bus_space_is_io(t)) { *bshp = bpa; return 0; } #ifndef XENPV if (bpa >= IOM_BEGIN && (bpa + size) != 0 && (bpa + size) <= IOM_END) { *bshp = (bus_space_handle_t)ISA_HOLE_VADDR(bpa); return 0; } #endif /* !XENPV */ /* * For memory space, map the bus physical address to * a kernel virtual address. */ return x86_mem_add_mapping(bpa, size, flags, bshp); } int _x86_memio_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size, int flags, bus_space_handle_t *bshp) { /* * For I/O space, just fill in the handle. */ if (x86_bus_space_is_io(t)) { if (flags & BUS_SPACE_MAP_LINEAR) return (EOPNOTSUPP); *bshp = bpa; return (0); } /* * For memory space, map the bus physical address to * a kernel virtual address. */ return x86_mem_add_mapping(bpa, size, flags, bshp); } int bus_space_reserve(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size, int flags, bus_space_reservation_t *bsrp) { struct extent *ex; int error; bus_space_tag_t it; if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVE) == 0) ; /* skip override */ else for (it = t; it != NULL; it = it->bst_super) { if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVE) == 0) continue; return (*it->bst_ov->ov_space_reserve)(it->bst_ctx, t, bpa, size, flags, bsrp); } /* * Pick the appropriate extent map. */ if (x86_bus_space_is_io(t)) { if (flags & BUS_SPACE_MAP_LINEAR) return (EOPNOTSUPP); ex = ioport_ex; } else if (x86_bus_space_is_mem(t)) ex = iomem_ex; else panic("x86_memio_alloc: bad bus space tag"); /* * Before we go any further, let's make sure that this * region is available. */ error = extent_alloc_region(ex, bpa, size, EX_NOWAIT | (ioport_malloc_safe ? EX_MALLOCOK : 0)); if (error != 0) return error; bus_space_reservation_init(bsrp, bpa, size); return 0; } int bus_space_reserve_subregion(bus_space_tag_t t, bus_addr_t rstart, bus_addr_t rend, const bus_size_t size, const bus_size_t alignment, const bus_size_t boundary, const int flags, bus_space_reservation_t *bsrp) { bus_space_reservation_t bsr; struct extent *ex; u_long bpa; int error; bus_space_tag_t it; if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVE_SUBREGION) == 0) ; /* skip override */ else for (it = t; it != NULL; it = it->bst_super) { if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVE_SUBREGION) == 0) continue; return (*it->bst_ov->ov_space_reserve_subregion)(it->bst_ctx, t, rstart, rend, size, alignment, boundary, flags, bsrp); } /* * Pick the appropriate extent map. */ if (x86_bus_space_is_io(t)) { if (flags & BUS_SPACE_MAP_LINEAR) return (EOPNOTSUPP); ex = ioport_ex; } else if (x86_bus_space_is_mem(t)) ex = iomem_ex; else panic("x86_memio_alloc: bad bus space tag"); /* * Sanity check the allocation against the extent's boundaries. */ rstart = MAX(rstart, ex->ex_start); rend = MIN(rend, ex->ex_end); if (rstart >= rend) panic("x86_memio_alloc: bad region start/end"); /* * Do the requested allocation. */ error = extent_alloc_subregion(ex, rstart, rend, size, alignment, boundary, EX_FAST | EX_NOWAIT | (ioport_malloc_safe ? EX_MALLOCOK : 0), &bpa); if (error) return (error); bus_space_reservation_init(&bsr, bpa, size); *bsrp = bsr; return 0; } void bus_space_release(bus_space_tag_t t, bus_space_reservation_t *bsr) { struct extent *ex; bus_space_tag_t it; if ((t->bst_exists & BUS_SPACE_OVERRIDE_RELEASE) == 0) ; /* skip override */ else for (it = t; it != NULL; it = it->bst_super) { if ((it->bst_present & BUS_SPACE_OVERRIDE_RELEASE) == 0) continue; (*it->bst_ov->ov_space_release)(it->bst_ctx, t, bsr); return; } /* * Pick the appropriate extent map. */ if (x86_bus_space_is_io(t)) { ex = ioport_ex; } else if (x86_bus_space_is_mem(t)) ex = iomem_ex; else panic("x86_memio_alloc: bad bus space tag"); if (extent_free(ex, bus_space_reservation_addr(bsr), bus_space_reservation_size(bsr), EX_NOWAIT | (ioport_malloc_safe ? EX_MALLOCOK : 0))) { printf("%s: pa 0x%jx, size 0x%jx\n", __func__, (uintmax_t)bus_space_reservation_addr(bsr), (uintmax_t)bus_space_reservation_size(bsr)); printf("%s: can't free region\n", __func__); } } int bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart, bus_addr_t rend, bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags, bus_addr_t *bpap, bus_space_handle_t *bshp) { bus_space_reservation_t bsr; bus_space_tag_t it; int error; if ((t->bst_exists & BUS_SPACE_OVERRIDE_ALLOC) == 0) ; /* skip override */ else for (it = t; it != NULL; it = it->bst_super) { if ((it->bst_present & BUS_SPACE_OVERRIDE_ALLOC) == 0) continue; return (*it->bst_ov->ov_space_alloc)(it->bst_ctx, t, rstart, rend, size, alignment, boundary, flags, bpap, bshp); } /* * Do the requested allocation. */ error = bus_space_reserve_subregion(t, rstart, rend, size, alignment, boundary, flags, &bsr); if (error != 0) return error; error = bus_space_reservation_map(t, &bsr, flags, bshp); if (error != 0) bus_space_release(t, &bsr); *bpap = bus_space_reservation_addr(&bsr); return error; } int x86_mem_add_mapping(bus_addr_t bpa, bus_size_t size, int flags, bus_space_handle_t *bshp) { paddr_t pa, endpa; vaddr_t va, sva; u_int pmapflags; pa = x86_trunc_page(bpa); endpa = x86_round_page(bpa + size); pmapflags = PMAP_NOCACHE; if ((flags & BUS_SPACE_MAP_CACHEABLE) != 0) pmapflags = 0; else if (flags & BUS_SPACE_MAP_PREFETCHABLE) pmapflags = PMAP_WRITE_COMBINE; #ifdef DIAGNOSTIC if (endpa != 0 && endpa <= pa) panic("x86_mem_add_mapping: overflow"); #endif #ifdef XENPV if (bpa >= IOM_BEGIN && (bpa + size) != 0 && (bpa + size) <= IOM_END) { sva = (vaddr_t)ISA_HOLE_VADDR(pa); } else #endif /* XENPV */ { sva = uvm_km_alloc(kernel_map, endpa - pa, 0, UVM_KMF_VAONLY | UVM_KMF_NOWAIT); if (sva == 0) return (ENOMEM); } *bshp = (bus_space_handle_t)(sva + (bpa & PGOFSET)); for (va = sva; pa != endpa; pa += PAGE_SIZE, va += PAGE_SIZE) { pmap_kenter_ma(va, pa, VM_PROT_READ | VM_PROT_WRITE, pmapflags); } pmap_update(pmap_kernel()); return 0; } bool bus_space_is_equal(bus_space_tag_t t1, bus_space_tag_t t2) { if (t1 == NULL || t2 == NULL) return false; return t1->bst_type == t2->bst_type; } /* * void _x86_memio_unmap(bus_space_tag bst, bus_space_handle bsh, * bus_size_t size, bus_addr_t *adrp) * * This function unmaps memory- or io-space mapped by the function * _x86_memio_map(). This function works nearly as same as * x86_memio_unmap(), but this function does not ask kernel * built-in extents and returns physical address of the bus space, * for the convenience of the extra extent manager. */ void _x86_memio_unmap(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size, bus_addr_t *adrp) { u_long va, endva; bus_addr_t bpa; /* * Find the correct extent and bus physical address. */ if (x86_bus_space_is_io(t)) { bpa = bsh; } else if (x86_bus_space_is_mem(t)) { if (bsh >= atdevbase && (bsh + size) != 0 && (bsh + size) <= (atdevbase + IOM_SIZE)) { bpa = (bus_addr_t)ISA_PHYSADDR(bsh); } else { va = x86_trunc_page(bsh); endva = x86_round_page(bsh + size); #ifdef DIAGNOSTIC if (endva <= va) { panic("_x86_memio_unmap: overflow"); } #endif if (pmap_extract_ma(pmap_kernel(), va, &bpa) == FALSE) { panic("_x86_memio_unmap:" " wrong virtual address"); } bpa += (bsh & PGOFSET); pmap_kremove(va, endva - va); pmap_update(pmap_kernel()); /* * Free the kernel virtual mapping. */ uvm_km_free(kernel_map, va, endva - va, UVM_KMF_VAONLY); } } else { panic("_x86_memio_unmap: bad bus space tag"); } if (adrp != NULL) { *adrp = bpa; } } static void bus_space_reservation_unmap1(bus_space_tag_t t, const bus_space_handle_t bsh, const bus_size_t size, bus_addr_t *bpap) { u_long va, endva; bus_addr_t bpa; /* * Find the correct extent and bus physical address. */ if (x86_bus_space_is_io(t)) { bpa = bsh; } else if (x86_bus_space_is_mem(t)) { if (bsh >= atdevbase && (bsh + size) != 0 && (bsh + size) <= (atdevbase + IOM_SIZE)) { bpa = (bus_addr_t)ISA_PHYSADDR(bsh); goto ok; } va = x86_trunc_page(bsh); endva = x86_round_page(bsh + size); #ifdef DIAGNOSTIC if (endva <= va) panic("x86_memio_unmap: overflow"); #endif (void) pmap_extract_ma(pmap_kernel(), va, &bpa); bpa += (bsh & PGOFSET); pmap_kremove(va, endva - va); pmap_update(pmap_kernel()); /* * Free the kernel virtual mapping. */ uvm_km_free(kernel_map, va, endva - va, UVM_KMF_VAONLY); } else panic("x86_memio_unmap: bad bus space tag"); ok: if (bpap != NULL) *bpap = bpa; } void bus_space_reservation_unmap(bus_space_tag_t t, const bus_space_handle_t bsh, const bus_size_t size) { bus_space_tag_t it; if ((t->bst_exists & BUS_SPACE_OVERRIDE_RESERVATION_UNMAP) == 0) ; /* skip override */ else for (it = t; it != NULL; it = it->bst_super) { if ((it->bst_present & BUS_SPACE_OVERRIDE_RESERVATION_UNMAP) == 0) continue; (*it->bst_ov->ov_space_reservation_unmap)(it->bst_ctx, t, bsh, size); return; } bus_space_reservation_unmap1(t, bsh, size, NULL); } void bus_space_unmap(bus_space_tag_t t, const bus_space_handle_t bsh, const bus_size_t size) { bus_addr_t addr; bus_space_reservation_t bsr; bus_space_tag_t it; if ((t->bst_exists & BUS_SPACE_OVERRIDE_UNMAP) == 0) ; /* skip override */ else for (it = t; it != NULL; it = it->bst_super) { if ((it->bst_present & BUS_SPACE_OVERRIDE_UNMAP) == 0) continue; (*it->bst_ov->ov_space_unmap)(it->bst_ctx, t, bsh, size); return; } bus_space_reservation_unmap1(t, bsh, size, &addr); bus_space_reservation_init(&bsr, addr, size); bus_space_release(t, &bsr); } void bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size) { bus_space_tag_t it; if ((t->bst_exists & BUS_SPACE_OVERRIDE_FREE) == 0) ; /* skip override */ else for (it = t; it != NULL; it = it->bst_super) { if ((it->bst_present & BUS_SPACE_OVERRIDE_FREE) == 0) continue; (*it->bst_ov->ov_space_free)(it->bst_ctx, t, bsh, size); return; } /* bus_space_unmap() does all that we need to do. */ bus_space_unmap(t, bsh, size); } int bus_space_subregion(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t offset, bus_size_t size, bus_space_handle_t *nbshp) { *nbshp = bsh + offset; return (0); } paddr_t bus_space_mmap(bus_space_tag_t t, bus_addr_t addr, off_t off, int prot, int flags) { paddr_t pflags = 0; /* Can't mmap I/O space. */ if (x86_bus_space_is_io(t)) return (-1); /* * "addr" is the base address of the device we're mapping. * "off" is the offset into that device. * * Note we are called for each "page" in the device that * the upper layers want to map. */ if (flags & BUS_SPACE_MAP_PREFETCHABLE) pflags |= X86_MMAP_FLAG_PREFETCH; return x86_btop(addr + off) | (pflags << X86_MMAP_FLAG_SHIFT); } void bus_space_set_multi_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, uint8_t v, size_t c) { vaddr_t addr = h + o; if (x86_bus_space_is_io(t)) while (c--) outb(addr, v); else while (c--) *(volatile uint8_t *)(addr) = v; } void bus_space_set_multi_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, uint16_t v, size_t c) { vaddr_t addr = h + o; BUS_SPACE_ADDRESS_SANITY(addr, uint16_t, "bus addr"); if (x86_bus_space_is_io(t)) while (c--) outw(addr, v); else while (c--) *(volatile uint16_t *)(addr) = v; } void bus_space_set_multi_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, uint32_t v, size_t c) { vaddr_t addr = h + o; BUS_SPACE_ADDRESS_SANITY(addr, uint32_t, "bus addr"); if (x86_bus_space_is_io(t)) while (c--) outl(addr, v); else while (c--) *(volatile uint32_t *)(addr) = v; } void bus_space_set_region_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, uint8_t v, size_t c) { vaddr_t addr = h + o; if (x86_bus_space_is_io(t)) for (; c != 0; c--, addr++) outb(addr, v); else for (; c != 0; c--, addr++) *(volatile uint8_t *)(addr) = v; } void bus_space_set_region_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, uint16_t v, size_t c) { vaddr_t addr = h + o; BUS_SPACE_ADDRESS_SANITY(addr, uint16_t, "bus addr"); if (x86_bus_space_is_io(t)) for (; c != 0; c--, addr += 2) outw(addr, v); else for (; c != 0; c--, addr += 2) *(volatile uint16_t *)(addr) = v; } void bus_space_set_region_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, uint32_t v, size_t c) { vaddr_t addr = h + o; BUS_SPACE_ADDRESS_SANITY(addr, uint32_t, "bus addr"); if (x86_bus_space_is_io(t)) for (; c != 0; c--, addr += 4) outl(addr, v); else for (; c != 0; c--, addr += 4) *(volatile uint32_t *)(addr) = v; } void bus_space_copy_region_1(bus_space_tag_t t, bus_space_handle_t h1, bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, size_t c) { vaddr_t addr1 = h1 + o1; vaddr_t addr2 = h2 + o2; if (x86_bus_space_is_io(t)) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; c != 0; c--, addr1++, addr2++) outb(addr2, inb(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += (c - 1), addr2 += (c - 1); c != 0; c--, addr1--, addr2--) outb(addr2, inb(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; c != 0; c--, addr1++, addr2++) *(volatile uint8_t *)(addr2) = *(volatile uint8_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += (c - 1), addr2 += (c - 1); c != 0; c--, addr1--, addr2--) *(volatile uint8_t *)(addr2) = *(volatile uint8_t *)(addr1); } } } void bus_space_copy_region_2(bus_space_tag_t t, bus_space_handle_t h1, bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, size_t c) { vaddr_t addr1 = h1 + o1; vaddr_t addr2 = h2 + o2; BUS_SPACE_ADDRESS_SANITY(addr1, uint16_t, "bus addr 1"); BUS_SPACE_ADDRESS_SANITY(addr2, uint16_t, "bus addr 2"); if (x86_bus_space_is_io(t)) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; c != 0; c--, addr1 += 2, addr2 += 2) outw(addr2, inw(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1); c != 0; c--, addr1 -= 2, addr2 -= 2) outw(addr2, inw(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; c != 0; c--, addr1 += 2, addr2 += 2) *(volatile uint16_t *)(addr2) = *(volatile uint16_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1); c != 0; c--, addr1 -= 2, addr2 -= 2) *(volatile uint16_t *)(addr2) = *(volatile uint16_t *)(addr1); } } } void bus_space_copy_region_4(bus_space_tag_t t, bus_space_handle_t h1, bus_size_t o1, bus_space_handle_t h2, bus_size_t o2, size_t c) { vaddr_t addr1 = h1 + o1; vaddr_t addr2 = h2 + o2; BUS_SPACE_ADDRESS_SANITY(addr1, uint32_t, "bus addr 1"); BUS_SPACE_ADDRESS_SANITY(addr2, uint32_t, "bus addr 2"); if (x86_bus_space_is_io(t)) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; c != 0; c--, addr1 += 4, addr2 += 4) outl(addr2, inl(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1); c != 0; c--, addr1 -= 4, addr2 -= 4) outl(addr2, inl(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; c != 0; c--, addr1 += 4, addr2 += 4) *(volatile uint32_t *)(addr2) = *(volatile uint32_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1); c != 0; c--, addr1 -= 4, addr2 -= 4) *(volatile uint32_t *)(addr2) = *(volatile uint32_t *)(addr1); } } } void bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, bus_size_t len, int flags) { /* I/O instructions always happen in program order. */ if (x86_bus_space_is_io(tag)) return; /* * For default mappings, which are mapped with UC-type memory * regions, all loads and stores are issued in program order. * * For BUS_SPACE_MAP_PREFETCHABLE mappings, which are mapped * with WC-type memory regions, loads and stores may be issued * out of order, potentially requiring any of the three x86 * fences -- LFENCE, SFENCE, MFENCE. * * For BUS_SPACE_MAP_CACHEABLE mappings, which are mapped with * WB-type memory regions (like normal memory), store/load may * be reordered to load/store, potentially requiring MFENCE. * * We can't easily tell here how the region was mapped (without * consulting the page tables), so just issue the fence * unconditionally. Chances are either it's necessary or the * cost is small in comparison to device register I/O. * * Reference: * * AMD64 Architecture Programmer's Manual, Volume 2: * System Programming, 24593--Rev. 3.38--November 2021, * Sec. 7.4.2 Memory Barrier Interaction with Memory * Types, Table 7-3, p. 196. * https://web.archive.org/web/20220625040004/https://www.amd.com/system/files/TechDocs/24593.pdf#page=256 */ switch (flags) { case 0: break; case BUS_SPACE_BARRIER_READ: x86_lfence(); break; case BUS_SPACE_BARRIER_WRITE: x86_sfence(); break; case BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE: x86_mfence(); break; default: panic("unknown bus space barrier: 0x%x", (unsigned)flags); } } void * bus_space_vaddr(bus_space_tag_t tag, bus_space_handle_t bsh) { return x86_bus_space_is_mem(tag) ? (void *)bsh : NULL; } static const void * bit_to_function_pointer(const struct bus_space_overrides *ov, uint64_t bit) { switch (bit) { case BUS_SPACE_OVERRIDE_MAP: return ov->ov_space_map; case BUS_SPACE_OVERRIDE_UNMAP: return ov->ov_space_unmap; case BUS_SPACE_OVERRIDE_ALLOC: return ov->ov_space_alloc; case BUS_SPACE_OVERRIDE_FREE: return ov->ov_space_free; case BUS_SPACE_OVERRIDE_RESERVE: return ov->ov_space_reserve; case BUS_SPACE_OVERRIDE_RELEASE: return ov->ov_space_release; case BUS_SPACE_OVERRIDE_RESERVATION_MAP: return ov->ov_space_reservation_map; case BUS_SPACE_OVERRIDE_RESERVATION_UNMAP: return ov->ov_space_reservation_unmap; case BUS_SPACE_OVERRIDE_RESERVE_SUBREGION: return ov->ov_space_reserve_subregion; default: return NULL; } } void bus_space_tag_destroy(bus_space_tag_t bst) { kmem_free(bst, sizeof(struct bus_space_tag)); } int bus_space_tag_create(bus_space_tag_t obst, const uint64_t present, const uint64_t extpresent, const struct bus_space_overrides *ov, void *ctx, bus_space_tag_t *bstp) { uint64_t bit, bits, nbits; bus_space_tag_t bst; const void *fp; if (ov == NULL || present == 0 || extpresent != 0) return EINVAL; bst = kmem_alloc(sizeof(struct bus_space_tag), KM_SLEEP); bst->bst_super = obst; bst->bst_type = obst->bst_type; for (bits = present; bits != 0; bits = nbits) { nbits = bits & (bits - 1); bit = nbits ^ bits; if ((fp = bit_to_function_pointer(ov, bit)) == NULL) { printf("%s: missing bit %" PRIx64 "\n", __func__, bit); goto einval; } } bst->bst_ov = ov; bst->bst_exists = obst->bst_exists | present; bst->bst_present = present; bst->bst_ctx = ctx; *bstp = bst; return 0; einval: kmem_free(bst, sizeof(struct bus_space_tag)); return EINVAL; }