diff options
Diffstat (limited to 'arch/tile/mm/fault.c')
| -rw-r--r-- | arch/tile/mm/fault.c | 924 |
1 files changed, 0 insertions, 924 deletions
diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c deleted file mode 100644 index f58fa06a2214..000000000000 --- a/arch/tile/mm/fault.c +++ /dev/null @@ -1,924 +0,0 @@ -/* - * Copyright 2010 Tilera Corporation. All Rights Reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, version 2. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or - * NON INFRINGEMENT. See the GNU General Public License for - * more details. - * - * From i386 code copyright (C) 1995 Linus Torvalds - */ - -#include <linux/signal.h> -#include <linux/sched.h> -#include <linux/sched/debug.h> -#include <linux/sched/task.h> -#include <linux/sched/task_stack.h> -#include <linux/kernel.h> -#include <linux/errno.h> -#include <linux/string.h> -#include <linux/types.h> -#include <linux/ptrace.h> -#include <linux/mman.h> -#include <linux/mm.h> -#include <linux/smp.h> -#include <linux/interrupt.h> -#include <linux/init.h> -#include <linux/tty.h> -#include <linux/vt_kern.h> /* For unblank_screen() */ -#include <linux/highmem.h> -#include <linux/extable.h> -#include <linux/kprobes.h> -#include <linux/hugetlb.h> -#include <linux/syscalls.h> -#include <linux/uaccess.h> -#include <linux/kdebug.h> - -#include <asm/pgalloc.h> -#include <asm/sections.h> -#include <asm/traps.h> -#include <asm/syscalls.h> - -#include <arch/interrupts.h> - -static noinline void force_sig_info_fault(const char *type, int si_signo, - int si_code, unsigned long address, - int fault_num, - struct task_struct *tsk, - struct pt_regs *regs) -{ - siginfo_t info; - - if (unlikely(tsk->pid < 2)) { - panic("Signal %d (code %d) at %#lx sent to %s!", - si_signo, si_code & 0xffff, address, - is_idle_task(tsk) ? "the idle task" : "init"); - } - - info.si_signo = si_signo; - info.si_errno = 0; - info.si_code = si_code; - info.si_addr = (void __user *)address; - info.si_trapno = fault_num; - trace_unhandled_signal(type, regs, address, si_signo); - force_sig_info(si_signo, &info, tsk); -} - -#ifndef __tilegx__ -/* - * Synthesize the fault a PL0 process would get by doing a word-load of - * an unaligned address or a high kernel address. - */ -SYSCALL_DEFINE1(cmpxchg_badaddr, unsigned long, address) -{ - struct pt_regs *regs = current_pt_regs(); - - if (address >= PAGE_OFFSET) - force_sig_info_fault("atomic segfault", SIGSEGV, SEGV_MAPERR, - address, INT_DTLB_MISS, current, regs); - else - force_sig_info_fault("atomic alignment fault", SIGBUS, - BUS_ADRALN, address, - INT_UNALIGN_DATA, current, regs); - - /* - * Adjust pc to point at the actual instruction, which is unusual - * for syscalls normally, but is appropriate when we are claiming - * that a syscall swint1 caused a page fault or bus error. - */ - regs->pc -= 8; - - /* - * Mark this as a caller-save interrupt, like a normal page fault, - * so that when we go through the signal handler path we will - * properly restore r0, r1, and r2 for the signal handler arguments. - */ - regs->flags |= PT_FLAGS_CALLER_SAVES; - - return 0; -} -#endif - -static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address) -{ - unsigned index = pgd_index(address); - pgd_t *pgd_k; - pud_t *pud, *pud_k; - pmd_t *pmd, *pmd_k; - - pgd += index; - pgd_k = init_mm.pgd + index; - - if (!pgd_present(*pgd_k)) - return NULL; - - pud = pud_offset(pgd, address); - pud_k = pud_offset(pgd_k, address); - if (!pud_present(*pud_k)) - return NULL; - - pmd = pmd_offset(pud, address); - pmd_k = pmd_offset(pud_k, address); - if (!pmd_present(*pmd_k)) - return NULL; - if (!pmd_present(*pmd)) - set_pmd(pmd, *pmd_k); - else - BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k)); - return pmd_k; -} - -/* - * Handle a fault on the vmalloc area. - */ -static inline int vmalloc_fault(pgd_t *pgd, unsigned long address) -{ - pmd_t *pmd_k; - pte_t *pte_k; - - /* Make sure we are in vmalloc area */ - if (!(address >= VMALLOC_START && address < VMALLOC_END)) - return -1; - - /* - * Synchronize this task's top level page-table - * with the 'reference' page table. - */ - pmd_k = vmalloc_sync_one(pgd, address); - if (!pmd_k) - return -1; - pte_k = pte_offset_kernel(pmd_k, address); - if (!pte_present(*pte_k)) - return -1; - return 0; -} - -/* Wait until this PTE has completed migration. */ -static void wait_for_migration(pte_t *pte) -{ - if (pte_migrating(*pte)) { - /* - * Wait until the migrater fixes up this pte. - * We scale the loop count by the clock rate so we'll wait for - * a few seconds here. - */ - int retries = 0; - int bound = get_clock_rate(); - while (pte_migrating(*pte)) { - barrier(); - if (++retries > bound) - panic("Hit migrating PTE (%#llx) and page PFN %#lx still migrating", - pte->val, pte_pfn(*pte)); - } - } -} - -/* - * It's not generally safe to use "current" to get the page table pointer, - * since we might be running an oprofile interrupt in the middle of a - * task switch. - */ -static pgd_t *get_current_pgd(void) -{ - HV_Context ctx = hv_inquire_context(); - unsigned long pgd_pfn = ctx.page_table >> PAGE_SHIFT; - struct page *pgd_page = pfn_to_page(pgd_pfn); - BUG_ON(PageHighMem(pgd_page)); - return (pgd_t *) __va(ctx.page_table); -} - -/* - * We can receive a page fault from a migrating PTE at any time. - * Handle it by just waiting until the fault resolves. - * - * It's also possible to get a migrating kernel PTE that resolves - * itself during the downcall from hypervisor to Linux. We just check - * here to see if the PTE seems valid, and if so we retry it. - * - * NOTE! We MUST NOT take any locks for this case. We may be in an - * interrupt or a critical region, and must do as little as possible. - * Similarly, we can't use atomic ops here, since we may be handling a - * fault caused by an atomic op access. - * - * If we find a migrating PTE while we're in an NMI context, and we're - * at a PC that has a registered exception handler, we don't wait, - * since this thread may (e.g.) have been interrupted while migrating - * its own stack, which would then cause us to self-deadlock. - */ -static int handle_migrating_pte(pgd_t *pgd, int fault_num, - unsigned long address, unsigned long pc, - int is_kernel_mode, int write) -{ - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - pte_t pteval; - - if (pgd_addr_invalid(address)) - return 0; - - pgd += pgd_index(address); - pud = pud_offset(pgd, address); - if (!pud || !pud_present(*pud)) - return 0; - pmd = pmd_offset(pud, address); - if (!pmd || !pmd_present(*pmd)) - return 0; - pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) : - pte_offset_kernel(pmd, address); - pteval = *pte; - if (pte_migrating(pteval)) { - if (in_nmi() && search_exception_tables(pc)) - return 0; - wait_for_migration(pte); - return 1; - } - - if (!is_kernel_mode || !pte_present(pteval)) - return 0; - if (fault_num == INT_ITLB_MISS) { - if (pte_exec(pteval)) - return 1; - } else if (write) { - if (pte_write(pteval)) - return 1; - } else { - if (pte_read(pteval)) - return 1; - } - - return 0; -} - -/* - * This routine is responsible for faulting in user pages. - * It passes the work off to one of the appropriate routines. - * It returns true if the fault was successfully handled. - */ -static int handle_page_fault(struct pt_regs *regs, - int fault_num, - int is_page_fault, - unsigned long address, - int write) -{ - struct task_struct *tsk; - struct mm_struct *mm; - struct vm_area_struct *vma; - unsigned long stack_offset; - int fault; - int si_code; - int is_kernel_mode; - pgd_t *pgd; - unsigned int flags; - - /* on TILE, protection faults are always writes */ - if (!is_page_fault) - write = 1; - - flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; - - is_kernel_mode = !user_mode(regs); - - tsk = validate_current(); - - /* - * Check to see if we might be overwriting the stack, and bail - * out if so. The page fault code is a relatively likely - * place to get trapped in an infinite regress, and once we - * overwrite the whole stack, it becomes very hard to recover. - */ - stack_offset = stack_pointer & (THREAD_SIZE-1); - if (stack_offset < THREAD_SIZE / 8) { - pr_alert("Potential stack overrun: sp %#lx\n", stack_pointer); - show_regs(regs); - pr_alert("Killing current process %d/%s\n", - tsk->pid, tsk->comm); - do_group_exit(SIGKILL); - } - - /* - * Early on, we need to check for migrating PTE entries; - * see homecache.c. If we find a migrating PTE, we wait until - * the backing page claims to be done migrating, then we proceed. - * For kernel PTEs, we rewrite the PTE and return and retry. - * Otherwise, we treat the fault like a normal "no PTE" fault, - * rather than trying to patch up the existing PTE. - */ - pgd = get_current_pgd(); - if (handle_migrating_pte(pgd, fault_num, address, regs->pc, - is_kernel_mode, write)) - return 1; - - si_code = SEGV_MAPERR; - - /* - * We fault-in kernel-space virtual memory on-demand. The - * 'reference' page table is init_mm.pgd. - * - * NOTE! We MUST NOT take any locks for this case. We may - * be in an interrupt or a critical region, and should - * only copy the information from the master page table, - * nothing more. - * - * This verifies that the fault happens in kernel space - * and that the fault was not a protection fault. - */ - if (unlikely(address >= TASK_SIZE && - !is_arch_mappable_range(address, 0))) { - if (is_kernel_mode && is_page_fault && - vmalloc_fault(pgd, address) >= 0) - return 1; - /* - * Don't take the mm semaphore here. If we fixup a prefetch - * fault we could otherwise deadlock. - */ - mm = NULL; /* happy compiler */ - vma = NULL; - goto bad_area_nosemaphore; - } - - /* - * If we're trying to touch user-space addresses, we must - * be either at PL0, or else with interrupts enabled in the - * kernel, so either way we can re-enable interrupts here - * unless we are doing atomic access to user space with - * interrupts disabled. - */ - if (!(regs->flags & PT_FLAGS_DISABLE_IRQ)) - local_irq_enable(); - - mm = tsk->mm; - - /* - * If we're in an interrupt, have no user context or are running in an - * region with pagefaults disabled then we must not take the fault. - */ - if (pagefault_disabled() || !mm) { - vma = NULL; /* happy compiler */ - goto bad_area_nosemaphore; - } - - if (!is_kernel_mode) - flags |= FAULT_FLAG_USER; - - /* - * When running in the kernel we expect faults to occur only to - * addresses in user space. All other faults represent errors in the - * kernel and should generate an OOPS. Unfortunately, in the case of an - * erroneous fault occurring in a code path which already holds mmap_sem - * we will deadlock attempting to validate the fault against the - * address space. Luckily the kernel only validly references user - * space from well defined areas of code, which are listed in the - * exceptions table. - * - * As the vast majority of faults will be valid we will only perform - * the source reference check when there is a possibility of a deadlock. - * Attempt to lock the address space, if we cannot we then validate the - * source. If this is invalid we can skip the address space check, - * thus avoiding the deadlock. - */ - if (!down_read_trylock(&mm->mmap_sem)) { - if (is_kernel_mode && - !search_exception_tables(regs->pc)) { - vma = NULL; /* happy compiler */ - goto bad_area_nosemaphore; - } - -retry: - down_read(&mm->mmap_sem); - } - - vma = find_vma(mm, address); - if (!vma) - goto bad_area; - if (vma->vm_start <= address) - goto good_area; - if (!(vma->vm_flags & VM_GROWSDOWN)) - goto bad_area; - if (regs->sp < PAGE_OFFSET) { - /* - * accessing the stack below sp is always a bug. - */ - if (address < regs->sp) - goto bad_area; - } - if (expand_stack(vma, address)) - goto bad_area; - -/* - * Ok, we have a good vm_area for this memory access, so - * we can handle it.. - */ -good_area: - si_code = SEGV_ACCERR; - if (fault_num == INT_ITLB_MISS) { - if (!(vma->vm_flags & VM_EXEC)) - goto bad_area; - } else if (write) { -#ifdef TEST_VERIFY_AREA - if (!is_page_fault && regs->cs == KERNEL_CS) - pr_err("WP fault at " REGFMT "\n", regs->eip); -#endif - if (!(vma->vm_flags & VM_WRITE)) - goto bad_area; - flags |= FAULT_FLAG_WRITE; - } else { - if (!is_page_fault || !(vma->vm_flags & VM_READ)) - goto bad_area; - } - - /* - * If for any reason at all we couldn't handle the fault, - * make sure we exit gracefully rather than endlessly redo - * the fault. - */ - fault = handle_mm_fault(vma, address, flags); - - if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) - return 0; - - if (unlikely(fault & VM_FAULT_ERROR)) { - if (fault & VM_FAULT_OOM) - goto out_of_memory; - else if (fault & VM_FAULT_SIGSEGV) - goto bad_area; - else if (fault & VM_FAULT_SIGBUS) - goto do_sigbus; - BUG(); - } - if (flags & FAULT_FLAG_ALLOW_RETRY) { - if (fault & VM_FAULT_MAJOR) - tsk->maj_flt++; - else - tsk->min_flt++; - if (fault & VM_FAULT_RETRY) { - flags &= ~FAULT_FLAG_ALLOW_RETRY; - flags |= FAULT_FLAG_TRIED; - - /* - * No need to up_read(&mm->mmap_sem) as we would - * have already released it in __lock_page_or_retry - * in mm/filemap.c. - */ - goto retry; - } - } - -#if CHIP_HAS_TILE_DMA() - /* If this was a DMA TLB fault, restart the DMA engine. */ - switch (fault_num) { - case INT_DMATLB_MISS: - case INT_DMATLB_MISS_DWNCL: - case INT_DMATLB_ACCESS: - case INT_DMATLB_ACCESS_DWNCL: - __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK); - break; - } -#endif - - up_read(&mm->mmap_sem); - return 1; - -/* - * Something tried to access memory that isn't in our memory map.. - * Fix it, but check if it's kernel or user first.. - */ -bad_area: - up_read(&mm->mmap_sem); - -bad_area_nosemaphore: - /* User mode accesses just cause a SIGSEGV */ - if (!is_kernel_mode) { - /* - * It's possible to have interrupts off here. - */ - local_irq_enable(); - - force_sig_info_fault("segfault", SIGSEGV, si_code, address, - fault_num, tsk, regs); - return 0; - } - -no_context: - /* Are we prepared to handle this kernel fault? */ - if (fixup_exception(regs)) - return 0; - -/* - * Oops. The kernel tried to access some bad page. We'll have to - * terminate things with extreme prejudice. - */ - - bust_spinlocks(1); - - /* FIXME: no lookup_address() yet */ -#ifdef SUPPORT_LOOKUP_ADDRESS - if (fault_num == INT_ITLB_MISS) { - pte_t *pte = lookup_address(address); - - if (pte && pte_present(*pte) && !pte_exec_kernel(*pte)) - pr_crit("kernel tried to execute non-executable page - exploit attempt? (uid: %d)\n", - current->uid); - } -#endif - if (address < PAGE_SIZE) - pr_alert("Unable to handle kernel NULL pointer dereference\n"); - else - pr_alert("Unable to handle kernel paging request\n"); - pr_alert(" at virtual address " REGFMT ", pc " REGFMT "\n", - address, regs->pc); - - show_regs(regs); - - if (unlikely(tsk->pid < 2)) { - panic("Kernel page fault running %s!", - is_idle_task(tsk) ? "the idle task" : "init"); - } - - /* - * More FIXME: we should probably copy the i386 here and - * implement a generic die() routine. Not today. - */ -#ifdef SUPPORT_DIE - die("Oops", regs); -#endif - bust_spinlocks(1); - - do_group_exit(SIGKILL); - -/* - * We ran out of memory, or some other thing happened to us that made - * us unable to handle the page fault gracefully. - */ -out_of_memory: - up_read(&mm->mmap_sem); - if (is_kernel_mode) - goto no_context; - pagefault_out_of_memory(); - return 0; - -do_sigbus: - up_read(&mm->mmap_sem); - - /* Kernel mode? Handle exceptions or die */ - if (is_kernel_mode) - goto no_context; - - force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address, - fault_num, tsk, regs); - return 0; -} - -#ifndef __tilegx__ - -/* We must release ICS before panicking or we won't get anywhere. */ -#define ics_panic(fmt, ...) \ -do { \ - __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); \ - panic(fmt, ##__VA_ARGS__); \ -} while (0) - -/* - * When we take an ITLB or DTLB fault or access violation in the - * supervisor while the critical section bit is set, the hypervisor is - * reluctant to write new values into the EX_CONTEXT_K_x registers, - * since that might indicate we have not yet squirreled the SPR - * contents away and can thus safely take a recursive interrupt. - * Accordingly, the hypervisor passes us the PC via SYSTEM_SAVE_K_2. - * - * Note that this routine is called before homecache_tlb_defer_enter(), - * which means that we can properly unlock any atomics that might - * be used there (good), but also means we must be very sensitive - * to not touch any data structures that might be located in memory - * that could migrate, as we could be entering the kernel on a dataplane - * cpu that has been deferring kernel TLB updates. This means, for - * example, that we can't migrate init_mm or its pgd. - */ -struct intvec_state do_page_fault_ics(struct pt_regs *regs, int fault_num, - unsigned long address, - unsigned long info) -{ - unsigned long pc = info & ~1; - int write = info & 1; - pgd_t *pgd = get_current_pgd(); - - /* Retval is 1 at first since we will handle the fault fully. */ - struct intvec_state state = { - do_page_fault, fault_num, address, write, 1 - }; - - /* Validate that we are plausibly in the right routine. */ - if ((pc & 0x7) != 0 || pc < PAGE_OFFSET || - (fault_num != INT_DTLB_MISS && - fault_num != INT_DTLB_ACCESS)) { - unsigned long old_pc = regs->pc; - regs->pc = pc; - ics_panic("Bad ICS page fault args: old PC %#lx, fault %d/%d at %#lx", - old_pc, fault_num, write, address); - } - - /* We might be faulting on a vmalloc page, so check that first. */ - if (fault_num != INT_DTLB_ACCESS && vmalloc_fault(pgd, address) >= 0) - return state; - - /* - * If we faulted with ICS set in sys_cmpxchg, we are providing - * a user syscall service that should generate a signal on - * fault. We didn't set up a kernel stack on initial entry to - * sys_cmpxchg, but instead had one set up by the fault, which - * (because sys_cmpxchg never releases ICS) came to us via the - * SYSTEM_SAVE_K_2 mechanism, and thus EX_CONTEXT_K_[01] are - * still referencing the original user code. We release the - * atomic lock and rewrite pt_regs so that it appears that we - * came from user-space directly, and after we finish the - * fault we'll go back to user space and re-issue the swint. - * This way the backtrace information is correct if we need to - * emit a stack dump at any point while handling this. - * - * Must match register use in sys_cmpxchg(). - */ - if (pc >= (unsigned long) sys_cmpxchg && - pc < (unsigned long) __sys_cmpxchg_end) { -#ifdef CONFIG_SMP - /* Don't unlock before we could have locked. */ - if (pc >= (unsigned long)__sys_cmpxchg_grab_lock) { - int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]); - __atomic_fault_unlock(lock_ptr); - } -#endif - regs->sp = regs->regs[27]; - } - - /* - * We can also fault in the atomic assembly, in which - * case we use the exception table to do the first-level fixup. - * We may re-fixup again in the real fault handler if it - * turns out the faulting address is just bad, and not, - * for example, migrating. - */ - else if (pc >= (unsigned long) __start_atomic_asm_code && - pc < (unsigned long) __end_atomic_asm_code) { - const struct exception_table_entry *fixup; -#ifdef CONFIG_SMP - /* Unlock the atomic lock. */ - int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]); - __atomic_fault_unlock(lock_ptr); -#endif - fixup = search_exception_tables(pc); - if (!fixup) - ics_panic("ICS atomic fault not in table: PC %#lx, fault %d", - pc, fault_num); - regs->pc = fixup->fixup; - regs->ex1 = PL_ICS_EX1(KERNEL_PL, 0); - } - - /* - * Now that we have released the atomic lock (if necessary), - * it's safe to spin if the PTE that caused the fault was migrating. - */ - if (fault_num == INT_DTLB_ACCESS) - write = 1; - if (handle_migrating_pte(pgd, fault_num, address, pc, 1, write)) - return state; - - /* Return zero so that we continue on with normal fault handling. */ - state.retval = 0; - return state; -} - -#endif /* !__tilegx__ */ - -/* - * This routine handles page faults. It determines the address, and the - * problem, and then passes it handle_page_fault() for normal DTLB and - * ITLB issues, and for DMA or SN processor faults when we are in user - * space. For the latter, if we're in kernel mode, we just save the - * interrupt away appropriately and return immediately. We can't do - * page faults for user code while in kernel mode. - */ -static inline void __do_page_fault(struct pt_regs *regs, int fault_num, - unsigned long address, unsigned long write) -{ - int is_page_fault; - -#ifdef CONFIG_KPROBES - /* - * This is to notify the fault handler of the kprobes. The - * exception code is redundant as it is also carried in REGS, - * but we pass it anyhow. - */ - if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1, - regs->faultnum, SIGSEGV) == NOTIFY_STOP) - return; -#endif - -#ifdef __tilegx__ - /* - * We don't need early do_page_fault_ics() support, since unlike - * Pro we don't need to worry about unlocking the atomic locks. - * There is only one current case in GX where we touch any memory - * under ICS other than our own kernel stack, and we handle that - * here. (If we crash due to trying to touch our own stack, - * we're in too much trouble for C code to help out anyway.) - */ - if (write & ~1) { - unsigned long pc = write & ~1; - if (pc >= (unsigned long) __start_unalign_asm_code && - pc < (unsigned long) __end_unalign_asm_code) { - struct thread_info *ti = current_thread_info(); - /* - * Our EX_CONTEXT is still what it was from the - * initial unalign exception, but now we've faulted - * on the JIT page. We would like to complete the - * page fault however is appropriate, and then retry - * the instruction that caused the unalign exception. - * Our state has been "corrupted" by setting the low - * bit in "sp", and stashing r0..r3 in the - * thread_info area, so we revert all of that, then - * continue as if this were a normal page fault. - */ - regs->sp &= ~1UL; - regs->regs[0] = ti->unalign_jit_tmp[0]; - regs->regs[1] = ti->unalign_jit_tmp[1]; - regs->regs[2] = ti->unalign_jit_tmp[2]; - regs->regs[3] = ti->unalign_jit_tmp[3]; - write &= 1; - } else { - pr_alert("%s/%d: ICS set at page fault at %#lx: %#lx\n", - current->comm, current->pid, pc, address); - show_regs(regs); - do_group_exit(SIGKILL); - } - } -#else - /* This case should have been handled by do_page_fault_ics(). */ - BUG_ON(write & ~1); -#endif - -#if CHIP_HAS_TILE_DMA() - /* - * If it's a DMA fault, suspend the transfer while we're - * handling the miss; we'll restart after it's handled. If we - * don't suspend, it's possible that this process could swap - * out and back in, and restart the engine since the DMA is - * still 'running'. - */ - if (fault_num == INT_DMATLB_MISS || - fault_num == INT_DMATLB_ACCESS || - fault_num == INT_DMATLB_MISS_DWNCL || - fault_num == INT_DMATLB_ACCESS_DWNCL) { - __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK); - while (__insn_mfspr(SPR_DMA_USER_STATUS) & - SPR_DMA_STATUS__BUSY_MASK) - ; - } -#endif - - /* Validate fault num and decide if this is a first-time page fault. */ - switch (fault_num) { - case INT_ITLB_MISS: - case INT_DTLB_MISS: -#if CHIP_HAS_TILE_DMA() - case INT_DMATLB_MISS: - case INT_DMATLB_MISS_DWNCL: -#endif - is_page_fault = 1; - break; - - case INT_DTLB_ACCESS: -#if CHIP_HAS_TILE_DMA() - case INT_DMATLB_ACCESS: - case INT_DMATLB_ACCESS_DWNCL: -#endif - is_page_fault = 0; - break; - - default: - panic("Bad fault number %d in do_page_fault", fault_num); - } - -#if CHIP_HAS_TILE_DMA() - if (!user_mode(regs)) { - struct async_tlb *async; - switch (fault_num) { -#if CHIP_HAS_TILE_DMA() - case INT_DMATLB_MISS: - case INT_DMATLB_ACCESS: - case INT_DMATLB_MISS_DWNCL: - case INT_DMATLB_ACCESS_DWNCL: - async = ¤t->thread.dma_async_tlb; - break; -#endif - default: - async = NULL; - } - if (async) { - - /* - * No vmalloc check required, so we can allow - * interrupts immediately at this point. - */ - local_irq_enable(); - - set_thread_flag(TIF_ASYNC_TLB); - if (async->fault_num != 0) { - panic("Second async fault %d; old fault was %d (%#lx/%ld)", - fault_num, async->fault_num, - address, write); - } - BUG_ON(fault_num == 0); - async->fault_num = fault_num; - async->is_fault = is_page_fault; - async->is_write = write; - async->address = address; - return; - } - } -#endif - - handle_page_fault(regs, fault_num, is_page_fault, address, write); -} - -void do_page_fault(struct pt_regs *regs, int fault_num, - unsigned long address, unsigned long write) -{ - __do_page_fault(regs, fault_num, address, write); -} - -#if CHIP_HAS_TILE_DMA() -/* - * This routine effectively re-issues asynchronous page faults - * when we are returning to user space. - */ -void do_async_page_fault(struct pt_regs *regs) -{ - struct async_tlb *async = ¤t->thread.dma_async_tlb; - - /* - * Clear thread flag early. If we re-interrupt while processing - * code here, we will reset it and recall this routine before - * returning to user space. - */ - clear_thread_flag(TIF_ASYNC_TLB); - - if (async->fault_num) { - /* - * Clear async->fault_num before calling the page-fault - * handler so that if we re-interrupt before returning - * from the function we have somewhere to put the - * information from the new interrupt. - */ - int fault_num = async->fault_num; - async->fault_num = 0; - handle_page_fault(regs, fault_num, async->is_fault, - async->address, async->is_write); - } -} -#endif /* CHIP_HAS_TILE_DMA() */ - - -void vmalloc_sync_all(void) -{ -#ifdef __tilegx__ - /* Currently all L1 kernel pmd's are static and shared. */ - BUILD_BUG_ON(pgd_index(VMALLOC_END - PAGE_SIZE) != - pgd_index(VMALLOC_START)); -#else - /* - * Note that races in the updates of insync and start aren't - * problematic: insync can only get set bits added, and updates to - * start are only improving performance (without affecting correctness - * if undone). - */ - static DECLARE_BITMAP(insync, PTRS_PER_PGD); - static unsigned long start = PAGE_OFFSET; - unsigned long address; - - BUILD_BUG_ON(PAGE_OFFSET & ~PGDIR_MASK); - for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) { - if (!test_bit(pgd_index(address), insync)) { - unsigned long flags; - struct list_head *pos; - - spin_lock_irqsave(&pgd_lock, flags); - list_for_each(pos, &pgd_list) - if (!vmalloc_sync_one(list_to_pgd(pos), - address)) { - /* Must be at first entry in list. */ - BUG_ON(pos != pgd_list.next); - break; - } - spin_unlock_irqrestore(&pgd_lock, flags); - if (pos != pgd_list.next) - set_bit(pgd_index(address), insync); - } - if (address == start && test_bit(pgd_index(address), insync)) - start = address + PGDIR_SIZE; - } -#endif -} |