bianbu-repo-mirror/bianbu-linux-6.6
1
0
Fork 0
mirror of https://gitee.com/bianbu-linux/linux-6.6 synced 2025-07-06 00:13:38 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

ARM: redo TTBR setup code for LPAE

Browse source 
Re-engineer the LPAE TTBR setup code.  Rather than passing some shifted
address in order to fit in a CPU register, pass either a full physical
address (in the case of r4, r5 for TTBR0) or a PFN (for TTBR1).

This removes the ARCH_PGD_SHIFT hack, and the last dangerous user of
cpu_set_ttbr() in the secondary CPU startup code path (which was there
to re-set TTBR1 to the appropriate high physical address space on
Keystone2.)

Tested-by: Murali Karicheri <m-karicheri2@ti.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
This commit is contained in:
Russell King 2015-04-04 20:09:46 +01:00
parent 1221ed10f2
commit b2c3e38a54
10 changed files with 60 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

16
arch/arm/include/asm/memory.h
View file

@ -18,8 +18,6 @@
#include <linux/types.h> #include <linux/types.h>
#include <linux/sizes.h> #include <linux/sizes.h>
#include <asm/cache.h>
#ifdef CONFIG_NEED_MACH_MEMORY_H #ifdef CONFIG_NEED_MACH_MEMORY_H
#include <mach/memory.h> #include <mach/memory.h>
#endif #endif
@ -132,20 +130,6 @@
#define page_to_phys(page) (__pfn_to_phys(page_to_pfn(page))) #define page_to_phys(page) (__pfn_to_phys(page_to_pfn(page)))
#define phys_to_page(phys) (pfn_to_page(__phys_to_pfn(phys))) #define phys_to_page(phys) (pfn_to_page(__phys_to_pfn(phys)))
/*
* Minimum guaranted alignment in pgd_alloc(). The page table pointers passed
* around in head.S and proc-*.S are shifted by this amount, in order to
* leave spare high bits for systems with physical address extension. This
* does not fully accomodate the 40-bit addressing capability of ARM LPAE, but
* gives us about 38-bits or so.
*/
#ifdef CONFIG_ARM_LPAE
#define ARCH_PGD_SHIFT L1_CACHE_SHIFT
#else
#define ARCH_PGD_SHIFT 0
#endif
#define ARCH_PGD_MASK ((1 << ARCH_PGD_SHIFT) - 1)
/* /*
* PLAT_PHYS_OFFSET is the offset (from zero) of the start of physical * PLAT_PHYS_OFFSET is the offset (from zero) of the start of physical
* memory. This is used for XIP and NoMMU kernels, and on platforms that don't * memory. This is used for XIP and NoMMU kernels, and on platforms that don't

7
arch/arm/include/asm/proc-fns.h
View file

@ -125,13 +125,6 @@ extern void cpu_resume(void);
ttbr; \ ttbr; \
}) })
#define cpu_set_ttbr(nr, val) \
do { \
u64 ttbr = val; \
__asm__("mcrr p15, " #nr ", %Q0, %R0, c2" \
: : "r" (ttbr)); \
} while (0)
#define cpu_get_pgd() \ #define cpu_get_pgd() \
({ \ ({ \
u64 pg = cpu_get_ttbr(0); \ u64 pg = cpu_get_ttbr(0); \

2
arch/arm/include/asm/smp.h
View file

@ -61,7 +61,7 @@ asmlinkage void secondary_start_kernel(void);
struct secondary_data { struct secondary_data {
union { union {
unsigned long mpu_rgn_szr; unsigned long mpu_rgn_szr;
unsigned long pgdir; u64 pgdir;
}; };
unsigned long swapper_pg_dir; unsigned long swapper_pg_dir;
void *stack; void *stack;

2
arch/arm/kernel/head-nommu.S
View file

@ -123,7 +123,7 @@ ENTRY(secondary_startup)
ENDPROC(secondary_startup) ENDPROC(secondary_startup)
ENTRY(__secondary_switched) ENTRY(__secondary_switched)
ldr sp, [r7, #8] @ set up the stack pointer ldr sp, [r7, #12] @ set up the stack pointer
mov fp, #0 mov fp, #0
b secondary_start_kernel b secondary_start_kernel
ENDPROC(__secondary_switched) ENDPROC(__secondary_switched)

42
arch/arm/kernel/head.S
View file

@ -131,13 +131,30 @@ ENTRY(stext)
* The following calls CPU specific code in a position independent * The following calls CPU specific code in a position independent
* manner. See arch/arm/mm/proc-*.S for details. r10 = base of * manner. See arch/arm/mm/proc-*.S for details. r10 = base of
* xxx_proc_info structure selected by __lookup_processor_type * xxx_proc_info structure selected by __lookup_processor_type
* above. On return, the CPU will be ready for the MMU to be * above.
* turned on, and r0 will hold the CPU control register value. *
* The processor init function will be called with:
* r1 - machine type
* r2 - boot data (atags/dt) pointer
* r4 - translation table base (low word)
* r5 - translation table base (high word, if LPAE)
* r8 - translation table base 1 (pfn if LPAE)
* r9 - cpuid
* r13 - virtual address for __enable_mmu -> __turn_mmu_on
*
* On return, the CPU will be ready for the MMU to be turned on,
* r0 will hold the CPU control register value, r1, r2, r4, and
* r9 will be preserved. r5 will also be preserved if LPAE.
*/ */
ldr r13, =__mmap_switched @ address to jump to after ldr r13, =__mmap_switched @ address to jump to after
@ mmu has been enabled @ mmu has been enabled
adr lr, BSYM(1f) @ return (PIC) address adr lr, BSYM(1f) @ return (PIC) address
#ifdef CONFIG_ARM_LPAE
mov r5, #0 @ high TTBR0
mov r8, r4, lsr #12 @ TTBR1 is swapper_pg_dir pfn
#else
mov r8, r4 @ set TTBR1 to swapper_pg_dir mov r8, r4 @ set TTBR1 to swapper_pg_dir
#endif
ldr r12, [r10, #PROCINFO_INITFUNC] ldr r12, [r10, #PROCINFO_INITFUNC]
add r12, r12, r10 add r12, r12, r10
ret r12 ret r12
@ -158,7 +175,7 @@ ENDPROC(stext)
* *
* Returns: * Returns:
* r0, r3, r5-r7 corrupted * r0, r3, r5-r7 corrupted
* r4 = page table (see ARCH_PGD_SHIFT in asm/memory.h) * r4 = physical page table address
*/ */
__create_page_tables: __create_page_tables:
pgtbl r4, r8 @ page table address pgtbl r4, r8 @ page table address
@ -333,7 +350,6 @@ __create_page_tables:
#endif #endif
#ifdef CONFIG_ARM_LPAE #ifdef CONFIG_ARM_LPAE
sub r4, r4, #0x1000 @ point to the PGD table sub r4, r4, #0x1000 @ point to the PGD table
mov r4, r4, lsr #ARCH_PGD_SHIFT
#endif #endif
ret lr ret lr
ENDPROC(__create_page_tables) ENDPROC(__create_page_tables)
@ -381,9 +397,9 @@ ENTRY(secondary_startup)
adr r4, __secondary_data adr r4, __secondary_data
ldmia r4, {r5, r7, r12} @ address to jump to after ldmia r4, {r5, r7, r12} @ address to jump to after
sub lr, r4, r5 @ mmu has been enabled sub lr, r4, r5 @ mmu has been enabled
ldr r4, [r7, lr] @ get secondary_data.pgdir add r3, r7, lr
add r7, r7, #4 ldrd r4, [r3, #0] @ get secondary_data.pgdir
ldr r8, [r7, lr] @ get secondary_data.swapper_pg_dir ldr r8, [r3, #8] @ get secondary_data.swapper_pg_dir
adr lr, BSYM(__enable_mmu) @ return address adr lr, BSYM(__enable_mmu) @ return address
mov r13, r12 @ __secondary_switched address mov r13, r12 @ __secondary_switched address
ldr r12, [r10, #PROCINFO_INITFUNC] ldr r12, [r10, #PROCINFO_INITFUNC]
@ -397,7 +413,7 @@ ENDPROC(secondary_startup_arm)
* r6 = &secondary_data * r6 = &secondary_data
*/ */
ENTRY(__secondary_switched) ENTRY(__secondary_switched)
ldr sp, [r7, #4] @ get secondary_data.stack ldr sp, [r7, #12] @ get secondary_data.stack
mov fp, #0 mov fp, #0
b secondary_start_kernel b secondary_start_kernel
ENDPROC(__secondary_switched) ENDPROC(__secondary_switched)
@ -416,12 +432,14 @@ __secondary_data:
/* /*
* Setup common bits before finally enabling the MMU. Essentially * Setup common bits before finally enabling the MMU. Essentially
* this is just loading the page table pointer and domain access * this is just loading the page table pointer and domain access
* registers. * registers. All these registers need to be preserved by the
* processor setup function (or set in the case of r0)
* *
* r0 = cp#15 control register * r0 = cp#15 control register
* r1 = machine ID * r1 = machine ID
* r2 = atags or dtb pointer * r2 = atags or dtb pointer
* r4 = page table (see ARCH_PGD_SHIFT in asm/memory.h) * r4 = TTBR pointer (low word)
* r5 = TTBR pointer (high word if LPAE)
* r9 = processor ID * r9 = processor ID
* r13 = *virtual* address to jump to upon completion * r13 = *virtual* address to jump to upon completion
*/ */
@ -440,7 +458,9 @@ __enable_mmu:
#ifdef CONFIG_CPU_ICACHE_DISABLE #ifdef CONFIG_CPU_ICACHE_DISABLE
bic r0, r0, #CR_I bic r0, r0, #CR_I
#endif #endif
#ifndef CONFIG_ARM_LPAE #ifdef CONFIG_ARM_LPAE
mcrr p15, 0, r4, r5, c2 @ load TTBR0
#else
mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \
domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \
domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \

10
arch/arm/kernel/smp.c
View file

@ -86,9 +86,11 @@ void __init smp_set_ops(struct smp_operations *ops)
static unsigned long get_arch_pgd(pgd_t *pgd) static unsigned long get_arch_pgd(pgd_t *pgd)
{ {
phys_addr_t pgdir = virt_to_idmap(pgd); #ifdef CONFIG_ARM_LPAE
BUG_ON(pgdir & ARCH_PGD_MASK); return __phys_to_pfn(virt_to_phys(pgd));
return pgdir >> ARCH_PGD_SHIFT; #else
return virt_to_phys(pgd);
#endif
} }
int __cpu_up(unsigned int cpu, struct task_struct *idle) int __cpu_up(unsigned int cpu, struct task_struct *idle)
@ -108,7 +110,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
#endif #endif
#ifdef CONFIG_MMU #ifdef CONFIG_MMU
secondary_data.pgdir = get_arch_pgd(idmap_pgd); secondary_data.pgdir = virt_to_phys(idmap_pgd);
secondary_data.swapper_pg_dir = get_arch_pgd(swapper_pg_dir); secondary_data.swapper_pg_dir = get_arch_pgd(swapper_pg_dir);
#endif #endif
sync_cache_w(&secondary_data); sync_cache_w(&secondary_data);

13
arch/arm/mach-keystone/platsmp.c
View file

@ -39,19 +39,6 @@ static int keystone_smp_boot_secondary(unsigned int cpu,
return error; return error;
} }
#ifdef CONFIG_ARM_LPAE
static void __cpuinit keystone_smp_secondary_initmem(unsigned int cpu)
{
pgd_t *pgd0 = pgd_offset_k(0);
cpu_set_ttbr(1, __pa(pgd0) + TTBR1_OFFSET);
local_flush_tlb_all();
}
#else
static inline void __cpuinit keystone_smp_secondary_initmem(unsigned int cpu)
{}
#endif
struct smp_operations keystone_smp_ops __initdata = { struct smp_operations keystone_smp_ops __initdata = {
.smp_boot_secondary = keystone_smp_boot_secondary, .smp_boot_secondary = keystone_smp_boot_secondary,
.smp_secondary_init = keystone_smp_secondary_initmem,
}; };

6
arch/arm/mm/proc-v7-2level.S
View file

@ -148,10 +148,10 @@ ENDPROC(cpu_v7_set_pte_ext)
* Macro for setting up the TTBRx and TTBCR registers. * Macro for setting up the TTBRx and TTBCR registers.
* - \ttb0 and \ttb1 updated with the corresponding flags. * - \ttb0 and \ttb1 updated with the corresponding flags.
*/ */
.macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp .macro v7_ttb_setup, zero, ttbr0l, ttbr0h, ttbr1, tmp
mcr p15, 0, \zero, c2, c0, 2 @ TTB control register mcr p15, 0, \zero, c2, c0, 2 @ TTB control register
ALT_SMP(orr \ttbr0, \ttbr0, #TTB_FLAGS_SMP) ALT_SMP(orr \ttbr0l, \ttbr0l, #TTB_FLAGS_SMP)
ALT_UP(orr \ttbr0, \ttbr0, #TTB_FLAGS_UP) ALT_UP(orr \ttbr0l, \ttbr0l, #TTB_FLAGS_UP)
ALT_SMP(orr \ttbr1, \ttbr1, #TTB_FLAGS_SMP) ALT_SMP(orr \ttbr1, \ttbr1, #TTB_FLAGS_SMP)
ALT_UP(orr \ttbr1, \ttbr1, #TTB_FLAGS_UP) ALT_UP(orr \ttbr1, \ttbr1, #TTB_FLAGS_UP)
mcr p15, 0, \ttbr1, c2, c0, 1 @ load TTB1 mcr p15, 0, \ttbr1, c2, c0, 1 @ load TTB1

14
arch/arm/mm/proc-v7-3level.S
View file

@ -126,11 +126,10 @@ ENDPROC(cpu_v7_set_pte_ext)
* Macro for setting up the TTBRx and TTBCR registers. * Macro for setting up the TTBRx and TTBCR registers.
* - \ttbr1 updated. * - \ttbr1 updated.
*/ */
.macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp .macro v7_ttb_setup, zero, ttbr0l, ttbr0h, ttbr1, tmp
ldr \tmp, =swapper_pg_dir @ swapper_pg_dir virtual address ldr \tmp, =swapper_pg_dir @ swapper_pg_dir virtual address
mov \tmp, \tmp, lsr #ARCH_PGD_SHIFT cmp \ttbr1, \tmp, lsr #12 @ PHYS_OFFSET > PAGE_OFFSET?
cmp \ttbr1, \tmp @ PHYS_OFFSET > PAGE_OFFSET? mrc p15, 0, \tmp, c2, c0, 2 @ TTB control egister
mrc p15, 0, \tmp, c2, c0, 2 @ TTB control register
orr \tmp, \tmp, #TTB_EAE orr \tmp, \tmp, #TTB_EAE
ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP) ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP)
ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP) ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP)
@ -143,13 +142,10 @@ ENDPROC(cpu_v7_set_pte_ext)
*/ */
orrls \tmp, \tmp, #TTBR1_SIZE @ TTBCR.T1SZ orrls \tmp, \tmp, #TTBR1_SIZE @ TTBCR.T1SZ
mcr p15, 0, \tmp, c2, c0, 2 @ TTBCR mcr p15, 0, \tmp, c2, c0, 2 @ TTBCR
mov \tmp, \ttbr1, lsr #(32 - ARCH_PGD_SHIFT) @ upper bits mov \tmp, \ttbr1, lsr #20
mov \ttbr1, \ttbr1, lsl #ARCH_PGD_SHIFT @ lower bits mov \ttbr1, \ttbr1, lsl #12
addls \ttbr1, \ttbr1, #TTBR1_OFFSET addls \ttbr1, \ttbr1, #TTBR1_OFFSET
mcrr p15, 1, \ttbr1, \tmp, c2 @ load TTBR1 mcrr p15, 1, \ttbr1, \tmp, c2 @ load TTBR1
mov \tmp, \ttbr0, lsr #(32 - ARCH_PGD_SHIFT) @ upper bits
mov \ttbr0, \ttbr0, lsl #ARCH_PGD_SHIFT @ lower bits
mcrr p15, 0, \ttbr0, \tmp, c2 @ load TTBR0
.endm .endm
/* /*

26
arch/arm/mm/proc-v7.S
View file

@ -343,9 +343,9 @@ __v7_setup:
and r10, r0, #0xff000000 @ ARM? and r10, r0, #0xff000000 @ ARM?
teq r10, #0x41000000 teq r10, #0x41000000
bne 3f bne 3f
and r5, r0, #0x00f00000 @ variant and r3, r0, #0x00f00000 @ variant
and r6, r0, #0x0000000f @ revision and r6, r0, #0x0000000f @ revision
orr r6, r6, r5, lsr #20-4 @ combine variant and revision orr r6, r6, r3, lsr #20-4 @ combine variant and revision
ubfx r0, r0, #4, #12 @ primary part number ubfx r0, r0, #4, #12 @ primary part number
/* Cortex-A8 Errata */ /* Cortex-A8 Errata */
@ -354,7 +354,7 @@ __v7_setup:
bne 2f bne 2f
#if defined(CONFIG_ARM_ERRATA_430973) && !defined(CONFIG_ARCH_MULTIPLATFORM) #if defined(CONFIG_ARM_ERRATA_430973) && !defined(CONFIG_ARCH_MULTIPLATFORM)
teq r5, #0x00100000 @ only present in r1p* teq r3, #0x00100000 @ only present in r1p*
mrceq p15, 0, r10, c1, c0, 1 @ read aux control register mrceq p15, 0, r10, c1, c0, 1 @ read aux control register
orreq r10, r10, #(1 << 6) @ set IBE to 1 orreq r10, r10, #(1 << 6) @ set IBE to 1
mcreq p15, 0, r10, c1, c0, 1 @ write aux control register mcreq p15, 0, r10, c1, c0, 1 @ write aux control register
@ -395,7 +395,7 @@ __v7_setup:
mcreq p15, 0, r10, c15, c0, 1 @ write diagnostic register mcreq p15, 0, r10, c15, c0, 1 @ write diagnostic register
#endif #endif
#ifdef CONFIG_ARM_ERRATA_743622 #ifdef CONFIG_ARM_ERRATA_743622
teq r5, #0x00200000 @ only present in r2p* teq r3, #0x00200000 @ only present in r2p*
mrceq p15, 0, r10, c15, c0, 1 @ read diagnostic register mrceq p15, 0, r10, c15, c0, 1 @ read diagnostic register
orreq r10, r10, #1 << 6 @ set bit #6 orreq r10, r10, #1 << 6 @ set bit #6
mcreq p15, 0, r10, c15, c0, 1 @ write diagnostic register mcreq p15, 0, r10, c15, c0, 1 @ write diagnostic register
@ -425,10 +425,10 @@ __v7_setup:
mcr p15, 0, r10, c7, c5, 0 @ I+BTB cache invalidate mcr p15, 0, r10, c7, c5, 0 @ I+BTB cache invalidate
#ifdef CONFIG_MMU #ifdef CONFIG_MMU
mcr p15, 0, r10, c8, c7, 0 @ invalidate I + D TLBs mcr p15, 0, r10, c8, c7, 0 @ invalidate I + D TLBs
v7_ttb_setup r10, r4, r8, r5 @ TTBCR, TTBRx setup v7_ttb_setup r10, r4, r5, r8, r3 @ TTBCR, TTBRx setup
ldr r5, =PRRR @ PRRR ldr r3, =PRRR @ PRRR
ldr r6, =NMRR @ NMRR ldr r6, =NMRR @ NMRR
mcr p15, 0, r5, c10, c2, 0 @ write PRRR mcr p15, 0, r3, c10, c2, 0 @ write PRRR
mcr p15, 0, r6, c10, c2, 1 @ write NMRR mcr p15, 0, r6, c10, c2, 1 @ write NMRR
#endif #endif
dsb @ Complete invalidations dsb @ Complete invalidations
@ -437,22 +437,22 @@ __v7_setup:
and r0, r0, #(0xf << 12) @ ThumbEE enabled field and r0, r0, #(0xf << 12) @ ThumbEE enabled field
teq r0, #(1 << 12) @ check if ThumbEE is present teq r0, #(1 << 12) @ check if ThumbEE is present
bne 1f bne 1f
mov r5, #0 mov r3, #0
mcr p14, 6, r5, c1, c0, 0 @ Initialize TEEHBR to 0 mcr p14, 6, r3, c1, c0, 0 @ Initialize TEEHBR to 0
mrc p14, 6, r0, c0, c0, 0 @ load TEECR mrc p14, 6, r0, c0, c0, 0 @ load TEECR
orr r0, r0, #1 @ set the 1st bit in order to orr r0, r0, #1 @ set the 1st bit in order to
mcr p14, 6, r0, c0, c0, 0 @ stop userspace TEEHBR access mcr p14, 6, r0, c0, c0, 0 @ stop userspace TEEHBR access
1: 1:
#endif #endif
adr r5, v7_crval adr r3, v7_crval
ldmia r5, {r5, r6} ldmia r3, {r3, r6}
ARM_BE8(orr r6, r6, #1 << 25) @ big-endian page tables ARM_BE8(orr r6, r6, #1 << 25) @ big-endian page tables
#ifdef CONFIG_SWP_EMULATE #ifdef CONFIG_SWP_EMULATE
orr r5, r5, #(1 << 10) @ set SW bit in "clear" orr r3, r3, #(1 << 10) @ set SW bit in "clear"
bic r6, r6, #(1 << 10) @ clear it in "mmuset" bic r6, r6, #(1 << 10) @ clear it in "mmuset"
#endif #endif
mrc p15, 0, r0, c1, c0, 0 @ read control register mrc p15, 0, r0, c1, c0, 0 @ read control register
bic r0, r0, r5 @ clear bits them bic r0, r0, r3 @ clear bits them
orr r0, r0, r6 @ set them orr r0, r0, r6 @ set them
THUMB( orr r0, r0, #1 << 30 ) @ Thumb exceptions THUMB( orr r0, r0, #1 << 30 ) @ Thumb exceptions
ret lr @ return to head.S:__ret ret lr @ return to head.S:__ret