From: jbeulich@novell.com
Subject: properly predict phys<->mach translations
Patch-mainline: obsolete
--- head-2009-07-28.orig/arch/x86/include/mach-xen/asm/maddr_32.h 2009-07-28 12:14:16.000000000 +0200
+++ head-2009-07-28/arch/x86/include/mach-xen/asm/maddr_32.h 2009-07-29 10:56:35.000000000 +0200
@@ -30,17 +30,19 @@ extern unsigned int machine_to_phys_or
static inline unsigned long pfn_to_mfn(unsigned long pfn)
{
- if (xen_feature(XENFEAT_auto_translated_physmap))
+ if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
return pfn;
- BUG_ON(max_mapnr && pfn >= max_mapnr);
+ if (likely(max_mapnr))
+ BUG_ON(pfn >= max_mapnr);
return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT;
}
static inline int phys_to_machine_mapping_valid(unsigned long pfn)
{
- if (xen_feature(XENFEAT_auto_translated_physmap))
+ if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
return 1;
- BUG_ON(max_mapnr && pfn >= max_mapnr);
+ if (likely(max_mapnr))
+ BUG_ON(pfn >= max_mapnr);
return (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY);
}
@@ -48,7 +50,7 @@ static inline unsigned long mfn_to_pfn(u
{
unsigned long pfn;
- if (xen_feature(XENFEAT_auto_translated_physmap))
+ if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
return mfn;
if (unlikely((mfn >> machine_to_phys_order) != 0))
@@ -95,17 +97,18 @@ static inline unsigned long mfn_to_pfn(u
static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
{
unsigned long pfn = mfn_to_pfn(mfn);
- if ((pfn < max_mapnr)
- && !xen_feature(XENFEAT_auto_translated_physmap)
- && (phys_to_machine_mapping[pfn] != mfn))
+ if (likely(pfn < max_mapnr)
+ && likely(!xen_feature(XENFEAT_auto_translated_physmap))
+ && unlikely(phys_to_machine_mapping[pfn] != mfn))
return max_mapnr; /* force !pfn_valid() */
return pfn;
}
static inline void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
{
- BUG_ON(max_mapnr && pfn >= max_mapnr);
- if (xen_feature(XENFEAT_auto_translated_physmap)) {
+ if (likely(max_mapnr))
+ BUG_ON(pfn >= max_mapnr);
+ if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
return;
}
--- head-2009-07-28.orig/arch/x86/include/mach-xen/asm/maddr_64.h 2009-07-28 12:14:16.000000000 +0200
+++ head-2009-07-28/arch/x86/include/mach-xen/asm/maddr_64.h 2009-07-29 10:56:35.000000000 +0200
@@ -25,17 +25,19 @@ extern unsigned int machine_to_phys_or
static inline unsigned long pfn_to_mfn(unsigned long pfn)
{
- if (xen_feature(XENFEAT_auto_translated_physmap))
+ if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
return pfn;
- BUG_ON(max_mapnr && pfn >= max_mapnr);
+ if (likely(max_mapnr))
+ BUG_ON(pfn >= max_mapnr);
return phys_to_machine_mapping[pfn] & ~FOREIGN_FRAME_BIT;
}
static inline int phys_to_machine_mapping_valid(unsigned long pfn)
{
- if (xen_feature(XENFEAT_auto_translated_physmap))
+ if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
return 1;
- BUG_ON(max_mapnr && pfn >= max_mapnr);
+ if (likely(max_mapnr))
+ BUG_ON(pfn >= max_mapnr);
return (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY);
}
@@ -43,7 +45,7 @@ static inline unsigned long mfn_to_pfn(u
{
unsigned long pfn;
- if (xen_feature(XENFEAT_auto_translated_physmap))
+ if (unlikely(xen_feature(XENFEAT_auto_translated_physmap)))
return mfn;
if (unlikely((mfn >> machine_to_phys_order) != 0))
@@ -90,17 +92,18 @@ static inline unsigned long mfn_to_pfn(u
static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
{
unsigned long pfn = mfn_to_pfn(mfn);
- if ((pfn < max_mapnr)
- && !xen_feature(XENFEAT_auto_translated_physmap)
- && (phys_to_machine_mapping[pfn] != mfn))
+ if (likely(pfn < max_mapnr)
+ && likely(!xen_feature(XENFEAT_auto_translated_physmap))
+ && unlikely(phys_to_machine_mapping[pfn] != mfn))
return max_mapnr; /* force !pfn_valid() */
return pfn;
}
static inline void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
{
- BUG_ON(max_mapnr && pfn >= max_mapnr);
- if (xen_feature(XENFEAT_auto_translated_physmap)) {
+ if (likely(max_mapnr))
+ BUG_ON(pfn >= max_mapnr);
+ if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
return;
}
--- head-2009-07-28.orig/arch/x86/include/mach-xen/asm/pgtable_types.h 2009-07-28 13:14:11.000000000 +0200
+++ head-2009-07-28/arch/x86/include/mach-xen/asm/pgtable_types.h 2009-07-29 10:56:35.000000000 +0200
@@ -207,7 +207,7 @@ typedef struct { pgdval_t pgd; } pgd_t;
#define __pgd_ma(x) ((pgd_t) { (x) } )
static inline pgd_t xen_make_pgd(pgdval_t val)
{
- if (val & _PAGE_PRESENT)
+ if (likely(val & _PAGE_PRESENT))
val = pte_phys_to_machine(val);
return (pgd_t) { val };
}
@@ -217,10 +217,10 @@ static inline pgdval_t xen_pgd_val(pgd_t
{
pgdval_t ret = __pgd_val(pgd);
#if PAGETABLE_LEVELS == 2 && CONFIG_XEN_COMPAT <= 0x030002
- if (ret)
+ if (likely(ret))
ret = machine_to_phys(ret) | _PAGE_PRESENT;
#else
- if (ret & _PAGE_PRESENT)
+ if (likely(ret & _PAGE_PRESENT))
ret = pte_machine_to_phys(ret);
#endif
return ret;
@@ -237,7 +237,7 @@ typedef struct { pudval_t pud; } pud_t;
#define __pud_ma(x) ((pud_t) { (x) } )
static inline pud_t xen_make_pud(pudval_t val)
{
- if (val & _PAGE_PRESENT)
+ if (likely(val & _PAGE_PRESENT))
val = pte_phys_to_machine(val);
return (pud_t) { val };
}
@@ -246,7 +246,7 @@ static inline pud_t xen_make_pud(pudval_
static inline pudval_t xen_pud_val(pud_t pud)
{
pudval_t ret = __pud_val(pud);
- if (ret & _PAGE_PRESENT)
+ if (likely(ret & _PAGE_PRESENT))
ret = pte_machine_to_phys(ret);
return ret;
}
@@ -266,7 +266,7 @@ typedef struct { pmdval_t pmd; } pmd_t;
#define __pmd_ma(x) ((pmd_t) { (x) } )
static inline pmd_t xen_make_pmd(pmdval_t val)
{
- if (val & _PAGE_PRESENT)
+ if (likely(val & _PAGE_PRESENT))
val = pte_phys_to_machine(val);
return (pmd_t) { val };
}
@@ -276,10 +276,10 @@ static inline pmdval_t xen_pmd_val(pmd_t
{
pmdval_t ret = __pmd_val(pmd);
#if CONFIG_XEN_COMPAT <= 0x030002
- if (ret)
+ if (likely(ret))
ret = pte_machine_to_phys(ret) | _PAGE_PRESENT;
#else
- if (ret & _PAGE_PRESENT)
+ if (likely(ret & _PAGE_PRESENT))
ret = pte_machine_to_phys(ret);
#endif
return ret;
@@ -308,7 +308,7 @@ static inline pmdval_t pmd_flags(pmd_t p
#define __pte_ma(x) ((pte_t) { .pte = (x) } )
static inline pte_t xen_make_pte(pteval_t val)
{
- if ((val & (_PAGE_PRESENT|_PAGE_IOMAP)) == _PAGE_PRESENT)
+ if (likely((val & (_PAGE_PRESENT|_PAGE_IOMAP)) == _PAGE_PRESENT))
val = pte_phys_to_machine(val);
return (pte_t) { .pte = val };
}
@@ -317,7 +317,7 @@ static inline pte_t xen_make_pte(pteval_
static inline pteval_t xen_pte_val(pte_t pte)
{
pteval_t ret = __pte_val(pte);
- if ((pte.pte_low & (_PAGE_PRESENT|_PAGE_IOMAP)) == _PAGE_PRESENT)
+ if (likely((pte.pte_low & (_PAGE_PRESENT|_PAGE_IOMAP)) == _PAGE_PRESENT))
ret = pte_machine_to_phys(ret);
return ret;
}
