PAT-1.patch

aoiko, 04/19/2010 05:15 PM

Download (12.9 KB)

View differences:

sys/cpu/i386/include/cpufunc.h
396 396

  
397 397
#endif
398 398

  
399
#ifndef _CPU_WBINVL_DEFINED
400
static __inline void
401
cpu_wbinvl(void)
402
{
403
	__asm __volatile("wbinvd");
404
}
405
#endif
406

  
399 407
static __inline void
400 408
cpu_nop(void)
401 409
{
sys/cpu/i386/include/pmap.h
62 62
#define PG_A		0x020	/* A	Accessed		*/
63 63
#define	PG_M		0x040	/* D	Dirty			*/
64 64
#define	PG_PS		0x080	/* PS	Page size (0=4k,1=4M)	*/
65
#define PG_PTE_PAT	0x080	/* PAT	PAT index		*/
65 66
#define	PG_G		0x100	/* G	Global			*/
66 67
#define	PG_AVAIL1	0x200	/*    /	Available for system	*/
67 68
#define	PG_AVAIL2	0x400	/*   <	programmers use		*/
68 69
#define	PG_AVAIL3	0x800	/*    \				*/
70
#define PG_PDE_PAT	0x1000	/* PAT	PAT index		*/
69 71

  
70 72

  
71 73
/* Our various interpretations of the above */
sys/cpu/i386/include/specialreg.h
159 159
#define MSR_MTRR64kBase		0x250
160 160
#define MSR_MTRR16kBase		0x258
161 161
#define MSR_MTRR4kBase		0x268
162
#define	MSR_PAT			0x277
162 163
#define MSR_MTRRdefType		0x2ff
163 164
#define MSR_MC0_CTL		0x400
164 165
#define MSR_MC0_STATUS		0x401
......
184 185
#define MSR_THERM_INTERRUPT	0x19b
185 186
#define MSR_THERM_STATUS	0x19c
186 187

  
188
/*
189
 * PAT modes.
190
 */
191
#define	PAT_UNCACHEABLE		0x00
192
#define	PAT_WRITE_COMBINING	0x01
193
#define	PAT_WRITE_THROUGH	0x04
194
#define	PAT_WRITE_PROTECTED	0x05
195
#define	PAT_WRITE_BACK		0x06
196
#define	PAT_UNCACHED		0x07
197
#define	PAT_VALUE(i, m)		((long long)(m) << (8 * (i)))
198
#define	PAT_MASK(i)		PAT_VALUE(i, 0xff)
187 199

  
188 200
/*
189 201
 * Constants related to MTRRs
sys/platform/pc32/i386/mp_machdep.c
624 624
	load_cr0(cr0);
625 625
	pmap_set_opt();		/* PSE/4MB pages, etc */
626 626

  
627
	pmap_init_pat();	/* Page Attribute Table */
628

  
627 629
	/* set up CPU registers and state */
628 630
	cpu_setregs();
629 631

  
sys/platform/pc32/i386/pmap.c
158 158
static boolean_t pmap_initialized = FALSE;	/* Has pmap_init completed? */
159 159
static int pgeflag;		/* PG_G or-in */
160 160
static int pseflag;		/* PG_PS or-in */
161
static int pat_works;		/* Is page attribute table sane? */
161 162

  
162 163
static vm_object_t kptobj;
163 164

  
......
216 217
static vm_page_t pmap_page_lookup (vm_object_t object, vm_pindex_t pindex);
217 218
static int pmap_unuse_pt (pmap_t, vm_offset_t, vm_page_t, pmap_inval_info_t);
218 219
static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
220
static int pmap_cache_bits(int, boolean_t);
219 221

  
220 222
static unsigned pdir4mb;
221 223

  
......
295 297
	return (0);
296 298
}
297 299

  
300
/*
301
 * Setup the PAT MSR.
302
 */
303
void
304
pmap_init_pat(void)
305
{
306
	uint64_t pat_msr;
307

  
308
	/* Bail if this CPU doesn't implement PAT. */
309
	if (!(cpu_feature & CPUID_PAT))
310
		return;
311

  
312
#ifdef notyet
313
	if (cpu_vendor_id != CPU_VENDOR_INTEL ||
314
	    (I386_CPU_FAMILY(cpu_id) == 6 && I386_CPU_MODEL(cpu_id) >= 0xe)) {
315
#else
316
	if (!0) {
317
#endif
318
		/*
319
		 * Leave the indices 0-3 at the default of WB, WT, UC, and UC-.
320
		 * Program 4 and 5 as WP and WC.
321
		 * Leave 6 and 7 as UC and UC-.
322
		 */
323
		pat_msr = rdmsr(MSR_PAT);
324
		pat_msr &= ~(PAT_MASK(4) | PAT_MASK(5));
325
		pat_msr |= PAT_VALUE(4, PAT_WRITE_PROTECTED) |
326
		    PAT_VALUE(5, PAT_WRITE_COMBINING);
327
		pat_works = 1;
328
	} else {
329
		/*
330
		 * Due to some Intel errata, we can only safely use the lower 4
331
		 * PAT entries.  Thus, just replace PAT Index 2 with WC instead
332
		 * of UC-.
333
		 *
334
		 *   Intel Pentium III Processor Specification Update
335
		 * Errata E.27 (Upper Four PAT Entries Not Usable With Mode B
336
		 * or Mode C Paging)
337
		 *
338
		 *   Intel Pentium IV  Processor Specification Update
339
		 * Errata N46 (PAT Index MSB May Be Calculated Incorrectly)
340
		 */
341
		pat_msr = rdmsr(MSR_PAT);
342
		pat_msr &= ~PAT_MASK(2);
343
		pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING);
344
		pat_works = 0;
345
	}
346
	wrmsr(MSR_PAT, pat_msr);
347
}
348

  
298 349

  
299 350
/*
300 351
 *	Bootstrap the system enough to run with virtual memory.
......
446 497
	}
447 498
#endif
448 499

  
500
	pmap_init_pat();
449 501
	/*
450 502
	 * We need to finish setting up the globaldata page for the BSP.
451 503
	 * locore has already populated the page table for the mdglobaldata
......
554 606
 * Low level helper routines.....
555 607
 ***************************************************/
556 608

  
609
/*
610
 * Determine the appropriate bits to set in a PTE or PDE for a specified
611
 * caching mode.
612
 */
613
static int
614
pmap_cache_bits(int mode, boolean_t is_pde)
615
{
616
	int pat_flag, pat_index, cache_bits;
617

  
618
	/* The PAT bit is different for PTE's and PDE's. */
619
	pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
620

  
621
	/* If we don't support PAT, map extended modes to older ones. */
622
	if (!(cpu_feature & CPUID_PAT)) {
623
		switch (mode) {
624
		case PAT_UNCACHEABLE:
625
		case PAT_WRITE_THROUGH:
626
		case PAT_WRITE_BACK:
627
			break;
628
		case PAT_UNCACHED:
629
		case PAT_WRITE_COMBINING:
630
		case PAT_WRITE_PROTECTED:
631
			mode = PAT_UNCACHEABLE;
632
			break;
633
		}
634
	}
635
	
636
	/* Map the caching mode to a PAT index. */
637
	if (pat_works) {
638
		switch (mode) {
639
		case PAT_UNCACHEABLE:
640
			pat_index = 3;
641
			break;
642
		case PAT_WRITE_THROUGH:
643
			pat_index = 1;
644
			break;
645
		case PAT_WRITE_BACK:
646
			pat_index = 0;
647
			break;
648
		case PAT_UNCACHED:
649
			pat_index = 2;
650
			break;
651
		case PAT_WRITE_COMBINING:
652
			pat_index = 5;
653
			break;
654
		case PAT_WRITE_PROTECTED:
655
			pat_index = 4;
656
			break;
657
		default:
658
			panic("Unknown caching mode %d\n", mode);
659
		}
660
	} else {
661
		switch (mode) {
662
		case PAT_UNCACHED:
663
		case PAT_UNCACHEABLE:
664
		case PAT_WRITE_PROTECTED:
665
			pat_index = 3;
666
			break;
667
		case PAT_WRITE_THROUGH:
668
			pat_index = 1;
669
			break;
670
		case PAT_WRITE_BACK:
671
			pat_index = 0;
672
			break;
673
		case PAT_WRITE_COMBINING:
674
			pat_index = 2;
675
			break;
676
		default:
677
			panic("Unknown caching mode %d\n", mode);
678
		}
679
	}
680

  
681
	/* Map the 3-bit index value into the PAT, PCD, and PWT bits. */
682
	cache_bits = 0;
683
	if (pat_index & 0x4)
684
		cache_bits |= pat_flag;
685
	if (pat_index & 0x2)
686
		cache_bits |= PG_NC_PCD;
687
	if (pat_index & 0x1)
688
		cache_bits |= PG_NC_PWT;
689
	return (cache_bits);
690
}
691

  
557 692
#if defined(PMAP_DIAGNOSTIC)
558 693

  
559 694
/*
......
3210 3345
	kmem_free(&kernel_map, base, size);
3211 3346
}
3212 3347

  
3348
int
3349
pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
3350
{
3351
	vm_offset_t base, offset, tmpva;
3352
	pt_entry_t *pte;
3353
	u_int opte, npte;
3354
	pd_entry_t *pde;
3355
	pmap_inval_info info;
3356

  
3357
	base = trunc_page(va);
3358
	offset = va & PAGE_MASK;
3359
	size = roundup(offset + size, PAGE_SIZE);
3360

  
3361
	/*
3362
	 * Only supported on kernel virtual addresses
3363
	 */
3364
	if (base < KvaStart)
3365
		return (EINVAL);
3366

  
3367
	/* 4MB pages and pages that aren't mapped aren't supported. */
3368
	for (tmpva = base; tmpva < (base + size); tmpva += PAGE_SIZE) {
3369
		pde = pmap_pde(&kernel_pmap, tmpva);
3370
		if (*pde & PG_PS)
3371
			return (EINVAL);
3372
		if (*pde == 0)
3373
			return (EINVAL);
3374
		pte = vtopte(tmpva);
3375
		if (*pte == 0)
3376
			return (EINVAL);
3377
	}
3378

  
3379
	pmap_inval_init(&info);
3380
	/*
3381
	 * Ok, all the pages exist and are 4k, so run through them updating
3382
	 * their cache mode.
3383
	 */
3384
	for (tmpva = base; size > 0; ) {
3385
		pte = vtopte(tmpva);
3386

  
3387
		/*
3388
		 * The cache mode bits are all in the low 32-bits of the
3389
		 * PTE, so we can just spin on updating the low 32-bits.
3390
		 */
3391
		do {
3392
			opte = *(u_int *)pte;
3393
			npte = opte & ~(PG_PTE_PAT | PG_NC_PCD | PG_NC_PWT);
3394
			npte |= pmap_cache_bits(mode, 0);
3395
		} while (npte != opte &&
3396
		    !atomic_cmpset_int((u_int *)pte, opte, npte));
3397
		pmap_inval_add(&info, &kernel_pmap, tmpva);
3398
		tmpva += PAGE_SIZE;
3399
		size -= PAGE_SIZE;
3400
	}
3401

  
3402
	/*
3403
	 * Flush CPU caches to make sure any data isn't cached that shouldn't
3404
	 * be, etc.
3405
	 */
3406
	pmap_inval_cache_add(&info, &kernel_pmap, -1);
3407
	pmap_inval_flush(&info);
3408
	return (0);
3409
}
3410

  
3411

  
3213 3412
/*
3214 3413
 * perform the pmap work for mincore
3215 3414
 */
sys/platform/pc32/i386/pmap_inval.c
67 67
#ifdef SMP
68 68

  
69 69
static void
70
_cpu_invltlb(void *dummy)
70
_cpu_invltlb(void *dummy __unused)
71 71
{
72 72
    cpu_invltlb();
73 73
}
......
78 78
    cpu_invlpg(data);
79 79
}
80 80

  
81
static void
82
_cpu_wbinvl(void *dummy __unused)
83
{
84
	cpu_wbinvl();
85
}
86

  
81 87
#endif
82 88

  
83 89
/*
......
89 95
    info->pir_flags = 0;
90 96
}
91 97

  
98
#ifdef SMP
92 99
/*
93 100
 * Add a (pmap, va) pair to the invalidation list and protect access
94 101
 * as appropriate.
......
96 103
void
97 104
pmap_inval_add(pmap_inval_info_t info, pmap_t pmap, vm_offset_t va)
98 105
{
99
#ifdef SMP
100 106
    if ((info->pir_flags & PIRF_CPUSYNC) == 0) {
101 107
	info->pir_flags |= PIRF_CPUSYNC;
102 108
	info->pir_cpusync.cs_run_func = NULL;
......
106 112
    } else if (pmap->pm_active & ~info->pir_cpusync.cs_mask) {
107 113
	lwkt_cpusync_add(pmap->pm_active, &info->pir_cpusync);
108 114
    }
109
#else
110
    if (pmap->pm_active == 0)
111
	return;
112
#endif
113 115
    if ((info->pir_flags & (PIRF_INVLTLB|PIRF_INVL1PG)) == 0) {
114 116
	if (va == (vm_offset_t)-1) {
115 117
	    info->pir_flags |= PIRF_INVLTLB;
116
#ifdef SMP
117 118
	    info->pir_cpusync.cs_fin2_func = _cpu_invltlb;
118
#endif
119 119
	} else {
120 120
	    info->pir_flags |= PIRF_INVL1PG;
121 121
	    info->pir_cpusync.cs_data = (void *)va;
122
#ifdef SMP
123 122
	    info->pir_cpusync.cs_fin2_func = _cpu_invl1pg;
124
#endif
125 123
	}
126 124
    } else {
127 125
	info->pir_flags |= PIRF_INVLTLB;
128
#ifdef SMP
129 126
	info->pir_cpusync.cs_fin2_func = _cpu_invltlb;
130
#endif
131 127
    }
132 128
}
133 129

  
130
void
131
pmap_inval_cache_add(pmap_inval_info_t info, pmap_t pmap,
132
		     vm_offset_t va __unused)
133
{
134
    if ((info->pir_flags & PIRF_CPUSYNC) == 0) {
135
	info->pir_flags |= PIRF_CPUSYNC;
136
	info->pir_cpusync.cs_run_func = NULL;
137
	info->pir_cpusync.cs_fin1_func = NULL;
138
	info->pir_cpusync.cs_fin2_func = NULL;
139
	lwkt_cpusync_start(pmap->pm_active, &info->pir_cpusync);
140
    } else if (pmap->pm_active & ~info->pir_cpusync.cs_mask) {
141
	lwkt_cpusync_add(pmap->pm_active, &info->pir_cpusync);
142
    }
143
    info->pir_flags |= PIRF_WBINVL;
144
    info->pir_cpusync.cs_fin2_func = _cpu_wbinvl;
145
}
146

  
134 147
/*
135 148
 * Synchronize changes with target cpus.
136 149
 */
137 150
void
138 151
pmap_inval_flush(pmap_inval_info_t info)
139 152
{
140
#ifdef SMP
141 153
    if (info->pir_flags & PIRF_CPUSYNC)
142 154
	lwkt_cpusync_finish(&info->pir_cpusync);
143
#else
155
    info->pir_flags = 0;
156
}
157

  
158
#else	/* !SMP */
159

  
160
void
161
pmap_inval_add(pmap_inval_info_t info, pmap_t pmap, vm_offset_t va)
162
{
163
    if (pmap->pm_active == 0)
164
	return;
165
    if ((info->pir_flags & (PIRF_INVLTLB|PIRF_INVL1PG)) == 0) {
166
	if (va == (vm_offset_t)-1) {
167
	    info->pir_flags |= PIRF_INVLTLB;
168
	} else {
169
	    info->pir_flags |= PIRF_INVL1PG;
170
	    info->pir_cpusync.cs_data = (void *)va;
171
	}
172
    } else {
173
	info->pir_flags |= PIRF_INVLTLB;
174
    }
175
}
176

  
177
void
178
pmap_inval_cache_add(pmap_inval_info_t info, pmap_t pmap, vm_offset_t va)
179
{
180
    if (pmap->pm_active == 0)
181
	return;
182
    info->pir_flags |= PIRF_WBINVL;
183
}
184

  
185
void
186
pmap_inval_flush(pmap_inval_info_t info)
187
{
144 188
    if (info->pir_flags & PIRF_INVLTLB)
145 189
	cpu_invltlb();
146 190
    else if (info->pir_flags & PIRF_INVL1PG)
147 191
	cpu_invlpg(info->pir_cpusync.cs_data);
148
#endif
192
    if (info->pir_flags & PIRF_WBINVL)
193
	    cpu_wbinvl();
149 194
    info->pir_flags = 0;
150 195
}
151 196

  
197
#endif	/* SMP */
sys/platform/pc32/include/pmap.h
249 249
#ifdef SMP
250 250
void	pmap_set_opt (void);
251 251
#endif
252
void	pmap_init_pat(void);
253
int	pmap_change_attr(vm_offset_t, vm_size_t, int);
252 254

  
253 255
#endif /* _KERNEL */
254 256

  
sys/platform/pc32/include/pmap_inval.h
51 51
#define PIRF_INVLTLB	0x0001	/* request invalidation of whole table */
52 52
#define PIRF_INVL1PG	0x0002	/* else request invalidation of one page */
53 53
#define PIRF_CPUSYNC	0x0004	/* cpusync is currently active */
54
#define PIRF_WBINVL	0x0008	/* request cache invalidation */
54 55

  
55 56
#ifdef _KERNEL
56 57

  
......
60 61

  
61 62
void pmap_inval_init(pmap_inval_info_t);
62 63
void pmap_inval_add(pmap_inval_info_t, pmap_t, vm_offset_t);
64
void pmap_inval_cache_add(pmap_inval_info_t, pmap_t, vm_offset_t);
63 65
void pmap_inval_flush(pmap_inval_info_t);
64 66

  
65 67
#endif