[2/3] Add the amd64_edac driver from the EDAC SVN tree.

From: Vernon Mauery <vernux@us.ibm.com>

The amd64_edac driver is the preferred driver for AMD edac.  It supercedes
the k8_edac driver for revF and earlier.

From: Keith Mannthey <kmannth@us.ibm.com>
Signed-off-by: Vernon Mauery <vernux@us.ibm.com>
---
 drivers/edac/Kconfig      |    6 
 drivers/edac/Makefile     |    1 
 drivers/edac/amd64_edac.c | 6937 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 6944 insertions(+), 0 deletions(-)
 create mode 100644 drivers/edac/amd64_edac.c

diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig
index 167b030..63b7400 100644
--- a/drivers/edac/Kconfig
+++ b/drivers/edac/Kconfig
@@ -51,6 +51,12 @@ config EDAC_MM_EDAC
 	  occurred so that a particular failing memory module can be
 	  replaced.  If unsure, select 'Y'.
 
+config EDAC_AMD64_OPTERON
+	tristate "AMD64 (Opteron, Athlon64) K8, F10h, F11h"
+	depends on EDAC_MM_EDAC && X86 && PCI
+	help
+	Support for error detection and correction on the AMD 64
+	Families of Memory Controllers (K8, F10h and F11h)
 
 config EDAC_AMD76X
 	tristate "AMD 76x (760, 762, 768)"
diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile
index 7700dee..c787d2a 100644
--- a/drivers/edac/Makefile
+++ b/drivers/edac/Makefile
@@ -28,5 +28,6 @@ obj-$(CONFIG_EDAC_I3000)		+= i3000_edac.o
 obj-$(CONFIG_EDAC_I82860)		+= i82860_edac.o
 obj-$(CONFIG_EDAC_K8)			+= k8_edac.o
 obj-$(CONFIG_EDAC_R82600)		+= r82600_edac.o
+obj-$(CONFIG_EDAC_AMD64_OPTERON)	+= amd64_edac.o
 obj-$(CONFIG_EDAC_PASEMI)		+= pasemi_edac.o
 
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
new file mode 100644
index 0000000..79c9195
--- /dev/null
+++ b/drivers/edac/amd64_edac.c
@@ -0,0 +1,6937 @@
+/*
+ * AMD64 class Memory Controller kernel module
+ *
+ * Copyright 2008 - SoftwareBitMaker
+ *
+ * This file may be distributed under the terms of the
+ * GNU General Public License.
+ *
+ * Originally Written by Thayne Harbaugh
+ *
+ *      Changes by Douglas "norsk" Thompson  <dougthompson@xmission.com>:
+ *      	- K8 CPU Revision D and greater support
+ *
+ *      Changes by Dave Peterson <dsp@llnl.gov> <dave_peterson@pobox.com>:
+ *		- Module largely rewritten, with new (and hopefully correct)
+ *		code for dealing with node and chip select interleaving,
+ *		various code cleanup, and bug fixes
+ *		- Added support for memory hoisting using DRAM hole address
+ *		register
+ *
+ *	Changes by Douglas "norsk" Thompson <dougthompson@xmission.com>:
+ *		-K8 Rev (1207) revision support added, required Revision
+ *		specific mini-driver code to support Rev F as well as
+ *		prior revisions
+ *
+ *	Changes by Douglas "norsk" Thompson <dougthompson@xmission.com>:
+ *		-Family 10h revision support added. New PCI Device IDs,
+ *		indicating new changes. Actual registers modified
+ *		were slight, less than the Rev E to Rev F transition
+ *		but changing the PCI Device ID was the proper thing to
+ *		do, as it provides for almost automactic family
+ *		detection. The mods to Rev F required more family
+ *		information detection.
+ *
+ * This module is based on the following documents
+ * (available from http://www.amd.com/):
+ *
+ *	Title:	BIOS and Kernel Developer's Guide for AMD Athlon 64 and AMD
+ *		Opteron Processors
+ *	AMD publication #: 26094
+ *`	Revision: 3.26
+ *
+ *	Title:	BIOS and Kernel Developer's Guide for AMD NPT Family 0Fh
+ *		Processors
+ *	AMD publication #: 32559
+ *	Revision: 3.00
+ *	Issue Date: May 2006
+ *
+ *	Title:	BIOS and Kernel Developer's Guide (BKDG) For AMD Family 10h
+ *		Processors
+ *	AMD publication #: 31116
+ *	Revision: 3.00
+ *	Issue Date: September 07, 2007
+ *
+ * Sections in the first 2 documents are no longer in sync with each other.
+ * The Family 10h BKDG was totally re-written from scratch with a new
+ * presentation model.
+ * Therefore, comments that refer to a Document section might be off.
+ */
+
+#include <linux/module.h>
+#include <linux/ctype.h>
+#include <linux/init.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/slab.h>
+#include <linux/mmzone.h>
+#include <linux/edac.h>
+#include "edac_core.h"
+
+#define amd64_printk(level, fmt, arg...) \
+	edac_printk(level, "amd64", fmt, ##arg)
+
+#define amd64_mc_printk(mci, level, fmt, arg...) \
+	edac_mc_chipset_printk(mci, level, "amd64", fmt, ##arg)
+
+/* Throughout the comments in this code, the following terms are used:
+ *
+ *	SysAddr, DramAddr, and InputAddr
+ *
+ *  These terms come directly from the amd64 documentation
+ * (AMD publication #26094).  They are defined as follows:
+ *
+ *     SysAddr:
+ *         This is a physical address generated by a CPU core or a device
+ *         doing DMA.  If generated by a CPU core, a SysAddr is the result of
+ *         a virtual to physical address translation by the CPU core's address
+ *         translation mechanism (MMU).
+ *
+ *     DramAddr:
+ *         A DramAddr is derived from a SysAddr by subtracting an offset that
+ *         depends on which node the SysAddr maps to and whether the SysAddr
+ *         is within a range affected by memory hoisting.  The DRAM Base
+ *         (section 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers
+ *         determine which node a SysAddr maps to.
+ *
+ *         If the DRAM Hole Address Register (DHAR) is enabled and the SysAddr
+ *         is within the range of addresses specified by this register, then
+ *         a value x from the DHAR is subtracted from the SysAddr to produce a
+ *         DramAddr.  Here, x represents the base address for the node that
+ *         the SysAddr maps to plus an offset due to memory hoisting.  See
+ *         section 3.4.8 and the comments in get_dram_hole_info() and
+ *         sys_addr_to_dram_addr() below for more information.
+ *
+ *         If the SysAddr is not affected by the DHAR then a value y is
+ *         subtracted from the SysAddr to produce a DramAddr.  Here, y is the
+ *         base address for the node that the SysAddr maps to.  See section
+ *         3.4.4 and the comments in sys_addr_to_dram_addr() below for more
+ *         information.
+ *
+ *     InputAddr:
+ *         A DramAddr is translated to an InputAddr before being passed to the
+ *         memory controller for the node that the DramAddr is associated
+ *         with.  The memory controller then maps the InputAddr to a csrow.
+ *         If node interleaving is not in use, then the InputAddr has the same
+ *         value as the DramAddr.  Otherwise, the InputAddr is produced by
+ *         discarding the bits used for node interleaving from the DramAddr.
+ *         See section 3.4.4 for more information.
+ *
+ *         The memory controller for a given node uses its DRAM CS Base and
+ *         DRAM CS Mask registers to map an InputAddr to a csrow.  See
+ *         sections 3.5.4 and 3.5.5 for more information.
+ */
+
+/*
+ * Alter this version for the K8 module when modifications are made
+ */
+#define EDAC_AMD64_VERSION	" Ver: 3.1.5 " __DATE__
+#define EDAC_MOD_STR		"amd64_edac"
+
+/* PCI Device IDs we look for
+ *   Rev F and prior
+ */
+#ifndef PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP
+#define PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP		0x1101
+#endif				/* PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP */
+
+#ifndef PCI_DEVICE_ID_AMD_K8_NB_MEMCTL
+#define PCI_DEVICE_ID_AMD_K8_NB_MEMCTL		0x1102
+#endif				/* PCI_DEVICE_ID_AMD_K8_NB_MEMCTL */
+
+#ifndef PCI_DEVICE_ID_AMD_K8_NB_MISC
+#define PCI_DEVICE_ID_AMD_K8_NB_MISC		0x1103
+#endif				/* PCI_DEVICE_ID_AMD_K8_NB_MISC */
+
+/* PCI Device IDs for Family 10h - Quad Core */
+#ifndef PCI_DEVICE_ID_AMD_F10_NB_ADDRMAP
+#define PCI_DEVICE_ID_AMD_F10_NB_ADDRMAP	0x1201
+#endif				/* PCI_DEVICE_ID_AMD_F10_NB_ADDRMAP */
+
+#ifndef PCI_DEVICE_ID_AMD_F10_NB_MEMCTL
+#define PCI_DEVICE_ID_AMD_F10_NB_MEMCTL		0x1202
+#endif				/* PCI_DEVICE_ID_AMD_F10_NB_MEMCTL */
+
+#ifndef PCI_DEVICE_ID_AMD_F10_NB_MISC
+#define PCI_DEVICE_ID_AMD_F10_NB_MISC		0x1203
+#endif				/* PCI_DEVICE_ID_AMD_F10_NB_MISC */
+
+/* PCI Device IDs for Family 11h */
+#ifndef PCI_DEVICE_ID_AMD_F11_NB_ADDRMAP
+#define PCI_DEVICE_ID_AMD_F11_NB_ADDRMAP	0x1301
+#endif				/* PCI_DEVICE_ID_AMD_F11_NB_ADDRMAP */
+
+#ifndef PCI_DEVICE_ID_AMD_F11_NB_MEMCTL
+#define PCI_DEVICE_ID_AMD_F11_NB_MEMCTL		0x1302
+#endif				/* PCI_DEVICE_ID_AMD_F11_NB_MEMCTL */
+
+#ifndef PCI_DEVICE_ID_AMD_F11_NB_MISC
+#define PCI_DEVICE_ID_AMD_F11_NB_MISC		0x1303
+#endif				/* PCI_DEVICE_ID_AMD_F11_NB_MISC */
+
+
+/* Extended Model from CPUID, for CPU Revision numbers */
+#define OPTERON_CPU_LE_REV_C    0
+#define OPTERON_CPU_REV_D       1
+#define OPTERON_CPU_REV_E       2
+
+/* Unknown Extended Model value */
+#define OPTERON_CPU_REV_X	3
+
+/* NPT processors have the following Extended Models */
+#define OPTERON_CPU_REV_F       4
+#define OPTERON_CPU_REV_FA	5
+
+/* Hardware limit on ChipSelect rows per MC
+ * and Processors per system
+ */
+#define CHIPSELECT_COUNT	8
+#define DRAM_REG_COUNT		8
+
+/*************************************************************/
+/* K8 register addresses - device 0 Function 1 - Address Map */
+/*************************************************************/
+#define K8_DRAM_BASE_LOW	0x40
+				/* Function 1: DRAM Base Register (8 x 32b
+				 * interlaced with K8_DRAM_LIMIT_LOW)
+				 *
+				 * 31:16 DRAM base address reg bits[39:24]
+				 * 15:11 reserved
+				 * 10:8  interleave enable
+				 *  7:2  reserved
+				 *  1    write enable
+				 *  0    read enable
+				 *
+				 */
+
+#define K8_DRAM_LIMIT_LOW	0x44
+				/* Function 1: DRAM Limit Register (8 x 32b
+				 * interlaced with K8_DRAM_BASE_LOW)
+				 *
+				 * 31:16 DRAM Limit addr 32:24
+				 * 15:11 reserved
+				 * 10:8  interleave select
+				 *  7:3  reserved
+				 *  2:0  destination node ID
+				 */
+
+#define K8_DHAR         0xf0
+				/* Function 1: DRAM Hole Address Register
+				 *
+				 * K8
+				 * 31:24 DramHoleBase
+				 * 23:16 reserved
+				 * 15:8  DramHoleOffset
+				 *  7:1  reserved
+				 *    0  DramHoleValid
+				 *
+				 * F10
+				 * 31:24 DramHoleBase
+				 * 23:16 reserved
+				 * 15:7  DramHoleOffset
+				 *  6:2  reserved
+				 *    1  DramMemHoistValid
+				 *    0  DramHoleValid
+				 */
+#define K8_DHAR_VALID		BIT(0)
+#define F10_DRAM_MEM_HOIST_VALID BIT(1)
+
+#define K8_DHAR_BASE_MASK	0xff000000
+#define K8_DHAR_OFFSET_MASK	0x0000ff00
+#define EXTRACT_K8_DHAR_BASE(dhar)	(dhar & K8_DHAR_BASE_MASK)
+#define EXTRACT_K8_DHAR_OFFSET(dhar)	((dhar & K8_DHAR_OFFSET_MASK) << 16)
+
+#define F10_DHAR_MEM_HOIST_VALID	BIT(1)
+#define F10_DHAR_BASE_MASK	0xff000000
+#define F10_DHAR_OFFSET_MASK	0x0000ff10
+				/* NOTE: Extra mask bit vs K8 */
+
+#define EXTRACT_F10_DHAR_BASE(dhar)	(dhar & F10_DHAR_BASE_MASK)
+#define EXTRACT_F10_DHAR_OFFSET(dhar)	((dhar & F10_DHAR_OFFSET_MASK) << 16)
+
+
+/* F10 High BASE/LIMIT registers */
+
+#define F10_DRAM_BASE_HIGH	0x140
+				/* Function 1: DRAM Base register HIGH
+				 *
+				 * 7:0  Drambase[47:40] DRAM base address reg
+				 *	bits[47:40]
+				 */
+
+#define F10_DRAM_LIMIT_HIGH	0x144
+				/* Function 1: DRAM Limit Register HIGH
+				 *
+				 * 7:0  DRAM limit address register bits[47:40]
+				 */
+
+/*****************************************************************/
+/* K8 register addresses - device 0 Function 2 - DRAM controller */
+/*************************************************************/
+#define K8_DCSB0		0x40
+#define F10_DCSB1		0x140
+				/* Function 2: DRAM Chip-Select Base (8 x 32b)
+				 *
+				 * For Rev E and prior
+				 * 31:21 Base addr high 35:25
+				 * 20:16 reserved
+				 * 15:9  Base addr low 19:13 (interlvd)
+				 *  8:1  reserved
+				 *  0    chip-select bank enable
+				 *
+				 * For Rev F (NPT) and later
+				 * 31:29 reserved
+				 * 28:19 Base address (36:27)
+				 * 18:14 reserved
+				 * 13:5  Base address (21:13)
+				 * 4:3   reserved
+				 * 2     TestFail
+				 * 1     Spare Rank
+				 * 0     CESenable
+				 *
+				 */
+#define K8_DCSB_CS_ENABLE	0x1
+#define K8_DCSB_NPT_SPARE	0x2
+#define K8_DCSB_NPT_TESTFAIL	0x4
+
+/* REV E: selects bits 31-21 and 15-9 from DCSB
+ * and the shift amount to form address
+ */
+#define REV_E_DCSB_BASE_BITS	(0xFFE0FE00ULL)
+#define REV_E_DCS_SHIFT		4
+#define REV_E_DCSM_SHIFT	0
+#define REV_E_DCSM_COUNT	8
+
+/* REV F: selects bits 28-19 and 13-5 from DCSB
+ * and the shift amount to form address
+ */
+#define REV_F_DCSB_BASE_BITS	(0x1FF83FE0ULL)
+#define REV_F_DCS_SHIFT		8
+#define REV_F_DCSM_SHIFT	1
+#define REV_F_DCSM_COUNT	4
+
+/* F10: selects bits 28-19 and 13-5 from DCSB
+ * and the shift amount to form address
+ */
+#define REV_F10_DCSB_BASE_BITS	(0x1FF83FE0ULL)
+#define REV_F10_DCS_SHIFT	8
+#define REV_F10_DCSM_SHIFT	1
+#define REV_F10_DCSM_COUNT	4
+
+/* DRAM CS Mask Registers */
+#define K8_DCSM0		0x60
+#define F10_DCSM1		0x160
+				/* Function 2: DRAM Chip-Select Mask (8 x 32b)
+				 *
+				 * 31:30 reserved
+				 * 29:21 addr mask high 33:25
+				 * 20:16 reserved
+				 * 15:9  addr mask low  19:13
+				 *  8:0  reserved
+				 */
+
+/* REV E: selects bits 29-21 and 15-9 from DCSM */
+#define REV_E_DCSM_MASK_BITS 		0x3FE0FE00
+/* 	represents unused bits [24-20] and [12-0] */
+#define REV_E_DCS_NOTUSED_BITS		0x1f01fff
+
+/* REV F: selects bits 28-19 and 13-5 from DCSM */
+#define REV_F_DCSM_MASK_BITS 		0x1FF83FC0
+/* 	represents unused bits [26-22] and [12-0] */
+#define REV_F_DCS_NOTUSED_BITS		0x03c1fff
+
+/* REV F10: selects bits 28-19 and 13-5 from DCSM */
+#define REV_F10_DCSM_MASK_BITS 		0x1FF83FC0
+/* 	represents unused bits [26-22] and [12-0] */
+#define REV_F10_DCS_NOTUSED_BITS		0x03c1fff
+
+
+#define DBAM0		0x80
+#define DBAM1		0x180
+				/* Function 2: DRAM Base Addr Mapping (32b) */
+
+/* Extract the DIMM 'type' on the i'th DIMM from the DBAM reg value passed */
+#define DBAM_DIMM(i, reg)	((((reg) >> (4*i))) & 0xF)
+
+#define DBAM_MAX_VALUE	11
+
+
+#define K8_DCL0_LOW	0x90
+#define F10_DCL1_LOW	0x190
+				/* Function 2: DRAM configuration low reg (32b)
+				 *
+				 * Rev E and earlier CPUS:
+				 *
+				 * 31:28 reserved
+				 * 27:25 Bypass Max: 000b=respect
+				 * 24    Dissable receivers - no sockets
+				 * 23:20 x4 DIMMS
+				 * 19    32byte chunks
+				 * 18    Unbuffered
+				 * 17    ECC enabled
+				 * 16    128/64 bit (dual/single chan)
+				 * 15:14 R/W Queue bypass count
+				 * 13    Self refresh
+				 * 12    exit self refresh
+				 * 11    mem clear status
+				 * 10    DRAM enable
+				 *  9    reserved
+				 *  8    DRAM init
+				 *  7:4  reserved
+				 *  3    dis DQS hysteresis
+				 *  2    QFC enabled
+				 *  1    DRAM drive strength
+				 *  0    Digital Locked Loop disable
+				 *
+				 * Rev F
+				 *
+				 * 31:20 reserved
+				 * 19    DIMM ECC Enable
+				 * 18:17 reserved
+				 * 16    Unbuffered DIMM
+				 * 15:12 x4 DIMMs
+				 * 11    Width128 bits
+				 * 10    burstLength32
+				 *  9    SelRefRateEn
+				 *  8    ParEn
+				 *  7    DramDrvWeak
+				 *  6    reserved
+				 * 5:4   DramTerm
+				 * 3:2   reserved
+				 *  1    ExitSelfRef
+				 *  0    InitDram
+				 *
+				 * Rev F10h
+				 *
+				 * 31:24 reserved
+				 * 23    FoceAutoPcg
+				 * 22:21 IdleCycLowLimit
+				 * 20    DynPageCloseEn
+				 * 19    DIMM ECC Enable
+				 * 18    PendRefPayback
+				 * 17    EnterSelRef
+				 * 16    Unbuffered DIMM
+				 * 15:12 x4 DIMMs
+				 * 11    Width128 bits
+				 * 10    burstLength32
+				 *  9    SelRefRateEn
+				 *  8    ParEn
+				 *  7    DramDrvWeak
+				 *  6    DisDqsBar
+				 * 5:4   DramTerm
+				 * 3:2   reserved
+				 *  1    ExitSelfRef
+				 *  0    InitDram
+				 */
+#define REVE_WIDTH_128	BIT(16)
+#define F10_WIDTH_128	BIT(11)
+
+
+#define F10_DCHR_0	0x94
+#define F10_DCHR_1	0x194
+
+#define F10_DCHR_FOUR_RANK_DIMM	BIT(18)
+
+				/* Function 2: DRAM ronfiguration High Reg */
+#define F10_DCHR_Ddr3Mode	BIT(8)
+#define F10_DCHR_MblMode	BIT(6)
+
+
+#define F10_DCTL_SEL_LOW	0x110
+				/* Function 2: DRAM Controller SELECT LOW */
+#define F10_DCTL_SEL_LOW_DctSelBaseAddr(x)  ((x) & 0xFFFFF800)
+#define F10_DCTL_SEL_LOW_MemCleared	BIT(10)
+#define F10_DCTL_SEL_LOW_DramEnable	BIT(8)
+#define F10_DCTL_SEL_LOW_DctDatIntLv	BIT(5)
+#define F10_DCTL_SEL_LOW_DctGangEn	BIT(4)
+#define F10_DCTL_SEL_LOW_DctSelIntLvEn	BIT(2)
+#define F10_DCTL_SEL_LOW_DctSelHi	BIT(1)
+#define F10_DCTL_SEL_LOW_DctSelHiRngEn	BIT(0)
+#define F10_DCTL_SEL_LOW_DctSelIntLvAddr(x) (((x) >> 6) & 0x3)
+#define F10_DCTL_SEL_LOW_DctSelBaseAddr(x) ((x) & 0xFFFFF800)
+
+
+#define F10_DCTL_SEL_HIGH	0x114
+				/* device 0 Function 2 - DRAM
+				 * Controller SELECT HIGH
+				 */
+
+/**************************************************************/
+/* K8 register addresses - device 0 Function 3 - Misc Control */
+/**************************************************************/
+#define K8_NBCTL	0x40
+				/* Function 3: MCA NB Control (32b)
+				 *
+				 *  1    MCA UE Reporting
+				 *  0    MCA CE Reporting
+				 */
+/* Correctable ECC error reporting enable */
+#define K8_NBCTL_CECCEn		BIT(0)
+
+/* UnCorrectable ECC error reporting enable */
+#define K8_NBCTL_UECCEn		BIT(1)
+
+#define K8_NBCFG	0x44
+				/* Function 3: MCA NB Config (32b)
+				 *
+				 * 23    Chip-kill x4 ECC enable
+				 * 22    ECC enable
+				 */
+#define		K8_NBCFG_CHIPKILL	BIT(23)
+#define		K8_NBCFG_ECC_ENABLE	BIT(22)
+
+#define K8_NBSL		0x48
+				/* Function 3: MCA NB Status Low (32b)
+				 *
+				 * 31:24 Syndrome 15:8 chip-kill x4
+				 * 23:20 reserved
+				 * 19:16 Extended err code (F0fh and earlier)
+				 * 20:16 Extended err code (F10h and later)
+				 * 15:0  Err code
+				 */
+
+
+#define	EXTRACT_HIGH_SYNDROME(x)	(((x) >> 24) & 0xff)
+#define EXTRACT_EXT_ERROR_CODE(x)	(((x) >> 16) & 0x1f)
+
+/* Start Family F10h: Normalized Extended Error Codes */
+#define F10_NBSL_EXT_ERR_RES		(0x0)
+#define F10_NBSL_EXT_ERR_CRC		(0x1)
+#define F10_NBSL_EXT_ERR_SYNC		(0x2)
+#define F10_NBSL_EXT_ERR_MST		(0x3)
+#define F10_NBSL_EXT_ERR_TGT		(0x4)
+#define F10_NBSL_EXT_ERR_GART		(0x5)
+#define F10_NBSL_EXT_ERR_RMW		(0x6)
+#define F10_NBSL_EXT_ERR_WDT		(0x7)
+#define F10_NBSL_EXT_ERR_ECC		(0x8)
+#define F10_NBSL_EXT_ERR_DEV		(0x9)
+#define F10_NBSL_EXT_ERR_LINK_DATA	(0xA)
+
+/* Next two are overloaded values */
+#define F10_NBSL_EXT_ERR_LINK_PROTO	(0xB)
+#define F10_NBSL_EXT_ERR_L3_PROTO	(0xB)
+
+#define F10_NBSL_EXT_ERR_NB_ARRAY	(0xC)
+#define F10_NBSL_EXT_ERR_DRAM_PARITY	(0xD)
+#define F10_NBSL_EXT_ERR_LINK_RETRY	(0xE)
+
+/* Next two are overloaded values */
+#define F10_NBSL_EXT_ERR_GART_WALK	(0xF)
+#define F10_NBSL_EXT_ERR_DEV_WALK	(0xF)
+
+/* 0x10 to 0x1B: Reserved */
+
+#define F10_NBSL_EXT_ERR_L3_DATA	(0x1C)
+#define F10_NBSL_EXT_ERR_L3_TAG		(0x1D)
+#define F10_NBSL_EXT_ERR_L3_LRU		(0x1E)
+/* End Family F10h: Extended Error Codes */
+
+/* Start K8: Normalized Extended Error Codes */
+#define K8_NBSL_EXT_ERR_ECC		(0x0)
+#define K8_NBSL_EXT_ERR_CRC		(0x1)
+#define K8_NBSL_EXT_ERR_SYNC		(0x2)
+#define K8_NBSL_EXT_ERR_MST		(0x3)
+#define K8_NBSL_EXT_ERR_TGT		(0x4)
+#define K8_NBSL_EXT_ERR_GART		(0x5)
+#define K8_NBSL_EXT_ERR_RMW		(0x6)
+#define K8_NBSL_EXT_ERR_WDT		(0x7)
+#define K8_NBSL_EXT_ERR_CHIPKILL_ECC	(0x8)
+#define K8_NBSL_EXT_ERR_DRAM_PARITY	(0xD)
+/* End K8: Extended Error Codes */
+
+
+/* Error Code */
+#define EXTRACT_ERROR_CODE(x)	((x) & 0xffff)
+#define		TEST_TLB_ERROR(x)	(((x) & 0xFFF0) == 0x0010)
+#define		TEST_MEM_ERROR(x)	(((x) & 0xFF00) == 0x0100)
+#define		TEST_BUS_ERROR(x)	(((x) & 0xF800) == 0x0800)
+#define 	EXTRACT_TT_CODE(x)	(((x) >> 2) & 0x3)
+#define 	EXTRACT_II_CODE(x)	(((x) >> 2) & 0x3)
+#define 	EXTRACT_LL_CODE(x)	(((x) >> 0) & 0x3)
+#define		EXTRACT_RRRR_CODE(x)	(((x) >> 4) & 0xf)
+#define 	EXTRACT_TO_CODE(x)	(((x) >> 8) & 0x1)
+#define		EXTRACT_PP_CODE(x)	(((x) >> 9) & 0x3)
+
+/* The following are for BUS type errors AFTER values have been
+ * normalized by shifting right
+ */
+#define K8_NBSL_PP_SRC		(0x0)
+#define K8_NBSL_PP_RES		(0x1)
+#define K8_NBSL_PP_OBS		(0x2)
+#define K8_NBSL_PP_GENERIC	(0x3)
+
+
+#define K8_NBSH		0x4C
+				/* Function 3: MCA NB Status High (32b)
+				 *
+				 * 31    Err valid
+				 * 30    Err overflow
+				 * 29    Uncorrected err
+				 * 28    Err enable
+				 * 27    Misc err reg valid
+				 * 26    Err addr valid
+				 * 25    proc context corrupt
+				 * 24:23 reserved
+				 * 22:15 Syndrome bits 7:0
+				 * 14    CE
+				 * 13    UE
+				 * 12:9  reserved
+				 *  8    err found by scrubber
+				 *  7    reserved
+				 *  6:4  Hyper-transport link number
+				 *    3:2  reserved= Rev F/Family 10h=Quad Core
+				 *  3    Err CPU 3	(F10)
+				 *  2    Err CPU 2	(F10)
+				 *  1    Err CPU 1	(Dual Core)
+				 *  0    Err CPU 0
+				 */
+
+#define K8_NBSH_VALID_BIT		BIT(31)
+#define K8_NBSH_OVERFLOW		BIT(30)
+#define K8_NBSH_UNCORRECTED_ERR		BIT(29)
+#define K8_NBSH_ERR_ENABLE		BIT(28)
+#define K8_NBSH_MISC_ERR_VALID		BIT(27)
+#define K8_NBSH_VALID_ERROR_ADDR	BIT(26)
+#define K8_NBSH_PCC			BIT(25)
+#define K8_NBSH_CECC			BIT(14)
+#define K8_NBSH_UECC			BIT(13)
+#define K8_NBSH_ERR_SCRUBER		BIT(8)
+#define K8_NBSH_CORE3			BIT(3)
+#define K8_NBSH_CORE2			BIT(2)
+#define K8_NBSH_CORE1			BIT(1)
+#define K8_NBSH_CORE0			BIT(0)
+
+#define EXTRACT_LDT_LINK(x) 		(((x) >> 4) & 0x7)
+#define EXTRACT_ERR_CPU_MAP(x)		((x) & 0xF)
+#define EXTRACT_LOW_SYNDROME(x)		(((x) >> 15) & 0xff)
+
+
+#define K8_NBEAL	0x50
+				/* Function 3: MCA NB err addr low (32b)
+				 *
+				 * 31:3  Err addr low 31:3
+				 *  2:0  reserved
+				 */
+
+#define K8_NBEAH	0x54
+				/* Function 3: MCA NB err addr high (32b)
+				 *
+				 * 31:8  reserved
+				 *  7:0  Err addr high 39:32
+				 */
+
+#define K8_SCRCTRL      0x58
+				/* Function 3: Memory scrub control register.
+				 *
+				 * 30:21 reserved
+				 * 20:16 dcache scrub
+				 * 15:13 reserved
+				 * 12:8  L2Scrub
+				 * 7:5   reserved
+				 * 4:0   dramscrub
+				 *
+				 */
+
+
+
+#define F10_NB_CFG_LOW	0x88
+				/* Function 3: NB Configuration reg low */
+#define 	F10_NB_CFG_LOW_ENABLE_EXT_CFG	BIT(14)
+
+#define F10_NB_CFG_HIGH	0x8C
+				/* Function 3: NB Configuration reg high */
+
+
+#define F10_ONLINE_SPARE	0xB0
+				/* Function 3: On-Line Spare Control Register */
+#define F10_ONLINE_SPARE_SWAPDONE0(x)	((x) & BIT(1))
+#define F10_ONLINE_SPARE_SWAPDONE1(x)	((x) & BIT(3))
+#define F10_ONLINE_SPARE_BADDRAM_CS0(x)	(((x) >> 4) & 0x00000007)
+#define F10_ONLINE_SPARE_BADDRAM_CS1(x)	(((x) >> 8) & 0x00000007)
+
+
+#define F10_NB_ARRAY_ADDR	0xB8
+				/* Function 3:
+				 *	Error injection NB Array Addr Reg
+				 *
+				 * For a 64-byte cacheline, we can select
+				 * one of 4 16-byte sections (0,1,2,3) of
+				 * that cacheline.
+				 * Bits 2:1 provide that selection.
+				 *
+				 * which 16-bit word of the section is
+				 * selected by bitmap 28:20 of F10_NB_ARRAY_DATA
+				 *
+				 * which bit of the the selected word
+				 * is then masked by bits 15:0 of the same reg
+				 *
+				 * Thus we have a tuple of:
+				 *	section,word,bit
+				 */
+
+				/* DRAM ECC Array Select */
+#define F10_NB_ARRAY_DRAM_ECC	0x80000000
+
+				/* Bits 2:1 are used to select 16-byte
+				 * section within a 64-byte cacheline
+				 */
+#define SET_NB_ARRAY_ADDRESS(section)	(((section) & 0x3) << 1)
+
+#define F10_NB_ARRAY_DATA	0xBC
+				/* Function 3:
+				 *	Error injection NB Array Data Reg
+				 *
+				 * 28:20	ErrInjEn selects 16-bit word
+				 *		(bit map: 0-8)
+				 * 17		EccWrReq
+				 * 16		EccRdReq
+				 * 15:0		EccVector, select bits
+				 */
+#define SET_NB_DRAM_INJECTION_WRITE(word,bits)  \
+				(BIT(((word) & 0xF) + 20) | \
+				BIT(17) |  \
+				((bits) & 0xF))
+#define SET_NB_DRAM_INJECTION_READ(word,bits)  \
+				(BIT(((word) & 0xF) + 20) | \
+				BIT(16) |  \
+				((bits) & 0xF))
+
+
+
+#define K8_NBCAP	0xE8
+				/* Function 3: MCA NB capabilities (32b)
+				 *
+				 * 31:9  reserved
+				 *  4    ChipKill S4ECD4ED capable
+				 *  3    SECDED capable
+				 */
+#define K8_NBCAP_CORES		(BIT(12)|BIT(13))
+#define K8_NBCAP_CHIPKILL	BIT(4)
+#define K8_NBCAP_SECDED		BIT(3)
+#define K8_NBCAP_8_NODE		BIT(2)
+#define K8_NBCAP_DUAL_NODE	BIT(1)
+#define K8_NBCAP_DCT_DUAL	BIT(0)
+
+				/* MSR's */
+
+				/*
+				 * K8_MSR_MCxCTL (64b)
+				 * (0x400,404,408,40C,410)
+				 * 63    Enable reporting source 63
+				 *  .
+				 *  .
+				 *  .
+				 *  2    Enable error source 2
+				 *  1    Enable error source 1
+				 *  0    Enable error source 0
+				 */
+
+				/*
+				 * K8_MSR_MCxSTAT (64b)
+				 * (0x401,405,409,40D,411)
+				 * 63    Error valid
+				 * 62    Status overflow
+				 * 61    UE
+				 * 60    Enabled error condition
+				 * 59    Misc register valid (not used)
+				 * 58    Err addr register valid
+				 * 57    Processor context corrupt
+				 * 56:32 Other information
+				 * 31:16 Model specific error code
+				 * 15:0  MCA err code
+				 */
+
+				/*
+				 * K8_MSR_MCxADDR (64b)
+				 * (0x402,406,40A,40E,412)
+				 * 63:48 reserved
+				 * 47:0  Address
+				 */
+
+				/*
+				 * K8_MSR_MCxMISC (64b)
+				 * (0x403,407,40B,40F,413)
+				 * Unused on Athlon64 and K8
+				 */
+
+/* MSR Regs */
+#define K8_MSR_MCGCTL	0x017b
+				/* Machine Chk Global report ctl (64b)
+				 *
+				 * 31:5  reserved
+				 *  4    North Bridge
+				 *  3    Load/Store
+				 *  2    Bus Unit
+				 *  1    Instruction Cache
+				 *  0    Data Cache
+				 */
+#define K8_MSR_MCGCTL_NBE	BIT(4)
+
+
+#define F10_EXT_NBCFG	0x180
+				/* Function 3: Extended MCA NB Config (32b)
+				 *
+				 */
+#define F10_EXT_NBCFG_ECC_SYM_SIZE	BIT(25)
+				/* 0 = x4 ChipKill symbol size
+				 * 1 = x8 ChipKill symbol size
+				 */
+
+#define K8_MSR_MC4CTL	0x0410	/* North Bridge Check report ctl (64b) */
+#define K8_MSR_MC4STAT	0x0411	/* North Bridge status (64b) */
+#define K8_MSR_MC4ADDR	0x0412	/* North Bridge Address (64b) */
+
+#define K8_MSR_TOP_MEM	0xC001001A	/* TOP_MEM */
+#define K8_MSR_TOP_MEM2	0xC001001D	/* TOP_MEM2 */
+
+
+static struct edac_pci_ctl_info *amd64_ctl_pci;
+
+static int report_gart_errors;
+module_param(report_gart_errors, int, 0644);
+
+/* Set by command line parameter. If BIOS has enabled the ECC,
+ * this override is cleared to prevent re-enabling the hardware by this
+ * driver.
+ */
+static int ecc_enable_override;
+module_param(ecc_enable_override, int, 0644);
+
+
+/*
+ * AMD sets the first MC device at device ID 0x18. Subsequent ones go up
+ * from there.
+ */
+static inline int get_mc_node_id_from_pdev(struct pci_dev *pdev)
+{
+	/* Take whatever SLOT we are and subtract 0x18 to get this node ID */
+	return PCI_SLOT(pdev->devfn) - 0x18;
+}
+
+/* Ugly hack that allows module to compile when built as part of a 32-bit
+ * kernel.  Just in case anyone wants to run a 32-bit kernel on their Opteron.
+ */
+#ifndef MAXNODE
+#define MAXNODE 8
+#endif
+
+/* Lookup table for all possible MC control instances */
+struct amd64_pvt;
+static struct mem_ctl_info *mci_lookup[MAXNODE];
+static struct amd64_pvt *pvt_lookup[MAXNODE];
+
+/* Each entry holds the CPU revision of all CPU cores for the given node. */
+static int amd64_node_revision_table[MAXNODE] = { 0 };
+
+static inline int node_rev(int node_id)
+{
+	return amd64_node_revision_table[node_id];
+}
+
+static void store_node_revision(void *param)
+{
+	int node_id, revision;
+
+	/* Multiple CPU cores on the same node will all write their revision
+	 * number to the same array entry.  This is ok.  For a given node, all
+	 * CPU cores are on the same piece of silicon and share the same
+	 * revision number.
+	 */
+	node_id = (cpuid_ebx(1) >> 24) & 0x07;
+	revision = (cpuid_eax(1) >> 16) & 0x0f;
+	amd64_node_revision_table[node_id] = revision;
+}
+
+/* build_node_revision_table
+ *	Initialize amd64_node_revision_table.
+ */
+static void build_node_revision_table(void)
+{
+	static int initialized;
+
+	if (initialized)
+		return;
+
+	on_each_cpu(store_node_revision, NULL, 1, 1);
+	initialized = 1;
+}
+
+/*
+ * AMD Athlon64 and Opterons now have several 'Families'
+ * as defined by the PCI DEVICE IDs:
+ *
+ *	First Family:
+ *		K8
+ *	New Families denoted by leading "F"amily mark
+ *		F10
+ *		F11
+ */
+enum amd64_chipset_families {
+	K8_CPUS = 0,
+	F10_CPUS,
+	F11_CPUS,
+};
+
+/*
+ * Structure to hold:
+ *	1) dynamicly read status and error address HW registers
+ *	2) sysfs entered values
+ *	3) MCE values
+ *
+ * Depends on entry into the modules
+ */
+struct amd64_error_info_regs {
+	u32 nbcfg;
+	u32 nbsh;
+	u32 nbsl;
+	u32 nbeah;
+	u32 nbeal;
+};
+
+struct amd64_pvt;
+
+/*
+ * Each of the PCI Device IDs types have their own set of hardware
+ * accessor function and per device encoding/decoding logic.
+ */
+struct low_ops {
+	int (*probe_valid_hardware)(struct amd64_pvt *pvt);
+	int (*early_channel_count)(struct amd64_pvt *pvt);
+	void (*setup)(struct amd64_pvt *pvt);
+	void (*teardown)(struct amd64_pvt *pvt);
+
+	void (*display_info)(struct amd64_pvt *pvt);
+	int (*set_scrub_rate)(struct mem_ctl_info *mci, u32 *band_width);
+	int (*get_scrub_rate)(struct mem_ctl_info *mci, u32 *band_width);
+	u64 (*get_error_address)(struct mem_ctl_info *mci,
+			struct amd64_error_info_regs *info);
+	void (*read_dram_base_limit)(struct amd64_pvt *pvt, int dram);
+	void (*read_dram_ctl_register)(struct amd64_pvt *pvt);
+	void (*read_dctBase_dctMask)(struct amd64_pvt *pvt);
+	void (*map_sysaddr_to_csrow)(struct mem_ctl_info *mci,
+					struct amd64_error_info_regs *info,
+					u64 SystemAddr);
+	int (*get_dram_hole_info)(struct mem_ctl_info *mci,
+					u64 *base, u64 *offset, u64 *size);
+	void (*decode_misc_regs)(struct amd64_pvt *pvt);
+	enum dev_type (*determine_dram_type)(struct amd64_pvt *pvt, int row);
+	enum mem_type (*determine_memory_type)(struct amd64_pvt *pvt);
+	enum edac_type (*determine_edac_cap)(struct amd64_pvt *pvt);
+	int (*dbam_map_to_pages)(struct amd64_pvt *pvt, int dram_map);
+	void (*read_dbam_reg)(struct amd64_pvt *pvt);
+};
+
+/*
+ * amd64 family unique informatoin
+ */
+struct amd64_family_type {
+	const char *ctl_name;
+	u16 addr_f1_ctl;
+	u16 misc_f3_ctl;
+	struct low_ops ops;
+};
+
+
+/*
+ * NO-OP K8 function for registers that don't exit in K8
+ *
+ * NO-OP K8 function for registers that don't exit in K8
+ */
+static void k8_noop_call(struct amd64_pvt *pvt)
+{
+}
+
+/*
+ * Forward references for the low_ops functions
+ *
+ * Common functions
+ */
+static int get_scrub_rate(struct mem_ctl_info *mci, u32 *bw);
+
+/* K8 device specific functions */
+static int k8_early_channel_count(struct amd64_pvt *pct);
+static void k8_display_info(struct amd64_pvt *pvt);
+static int k8_set_scrub_rate(struct mem_ctl_info *mci, u32 *bw);
+static u64 k8_get_error_address(struct mem_ctl_info *mci,
+			struct amd64_error_info_regs *info);
+static void k8_read_dram_base_limit(struct amd64_pvt *pvt, int dram);
+static void k8_read_dctBase_dctMask(struct amd64_pvt *pvt);
+static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
+					struct amd64_error_info_regs *info,
+					u64 SystemAddr);
+static int k8_get_dram_hole_info(struct mem_ctl_info *mci,
+				u64 *base, u64 *offset, u64 *size);
+static void k8_decode_misc_regs(struct amd64_pvt *pvt);
+static enum dev_type k8_determine_dram_type(struct amd64_pvt *pvt, int row);
+static enum mem_type k8_determine_memory_type(struct amd64_pvt *pvt);
+static enum edac_type k8_determine_edac_cap(struct amd64_pvt *pvt);
+static int k8_dbam_map_to_pages(struct amd64_pvt *pvt, int dram_map);
+static int k8_probe_valid_hardware(struct amd64_pvt *pvt);
+static void k8_read_dbam_reg(struct amd64_pvt *pvt);
+
+/* F10 device specific functions */
+static int f10_early_channel_count(struct amd64_pvt *pvt);
+static void f10_setup(struct amd64_pvt *pvt);
+static void f10_teardown(struct amd64_pvt *pvt);
+static void f10_display_info(struct amd64_pvt *pvt);
+static int f10_set_scrub_rate(struct mem_ctl_info *mci, u32 *bw);
+static u64 f10_get_error_address(struct mem_ctl_info *mci,
+			struct amd64_error_info_regs *info);
+static void f10_read_dram_base_limit(struct amd64_pvt *pvt, int dram);
+static void f10_read_dram_ctl_register(struct amd64_pvt *pvt);
+static void f10_read_dctBase_dctMask(struct amd64_pvt *pvt);
+static void f10_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
+					struct amd64_error_info_regs *info,
+					u64 SystemAddr);
+static int f10_get_dram_hole_info(struct mem_ctl_info *mci,
+				u64 *base, u64 *offset, u64 *size);
+static void f10_decode_misc_regs(struct amd64_pvt *pvt);
+static enum dev_type f10_determine_dram_type(struct amd64_pvt *pvt, int row);
+static enum mem_type f10_determine_memory_type(struct amd64_pvt *pvt);
+static enum edac_type f10_determine_edac_cap(struct amd64_pvt *pvt);
+static int f10_dbam_map_to_pages(struct amd64_pvt *pvt, int dram_map);
+static int f10_probe_valid_hardware(struct amd64_pvt *pvt);
+static void f10_read_dbam_reg(struct amd64_pvt *pvt);
+
+/* F11 functions - FUTURE */
+
+/*
+ * There currently are 3 types type of MC devices for AMD
+ * Athlon/Opterons (as per PCI DEVICE_IDs):
+ *
+ * Family K8: That is the Athlon64 and Opteron CPUs. They all have the
+ * same PCI DEVICE ID, even though there is differences between
+ * the different Revisions (CG,D,E,F).
+ *
+ * Family F10: AMD Quad Core
+ *
+ * Family F11: (Unknown at this time, but is in pci_ids file)
+ *
+ */
+static struct amd64_family_type amd64_family_types[] = {
+	[K8_CPUS] = {
+		.ctl_name = "Athlon64/Opteron/Rev F",
+		.addr_f1_ctl = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
+		.misc_f3_ctl = PCI_DEVICE_ID_AMD_K8_NB_MISC,
+		.ops = {
+			.probe_valid_hardware = k8_probe_valid_hardware,
+			.early_channel_count = k8_early_channel_count,
+			.setup = k8_noop_call,
+			.teardown = k8_noop_call,
+			.display_info = k8_display_info,
+			.get_scrub_rate = get_scrub_rate,
+			.set_scrub_rate = k8_set_scrub_rate,
+			.get_error_address = k8_get_error_address,
+			.read_dram_base_limit = k8_read_dram_base_limit,
+			.read_dram_ctl_register = k8_noop_call,
+			.read_dctBase_dctMask = k8_read_dctBase_dctMask,
+			.map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
+			.get_dram_hole_info = k8_get_dram_hole_info,
+			.decode_misc_regs = k8_decode_misc_regs,
+			.determine_dram_type = k8_determine_dram_type,
+			.determine_memory_type = k8_determine_memory_type,
+			.determine_edac_cap = k8_determine_edac_cap,
+			.dbam_map_to_pages = k8_dbam_map_to_pages,
+			.read_dbam_reg = k8_read_dbam_reg
+		}
+	},
+	[F10_CPUS] = {
+		.ctl_name = "Family-F10h-Quad-Core",
+		.addr_f1_ctl = PCI_DEVICE_ID_AMD_F10_NB_ADDRMAP,
+		.misc_f3_ctl = PCI_DEVICE_ID_AMD_F10_NB_MISC,
+		.ops = {
+			.probe_valid_hardware = f10_probe_valid_hardware,
+			.early_channel_count = f10_early_channel_count,
+			.setup = f10_setup,
+			.teardown = f10_teardown,
+			.display_info = f10_display_info,
+			.get_scrub_rate = get_scrub_rate,
+			.set_scrub_rate = f10_set_scrub_rate,
+			.get_error_address = f10_get_error_address,
+			.read_dram_base_limit = f10_read_dram_base_limit,
+			.read_dram_ctl_register = f10_read_dram_ctl_register,
+			.read_dctBase_dctMask = f10_read_dctBase_dctMask,
+			.map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow,
+			.get_dram_hole_info = f10_get_dram_hole_info,
+			.decode_misc_regs = f10_decode_misc_regs,
+			.determine_dram_type = f10_determine_dram_type,
+			.determine_memory_type = f10_determine_memory_type,
+			.determine_edac_cap = f10_determine_edac_cap,
+			.dbam_map_to_pages = f10_dbam_map_to_pages,
+			.read_dbam_reg = f10_read_dbam_reg
+		}
+	},
+	[F11_CPUS] = {
+		.ctl_name = "Unknown-at-this-time",
+		.addr_f1_ctl = PCI_DEVICE_ID_AMD_F11_NB_ADDRMAP,
+		.misc_f3_ctl = PCI_DEVICE_ID_AMD_F11_NB_MISC,
+		.ops = {
+			.probe_valid_hardware = f10_probe_valid_hardware,
+			.early_channel_count = f10_early_channel_count,
+			.setup = f10_setup,
+			.teardown = f10_teardown,
+			.get_scrub_rate = get_scrub_rate,
+			.set_scrub_rate = f10_set_scrub_rate,
+			.get_error_address = f10_get_error_address,
+			.read_dram_base_limit = f10_read_dram_base_limit,
+			.read_dram_ctl_register = f10_read_dram_ctl_register,
+			.read_dctBase_dctMask = f10_read_dctBase_dctMask,
+			.map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow,
+			.get_dram_hole_info = f10_get_dram_hole_info,
+			.decode_misc_regs = f10_decode_misc_regs,
+			.determine_dram_type = f10_determine_dram_type,
+			.determine_memory_type = f10_determine_memory_type,
+			.determine_edac_cap = f10_determine_edac_cap,
+			.dbam_map_to_pages = f10_dbam_map_to_pages,
+			.read_dbam_reg = f10_read_dbam_reg
+		}
+	},
+};
+
+/* Accessors for amd64 family table */
+static inline const char *get_amd_family_name(int index)
+{
+	return amd64_family_types[index].ctl_name;
+}
+
+static inline struct low_ops *get_amd_family_ops(int index)
+{
+	return &amd64_family_types[index].ops;
+}
+
+/*
+ * Error injection control structure
+ */
+struct error_injection {
+	u32	section;
+	u32	word;
+	u32	bit_map;
+};
+
+/*
+ * Private to this module: data and register state
+ */
+struct amd64_pvt {
+	/* pci_device handles which we utilize */
+	struct pci_dev *addr_f1_ctl;
+	struct pci_dev *dram_f2_ctl;
+	struct pci_dev *misc_f3_ctl;
+
+	int mc_node_id;		/* MC index of this MC node */
+	int ext_model;		/* extended model value of this node */
+
+	struct low_ops	*ops;	/* pointer to per PCI Device ID func table */
+
+	int channel_count;	/* Count of 'channels' */
+
+	/* Raw registers */
+	u32 dcl0;		/* DRAM Configuration Low DCT0 reg */
+	u32 dcl1;		/* DRAM Configuration Low DCT1 reg */
+	u32 nbcap;		/* North Bridge Capabilities */
+	u32 nbcfg;		/* F10 North Bridge Configuration */
+	u32 ext_nbcfg;		/* Extended F10 North Bridge Configuration */
+
+	/* DRAM CS Base Address Registers
+	 *   F2x[1,0][5C:40]
+	 */
+	u32 dcsb0[CHIPSELECT_COUNT];    /* DRAM CS Base Registers */
+	u32 dcsb1[CHIPSELECT_COUNT];    /* DRAM CS Base Registers */
+
+	/* DRAM CS Mask Registers
+	 *   F2x[1,0][6C:60]
+	 */
+	u32 dcsm0[CHIPSELECT_COUNT];    /* DRAM CS Mask Registers */
+	u32 dcsm1[CHIPSELECT_COUNT];    /* DRAM CS Mask Registers */
+
+	/* Decoded parts of DRAM BASE and LIMIT Registers
+	 * F1x[78,70,68,60,58,50,48,40]
+	 */
+	u64 dram_base[DRAM_REG_COUNT];/* DRAM Base Reg */
+	u64 dram_limit[DRAM_REG_COUNT];/* DRAM Limit Reg */
+	u8 dram_IntlvSel[DRAM_REG_COUNT];
+	u8 dram_IntlvEn[DRAM_REG_COUNT];
+	u8 dram_DstNode[DRAM_REG_COUNT];
+	u8 dram_rw_en[DRAM_REG_COUNT];
+
+	/* The following fields are set at (load) run time, after Revision has
+	 * been determine, since the dctBase and dctMask registers vary
+	 * by CPU Revsion
+	 */
+	u32 dcsb_mask;		/* DCSB mask bits */
+	u32 dcsm_mask;		/* DCSM mask bits */
+	u32 num_dcsm;		/* Number of DCSM registers */
+	u32 dcs_mask_notused;	/* DCSM notused mask bits */
+	u32 dcs_shift;		/* DCSB and DCSM shift value */
+
+	/* On Rev E there are 8 DCSM registers,
+	 * On Rev F and later there are 4 DCSM registers.
+	 *
+	 * This field is set as a 0 (Rev E) or 1 (Rev F) to indicate
+	 * number of bits to shift the index for DCSM array look ups.
+	 */
+	u32 dcsm_shift_bit;
+
+	u32 dhar;		/* DRAM Hoist reg */
+	u32 dbam0;		/* DRAM Base Address Mapping reg for DCT0 */
+	u32 dbam1;		/* DRAM Base Address Mapping reg for DCT1 */
+
+	u64 top_mem;		/* top of memory below 4GB */
+	u64 top_mem2;		/* top of memory above 4GB */
+
+	/* F10 registers */
+	u32 dram_ctl_select_low;	/* DRAM Controller Select Low Reg */
+	u32 dram_ctl_select_high;	/* DRAM Controller Select High Reg */
+	int dram_dct_ganged_enable;	/* boolean for DctGangedEnable */
+	u32 online_spare;               /* On-Line spare Reg */
+
+	/* sysfs storage area: Temp storage for when input
+	 * is received from sysfs
+	 */
+	struct amd64_error_info_regs ctl_error_info;
+
+	/* Place to store error injection parameters prior to issue */
+	struct error_injection injection;
+
+	/* Save old hw registers' values before we modified them */
+	u32 nbctl_mcgctl_saved;		/* When true, following 2 are valid */
+	u32 old_nbctl;
+	u32 old_mcgctl;
+
+	/* Old value of the Enable Cf8 Ext Config reg */
+	u32 old_enable_cf8_extcfg;
+
+	/* MC Type Index value: socket F vs Family 10h */
+	u32 mc_type_index;
+
+	/* Mark if this MC instance has fully initialized (TRUE) */
+	u32 initialization_completed;
+};
+
+
+/*
+ * display_info functions for each of the DEVICE IDs we support
+ */
+static void k8_display_info(struct amd64_pvt *pvt)
+{
+	edac_printk(KERN_DEBUG, EDAC_MC, "%s detected\n",
+		(pvt->ext_model >= OPTERON_CPU_REV_F) ?
+			"Rev F or later" : "Rev E or earlier");
+}
+
+static void f10_display_info(struct amd64_pvt *pvt)
+{
+	edac_printk(KERN_DEBUG, EDAC_MC,
+		"F10 class processor detected\n");
+}
+
+/*
+ * DRAM Bank Address Mapping (DBAM) Register Shift Table:
+ * REV F and Later processors the DBAM register has a set of DIMM
+ * Address mapping values.
+ * Each physical DIMM has a size capacity. An index value (0->0xF) indicates
+ * what size it is. This table is the 'bit shift' value to shift a value of
+ * one (1) to the left. The result will be size in MBs of this physical
+ * DIMM.
+ * 128Mib is the minimum DIMM size in REV F and later CPUs, on upto 8 Gb,
+ * in a step function progression
+ */
+static u32 revf_quad_ddr2_shift[] =
+{
+	0,	/* 0000b NULL DIMM (128mb) */
+	28,	/* 0001b 256mb */
+	29,	/* 0010b 512mb */
+	29,	/* 0011b 512mb */
+	29,	/* 0100b 512mb */
+	30,	/* 0101b 1gb */
+	30,	/* 0110b 1gb */
+	31,	/* 0111b 2gb */
+	31,	/* 1000b 2gb */
+	32,	/* 1001b 4gb */
+	32,	/* 1010b 4gb */
+	33,	/* 1011b 8gb */
+	0,	/* 1100b future */
+	0,	/* 1101b future */
+	0,	/* 1110b future */
+	0	/* 1111b future */
+};
+
+/* Valid scrub rates for the K8 hardware memory scrubber. We map
+ *	the scrubbing bandwidth to a valid bit pattern. The 'set'
+ *	operation finds the 'matching- or higher value'.
+ *
+ *FIXME: Produce a better mapping/linearisation.
+ */
+
+# define SDRATE_EOD 0xFFFFFFFF
+
+static struct scrubrate {
+	u32 scrubval;		/* bit pattern for scrub rate */
+	u32 bandwidth;		/* bandwidth consumed (bytes/sec) */
+} scrubrates[] = {
+	{ 0x00, 0UL},		/* Scrubbing Off */
+	{ 0x16, 761UL},		/* Slowest Rate: Family 10h and later */
+	{ 0x15, 1523UL},	/* Slowest Rate: Family 0Fh and prior */
+	{ 0x14, 3051UL},
+	{ 0x13, 6101UL},
+	{ 0x12, 12213UL},
+	{ 0x11, 24427UL},
+	{ 0x10, 48854UL},
+	{ 0x0F, 97650UL},
+	{ 0x0E, 195300UL},
+	{ 0x0D, 390720UL},
+	{ 0x0C, 781440UL},
+	{ 0x0B, 1560975UL},
+	{ 0x0A, 3121951UL},
+	{ 0x09, 6274509UL},
+	{ 0x08, 12284069UL},
+	{ 0x07, 25000000UL},
+	{ 0x06, 50000000UL},
+	{ 0x05, 100000000UL},
+	{ 0x04, 200000000UL},
+	{ 0x03, 400000000UL},
+	{ 0x02, 800000000UL},
+	{ 0x01, 1600000000UL},
+	{ 0x00, SDRATE_EOD}	/* End Of Data */
+};
+
+/* For future CPU version, verify the following as new 'slow' rates appear.
+ * And modify the necessary skip values for the supported CPU
+ */
+#define K8_SCRUB_SKIP_LOW	1
+#define K8_SCRUB_SKIP_HIGH	1
+#define F10_SCRUB_SKIP_LOW	(sizeof(scrubrates) / sizeof(struct scrubrate))
+#define F10_SCRUB_SKIP_HIGH	(sizeof(scrubrates) / sizeof(struct scrubrate))
+#define F11_SCRUB_SKIP_LOW	(sizeof(scrubrates) / sizeof(struct scrubrate))
+#define F11_SCRUB_SKIP_HIGH	(sizeof(scrubrates) / sizeof(struct scrubrate))
+
+
+/* map_dbam_to_csrow_size
+ *
+ * Input (index) is the DBAM DIMM value (1 of 4) used as an index into a
+ * shift table (revf_quad_ddr2_shift) which starts at 128MB DIMM size.
+ *
+ * Index of 0 indicates an empty DIMM slot, as reported by Hardware on
+ * empty slots.
+ *
+ * Normalize to 128MB by subracting 27 bit shift
+ */
+static int map_dbam_to_csrow_size(int index)
+{
+	int mega_bytes = 0;
+
+	/* Check if index is within valid limits */
+	if (index > 0 && index <= DBAM_MAX_VALUE)
+		mega_bytes = ((128 << (revf_quad_ddr2_shift[index]-27)));
+
+	return mega_bytes;
+}
+
+/* Memory scrubber control interface. For the K8 memory scrubbing is
+ *	handled by hardware and can involve the cache and the dcache as
+ *	well as the main memory. This causes the "units" for the scrubbing
+ *	speed to vary from 64 byte blocks (sdram) over cache lines (cache)
+ *	to bits (dcache).
+ *
+ *	This is nasty, so we will use bandwidth in bytes/sec  for the setting.
+ *
+ *	Currently, we only do scrubbing of sdram - the caches are assumed
+ *	to be excercised always by running code and if the scrubber is done
+ *	in software on other archs, we might not have access to the
+ *	caches directly.
+ */
+
+/*
+ * search_set_scrub_rate
+ *
+ *	scan the scrub rate mapping table for a close or matching
+ *	bandwidth value to issue. If requested is too big, then
+ *	use last maxium value found.
+ */
+static int search_set_scrub_rate(struct pci_dev *ctl, u32 new_bw,
+				int skip_low, int skip_high)
+{
+	u32 scrubval;
+	int i;
+
+	/* map the configured rate (new_bw) to a value specific to
+	 * the AMD64 memory controller and apply to register.
+	 * Search for the first bandwidth entry that is greater or equal
+	 * than the setting requested and program that.
+	 * If at last entry, accept that as the maximum. This saves
+	 * the user from determing maximum empirically
+	 */
+	for (i = 0;; i++) {
+
+		/* skip entries within the skip_window, move to next */
+		if ((i >= skip_low) && (i <= skip_high))
+			continue;
+
+		/* If we found the proper entry, use it */
+		if (scrubrates[i].bandwidth >= new_bw)
+			break;
+
+		/* If this is the LAST entry in table, use it */
+		if (scrubrates[i+1].bandwidth == SDRATE_EOD)
+			break;
+	}
+
+	scrubval = scrubrates[i].scrubval;
+	pci_write_bits32(ctl, K8_SCRCTRL, scrubval, 0x001F);
+
+	return 0;
+}
+
+/* K8 Set function */
+static int k8_set_scrub_rate(struct mem_ctl_info *mci, u32 *band_width)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	return search_set_scrub_rate(pvt->misc_f3_ctl, *band_width,
+				K8_SCRUB_SKIP_LOW, K8_SCRUB_SKIP_HIGH);
+}
+
+/* F10 Set Scrub Function */
+static int f10_set_scrub_rate(struct mem_ctl_info *mci, u32 *band_width)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	return search_set_scrub_rate(pvt->misc_f3_ctl, *band_width,
+				F10_SCRUB_SKIP_LOW, F10_SCRUB_SKIP_HIGH);
+}
+
+/*
+ * set_sdram_scrub_rate
+ *
+ *	Generic Set scrub rate function
+ */
+static int set_sdram_scrub_rate(struct mem_ctl_info *mci,
+				u32 *band_width)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	return (*(pvt->ops->set_scrub_rate))(mci, band_width);
+}
+
+
+/* get_scrub_rate
+ *
+ *	K8 GET Scrub rate function
+ *	F10 Get Scrub Rate
+ *	F11 Get Scrub Rate
+ */
+static int get_scrub_rate(struct mem_ctl_info *mci, u32 *bw)
+{
+	struct amd64_pvt *pvt;
+	u32 scrubval = 0;
+	int status = -1;
+	int i;
+	int err;
+
+	pvt = mci->pvt_info;
+
+	/* find the bandwidth matching the memory scrubber configuration
+	 * and return that value via *bw.
+	 */
+	err = pci_read_config_dword(pvt->misc_f3_ctl, K8_SCRCTRL, &scrubval);
+	if (err != 0)
+		debugf0("%s() Reading K8_SCRCTRL failed\n", __func__);
+
+	scrubval = scrubval & 0x001F;
+
+	edac_printk(KERN_DEBUG, EDAC_MC,
+		    "pci-read, sdram scrub control value: %d \n", scrubval);
+
+	for (i = 0; scrubrates[i].bandwidth != SDRATE_EOD; i++) {
+
+		if (scrubrates[i].scrubval == scrubval) {
+			*bw = scrubrates[i].bandwidth;
+			status = 0;
+			break;
+		}
+	}
+
+	/* the bit pattern is invalid - we might fix it
+	   by applying the slowest scrub rate as this is
+	   closest to the valid value, but we do not!
+	 */
+	if (scrubrates[i].bandwidth == SDRATE_EOD) {
+		edac_printk(KERN_WARNING, EDAC_MC,
+			    "Invalid sdram scrub control value: %d\n",
+			    scrubval);
+		status = -1;
+	}
+	return status;
+}
+
+/*
+ * get_sdram_scrub_rate
+ *
+ *	get the current value of the scrub rate register, then map
+ *	that value to a bytes/second value and return it
+ */
+static int get_sdram_scrub_rate(struct mem_ctl_info *mci, u32 *bw)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	return (*(pvt->ops->get_scrub_rate))(mci, bw);
+}
+
+/*
+ * Iterate over devices to find AMD related devices
+ */
+static struct pci_dev *pci_get_related_function(unsigned int vendor,
+						unsigned int device,
+						struct pci_dev *related)
+{
+	struct pci_dev *dev;
+
+	dev = NULL;
+
+	dev = pci_get_device(vendor, device, dev);
+	while (dev != NULL) {
+		if ((dev->bus->number == related->bus->number) &&
+		    (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
+			break;
+		dev = pci_get_device(vendor, device, dev);
+	}
+
+	return dev;
+}
+
+/* stolen from msr.c - the calls in msr.c could be exported */
+struct msr_command {
+	int cpu;
+	int err;
+	u32 reg;
+	u32 data[2];
+};
+
+static void smp_wrmsr(void *cmd_block)
+{
+	struct msr_command *cmd = cmd_block;
+	wrmsr(cmd->reg, cmd->data[0], cmd->data[1]);
+}
+
+static void smp_rdmsr(void *cmd_block)
+{
+	struct msr_command *cmd = cmd_block;
+	rdmsr(cmd->reg, cmd->data[0], cmd->data[1]);
+}
+
+static void do_wrmsr(int cpu, u32 reg, u32 eax, u32 edx)
+{
+	struct msr_command cmd;
+
+	cmd.cpu = raw_smp_processor_id();
+	cmd.reg = reg;
+	cmd.data[0] = eax;
+	cmd.data[1] = edx;
+	on_each_cpu(smp_wrmsr, &cmd, 1, 1);
+}
+
+static void do_rdmsr(int cpu, u32 reg, u32 *eax, u32 *edx)
+{
+	struct msr_command cmd;
+
+	cmd.cpu = raw_smp_processor_id();
+	cmd.reg = reg;
+	on_each_cpu(smp_rdmsr, &cmd, 1, 1);
+	*eax = cmd.data[0];
+	*edx = cmd.data[1];
+}
+
+/*
+ * syndrome mapping table for X4 ChipKill devices
+ *
+ * The comment in each row is the token (nibble) number that is in error.
+ * The least significant nibble of the syndrome is the mask for the bits
+ * that are in error (need to be toggled) for the particular nibble.
+ *
+ * Each row contains 16 entries.
+ * The first entry (0th) is the channel number for that row of syndromes.
+ * The remaining 15 entries are the syndromes for the respective Error
+ * bit mask index.
+ *
+ * 1st index entry is 0x0001 mask, indicating that the rightmost bit is the
+ * bit in error.
+ * The 2nd index entry is 0x0010 that the second bit is damaged.
+ * The 3rd index entry is 0x0011 indicating that the rightmost 2 bits
+ * are damaged.
+ * Thus so on until index 15, 0x1111, whose entry has the syndrome
+ * indicating that all 4 bits are damaged.
+ *
+ * A search is performed on this table looking for a given syndrome.
+ *
+ * See the AMD documentation for ECC syndromes. This x4 table is valid
+ * across all the versions of the AMD64 processors.
+ *
+ * A fast lookup is to use the LAST four bits of the 16-bit syndrome as a
+ * COLUMN index, then search all ROWS of that column, looking for a match
+ * with the input syndrome. The ROW value will be the token number.
+ *
+ * The 0'th entry on that row, can be returned as the CHANNEL (0 or 1) of this
+ * error.
+ */
+#define NUMBER_X4_ROWS  36
+static const unsigned short x4_chipkill_syndromes[NUMBER_X4_ROWS][16] = {
+	/* Channel 0 syndromes */
+	{/*0*/  0, 0xe821, 0x7c32, 0x9413, 0xbb44, 0x5365, 0xc776, 0x2f57,
+	   0xdd88, 0x35a9, 0xa1ba, 0x499b, 0x66cc, 0x8eed, 0x1afe, 0xf2df },
+	{/*1*/  0, 0x5d31, 0xa612, 0xfb23, 0x9584, 0xc8b5, 0x3396, 0x6ea7,
+	   0xeac8, 0xb7f9, 0x4cda, 0x11eb, 0x7f4c, 0x227d, 0xd95e, 0x846f },
+	{/*2*/  0, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
+	   0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f },
+	{/*3*/  0, 0x2021, 0x3032, 0x1013, 0x4044, 0x6065, 0x7076, 0x5057,
+	   0x8088, 0xa0a9, 0xb0ba, 0x909b, 0xc0cc, 0xe0ed, 0xf0fe, 0xd0df },
+	{/*4*/  0, 0x5041, 0xa082, 0xf0c3, 0x9054, 0xc015, 0x30d6, 0x6097,
+	   0xe0a8, 0xb0e9, 0x402a, 0x106b, 0x70fc, 0x20bd, 0xd07e, 0x803f },
+	{/*5*/  0, 0xbe21, 0xd732, 0x6913, 0x2144, 0x9f65, 0xf676, 0x4857,
+	   0x3288, 0x8ca9, 0xe5ba, 0x5b9b, 0x13cc, 0xaded, 0xc4fe, 0x7adf },
+	{/*6*/  0, 0x4951, 0x8ea2, 0xc7f3, 0x5394, 0x1ac5, 0xdd36, 0x9467,
+	   0xa1e8, 0xe8b9, 0x2f4a, 0x661b, 0xf27c, 0xbb2d, 0x7cde, 0x358f },
+	{/*7*/  0, 0x74e1, 0x9872, 0xec93, 0xd6b4, 0xa255, 0x4ec6, 0x3a27,
+	   0x6bd8, 0x1f39, 0xf3aa, 0x874b, 0xbd6c, 0xc98d, 0x251e, 0x51ff },
+	{/*8*/  0, 0x15c1, 0x2a42, 0x3f83, 0xcef4, 0xdb35, 0xe4b6, 0xf177,
+	   0x4758, 0x5299, 0x6d1a, 0x78db, 0x89ac, 0x9c6d, 0xa3ee, 0xb62f },
+	{/*9*/  0, 0x3d01, 0x1602, 0x2b03, 0x8504, 0xb805, 0x9306, 0xae07,
+	   0xca08, 0xf709, 0xdc0a, 0xe10b, 0x4f0c, 0x720d, 0x590e, 0x640f },
+	{/*a*/  0, 0x9801, 0xec02, 0x7403, 0x6b04, 0xf305, 0x8706, 0x1f07,
+	   0xbd08, 0x2509, 0x510a, 0xc90b, 0xd60c, 0x4e0d, 0x3a0e, 0xa20f },
+	{/*b*/  0, 0xd131, 0x6212, 0xb323, 0x3884, 0xe9b5, 0x5a96, 0x8ba7,
+	   0x1cc8, 0xcdf9, 0x7eda, 0xafeb, 0x244c, 0xf57d, 0x465e, 0x976f },
+	{/*c*/  0, 0xe1d1, 0x7262, 0x93b3, 0xb834, 0x59e5, 0xca56, 0x2b87,
+	   0xdc18, 0x3dc9, 0xae7a, 0x4fab, 0x542c, 0x85fd, 0x164e, 0xf79f },
+	{/*d*/  0, 0x6051, 0xb0a2, 0xd0f3, 0x1094, 0x70c5, 0xa036, 0xc067,
+	   0x20e8, 0x40b9, 0x904a, 0x601b, 0x307c, 0x502d, 0x80de, 0xe08f },
+	{/*e*/  0, 0xa4c1, 0xf842, 0x5c83, 0xe6f4, 0x4235, 0x1eb6, 0xba77,
+	   0x7b58, 0xdf99, 0x831a, 0x27db, 0x9dac, 0x396d, 0x65ee, 0xc12f },
+	{/*f*/  0, 0x11c1, 0x2242, 0x3383, 0xc8f4, 0xd935, 0xeab6, 0xfb77,
+	   0x4c58, 0x5d99, 0x6e1a, 0x7fdb, 0x84ac, 0x956d, 0xa6ee, 0xb72f },
+
+	/* Channel 1 syndromes */
+	{/*10*/ 1, 0x45d1, 0x8a62, 0xcfb3, 0x5e34, 0x1be5, 0xd456, 0x9187,
+	   0xa718, 0xe2c9, 0x2d7a, 0x68ab, 0xf92c, 0xbcfd, 0x734e, 0x369f },
+	{/*11*/ 1, 0x63e1, 0xb172, 0xd293, 0x14b4, 0x7755, 0xa5c6, 0xc627,
+	   0x28d8, 0x4b39, 0x99aa, 0xfa4b, 0x3c6c, 0x5f8d, 0x8d1e, 0xeeff },
+	{/*12*/ 1, 0xb741, 0xd982, 0x6ec3, 0x2254, 0x9515, 0xfbd6, 0x4c97,
+	   0x33a8, 0x84e9, 0xea2a, 0x5d6b, 0x11fc, 0xa6bd, 0xc87e, 0x7f3f },
+	{/*13*/ 1, 0xdd41, 0x6682, 0xbbc3, 0x3554, 0xe815, 0x53d6, 0xce97,
+	   0x1aa8, 0xc7e9, 0x7c2a, 0xa1fb, 0x2ffc, 0xf2bd, 0x497e, 0x943f },
+	{/*14*/ 1, 0x2bd1, 0x3d62, 0x16b3, 0x4f34, 0x64e5, 0x7256, 0x5987,
+	   0x8518, 0xaec9, 0xb87a, 0x93ab, 0xca2c, 0xe1fd, 0xf74e, 0xdc9f },
+	{/*15*/ 1, 0x83c1, 0xc142, 0x4283, 0xa4f4, 0x2735, 0x65b6, 0xe677,
+	   0xf858, 0x7b99, 0x391a, 0xbadb, 0x5cac, 0xdf6d, 0x9dee, 0x1e2f },
+	{/*16*/ 1, 0x8fd1, 0xc562, 0x4ab3, 0xa934, 0x26e5, 0x6c56, 0xe387,
+	   0xfe18, 0x71c9, 0x3b7a, 0xb4ab, 0x572c, 0xd8fd, 0x924e, 0x1d9f },
+	{/*17*/ 1, 0x4791, 0x89e2, 0xce73, 0x5264, 0x15f5, 0xdb86, 0x9c17,
+	   0xa3b8, 0xe429, 0x2a5a, 0x6dcb, 0xf1dc, 0xb64d, 0x783e, 0x3faf },
+	{/*18*/ 1, 0x5781, 0xa9c2, 0xfe43, 0x92a4, 0xc525, 0x3b66, 0x6ce7,
+	   0xe3f8, 0xb479, 0x4a3a, 0x1dbb, 0x715c, 0x26dd, 0xd89e, 0x8f1f },
+	{/*19*/ 1, 0xbf41, 0xd582, 0x6ac3, 0x2954, 0x9615, 0xfcd6, 0x4397,
+	   0x3ea8, 0x81e9, 0xeb2a, 0x546b, 0x17fc, 0xa8bd, 0xc27e, 0x7d3f },
+	{/*1a*/ 1, 0x9891, 0xe1e2, 0x7273, 0x6464, 0xf7f5, 0x8586, 0x1617,
+	   0xb8b8, 0x2b29, 0x595a, 0xcacb, 0xdcdc, 0x4f4d, 0x3d3e, 0xaeaf },
+	{/*1b*/ 1, 0xcce1, 0x4472, 0x8893, 0xfdb4, 0x3f55, 0xb9c6, 0x7527,
+	   0x56d8, 0x9a39, 0x12aa, 0xde4b, 0xab6c, 0x678d, 0xef1e, 0x23ff },
+	{/*1c*/ 1, 0xa761, 0xf9b2, 0x5ed3, 0xe214, 0x4575, 0x1ba6, 0xbcc7,
+	   0x7328, 0xd449, 0x8a9a, 0x2dfb, 0x913c, 0x365d, 0x688e, 0xcfef },
+	{/*1d*/ 1, 0xff61, 0x55b2, 0xaad3, 0x7914, 0x8675, 0x2ca6, 0xd3c7,
+	   0x9e28, 0x6149, 0xcb9a, 0x34fb, 0xe73c, 0x185d, 0xb28e, 0x4def },
+	{/*1e*/ 1, 0x5451, 0xa8a2, 0xfcf3, 0x9694, 0xc2c5, 0x3e36, 0x6a67,
+	   0xebe8, 0xbfb9, 0x434a, 0x171b, 0x7d7c, 0x292d, 0xd5de, 0x818f },
+	{/*1f*/ 1, 0x6fc1, 0xb542, 0xda83, 0x19f4, 0x7635, 0xacb6, 0xc377,
+	   0x2e58, 0x4199, 0x9b1a, 0xf4db, 0x37ac, 0x586d, 0x82ee, 0xed2f },
+
+	/* ECC bits are also in the set of tokens and they too can go bad
+	 * first 2 cover channel 0, while the second 2 cover channel 1
+	 */
+	{/*20*/ 0, 0xbe01, 0xd702, 0x6903, 0x2104, 0x9f05, 0xf606, 0x4807,
+	   0x3208, 0x8c09, 0xe50a, 0x5b0b, 0x130c, 0xad0d, 0xc40e, 0x7a0f },
+	{/*21*/ 0, 0x4101, 0x8202, 0xc303, 0x5804, 0x1905, 0xda06, 0x9b07,
+	   0xac08, 0xed09, 0x2e0a, 0x6f0b, 0x640c, 0xb50d, 0x760e, 0x370f },
+	{/*22*/ 1, 0xc441, 0x4882, 0x8cc3, 0xf654, 0x3215, 0xbed6, 0x7a97,
+	   0x5ba8, 0x9fe9, 0x132a, 0xd76b, 0xadfc, 0x69bd, 0xe57e, 0x213f },
+	{/*23*/ 1, 0x7621, 0x9b32, 0xed13, 0xda44, 0xac65, 0x4176, 0x3757,
+	   0x6f88, 0x19a9, 0xf4ba, 0x829b, 0xb5cc, 0xc3ed, 0x2efe, 0x58df }
+};
+#define NUMBER_X4_ROWS  36
+
+/*
+ * get_channel_from_x4_syndrome
+ *
+ *	given the syndrome argument, scan each of the channel tables
+ *	looking for a syndrome match. Depending on which table it is
+ *	found, return the channel number
+ */
+static int get_channel_from_x4_syndrome(unsigned short syndrome)
+{
+	int row;
+	int column;
+
+	debugf0("%s()\n", __func__);
+
+	/* Determine column to scan */
+	column = syndrome & 0xF;
+
+	/* Scan all rows, looking for syndrome, or end of table */
+	for (row = 0; row < NUMBER_X4_ROWS; row++) {
+		if (x4_chipkill_syndromes[row][column] == syndrome)
+			return x4_chipkill_syndromes[row][0];
+	}
+
+	debugf0("%s(): syndrome(%x) not found\n", __func__, syndrome);
+	return -1;
+}
+
+/*
+ * syndrome mapping table for X8 ChipKill devices
+ *
+ *  Each row is a list of 19 syndromes
+ *  The index value into the row is the number of the Memory
+ *  'Symbol In Error' value.
+ *
+ *  entry 0 is the 12h symbol
+ *  entry 1 is the 11h symbol
+ *  and so on til
+ *  entry 18 is the 0h symbol.
+ *
+ *  (This is how AMD produced the table)
+ *
+ *  For a fast lookup, use the lower byte of the syndrome as the row entry
+ *  then scan that row looking for the syndrome match
+ *  If not found, then use the upper byte of the syndrom as the row entry
+ *  then scan that row looking for the syndrome match.
+ *
+ *  If not found, then the corrected error only identifies the 2 physical
+ *  DIMMs.
+ *  If found then the 'Symbol In Error' index can be used to
+ *  tell which DIMM contains the error (and thus the channel number)
+ *
+ *  Symbols 00h-07h map to data bits 0-63 (channel 0)
+ *  Symbols 08h-0Fh map to data bits 62-127 (channel 1)
+ */
+#define NUMBER_X8_ROWS	256
+#define NUMBER_X8_COLS	19
+
+static const unsigned short
+		x8_chipkill_syndromes[NUMBER_X8_ROWS][NUMBER_X8_COLS] = {
+{/*00*/ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
+0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000
+},
+{/*01*/ 0x0100, 0x0001, 0x0101, 0x01B8, 0x015C, 0x012E, 0x01C6, 0x0163, 0x01FD,
+0x0189, 0x019D, 0xB801, 0x5C01, 0x2E01, 0xC601, 0x6301, 0xFD01, 0x8901, 0x9D01
+},
+{/*02*/ 0x0200, 0x0002, 0x0202, 0x0201, 0x02B8, 0x025C, 0x02FD, 0x02C6, 0x028B,
+0x0263, 0x024B, 0x0102, 0xB802, 0x5C02, 0xFD02, 0xC602, 0x8B02, 0x6302, 0x4B02
+},
+{/*03*/ 0x0300, 0x0003, 0x0303, 0x03B9, 0x03E4, 0x0372, 0x033B, 0x03A5, 0x0376,
+0x03EA, 0x03D6, 0xB903, 0xE403, 0x7203, 0x3B03, 0xA503, 0x7603, 0xEA03, 0xD603
+},
+{/*04*/ 0x0400, 0x0004, 0x0404, 0x0402, 0x0401, 0x04B8, 0x048B, 0x04FD, 0x0467,
+0x04C6, 0x0496, 0x0204, 0x0104, 0xB804, 0x8B04, 0xFD04, 0x6704, 0xC604, 0x9604
+},
+{/*05*/ 0x0500, 0x0005, 0x0505, 0x05BA, 0x055D, 0x0596, 0x054D, 0x059E, 0x059A,
+0x054F, 0x050B, 0xBA05, 0x5D05, 0x9605, 0x4D05, 0x9E05, 0x9A05, 0x4F05, 0x0B05
+},
+{/*06*/ 0x0600, 0x0006, 0x0606, 0x0603, 0x06B9, 0x06E4, 0x0676, 0x063B, 0x06EC,
+0x06A5, 0x06DD, 0x0306, 0xB906, 0xE406, 0x7606, 0x3B06, 0xEC06, 0xA506, 0xDD06
+},
+{/*07*/ 0x0700, 0x0007, 0x0707, 0x07BB, 0x07E5, 0x07CA, 0x07B0, 0x0758, 0x0711,
+0x072C, 0x0740, 0xBB07, 0xE507, 0xCA07, 0xB007, 0x5807, 0x1107, 0x2C07, 0x4007
+},
+{/*08*/ 0x0800, 0x0008, 0x0808, 0x0804, 0x0802, 0x0801, 0x0867, 0x088B, 0x08CE,
+0x08FD, 0x085D, 0x0408, 0x0208, 0x0108, 0x6708, 0x8B08, 0xCE08, 0xFD08, 0x5D08
+},
+{/*09*/ 0x0900, 0x0009, 0x0909, 0x09BC, 0x095E, 0x092F, 0x09A1, 0x09E8, 0x0933,
+0x0974, 0x09C0, 0xBC09, 0x5E09, 0x2F09, 0xA109, 0xE809, 0x3309, 0x7409, 0xC009
+},
+{/*0A*/ 0x0A00, 0x000A, 0x0A0A, 0x0A05, 0x0ABA, 0x0A5D, 0x0A9A, 0x0A4D, 0x0A45,
+0x0A9E, 0x0A16, 0x050A, 0xBA0A, 0x5D0A, 0x9A0A, 0x4D0A, 0x450A, 0x9E0A, 0x160A
+},
+{/*0B*/ 0x0B00, 0x000B, 0x0B0B, 0x0BBD, 0x0BE6, 0x0B73, 0x0B5C, 0x0B2E, 0x0BB8,
+0x0B17, 0x0B8B, 0xBD0B, 0xE60B, 0x730B, 0x5C0B, 0x2E0B, 0xB80B, 0x170B, 0x8B0B
+},
+{/*0C*/ 0x0C00, 0x000C, 0x0C0C, 0x0C06, 0x0C03, 0x0CB9, 0x0CEC, 0x0C76, 0x0CA9,
+0x0C3B, 0x0CCB, 0x060C, 0x030C, 0xB90C, 0xEC0C, 0x760C, 0xA90C, 0x3B0C, 0xCB0C
+},
+{/*0D*/ 0x0D00, 0x000D, 0x0D0D, 0x0DBE, 0x0D5F, 0x0D97, 0x0D2A, 0x0D15, 0x0D54,
+0x0DB2, 0x0D56, 0xBE0D, 0x5F0D, 0x970D, 0x2A0D, 0x150D, 0x540D, 0xB20D, 0x560D
+},
+{/*0E*/ 0x0E00, 0x000E, 0x0E0E, 0x0E07, 0x0EBB, 0x0EE5, 0x0E11, 0x0EB0, 0x0E22,
+0x0E58, 0x0E80, 0x070E, 0xBB0E, 0xE50E, 0x110E, 0xB00E, 0x220E, 0x580E, 0x800E
+},
+{/*0F*/ 0x0F00, 0x000F, 0x0F0F, 0x0FBF, 0x0FE7, 0x0FCB, 0x0FD7, 0x0FD3, 0x0FDF,
+0x0FD1, 0x0F1D, 0xBF0F, 0xE70F, 0xCB0F, 0xD70F, 0xD30F, 0xDF0F, 0xD10F, 0x1D0F
+},
+{/*10*/ 0x1000, 0x0010, 0x1010, 0x1008, 0x1004, 0x1002, 0x10CE, 0x1067, 0x10ED,
+0x108B, 0x10BA, 0x0810, 0x0410, 0x0210, 0xCE10, 0x6710, 0xED10, 0x8B10, 0xBA10
+},
+{/*11*/ 0x1100, 0x0011, 0x1111, 0x11B0, 0x1158, 0x112C, 0x1108, 0x1104, 0x1110,
+0x1102, 0x1127, 0xB011, 0x5811, 0x2C11, 0x0811, 0x0411, 0x1011, 0x0211, 0x2711
+},
+{/*12*/ 0x1200, 0x0012, 0x1212, 0x1209, 0x12BC, 0x125E, 0x1233, 0x12A1, 0x1266,
+0x12E8, 0x12F1, 0x0912, 0xBC12, 0x5E12, 0x3312, 0xA112, 0x6612, 0xE812, 0xF112
+},
+{/*13*/ 0x1300, 0x0013, 0x1313, 0x13B1, 0x13E0, 0x1370, 0x13F5, 0x13C2, 0x139B,
+0x1361, 0x136C, 0xB113, 0xE013, 0x7013, 0xF513, 0xC213, 0x9B13, 0x6113, 0x6C13
+},
+{/*14*/ 0x1400, 0x0014, 0x1414, 0x140A, 0x1405, 0x14BA, 0x1445, 0x149A, 0x148A,
+0x144D, 0x142C, 0x0A14, 0x0514, 0xBA14, 0x4514, 0x9A14, 0x8A14, 0x4D14, 0x2C14
+},
+{/*15*/ 0x1500, 0x0015, 0x1515, 0x15B2, 0x1559, 0x1594, 0x1583, 0x15F9, 0x1577,
+0x15C4, 0x15B1, 0xB215, 0x5915, 0x9415, 0x8315, 0xF915, 0x7715, 0xC415, 0xB115
+},
+{/*16*/ 0x1600, 0x0016, 0x1616, 0x160B, 0x16BD, 0x16E6, 0x16B8, 0x165C, 0x1601,
+0x162E, 0x1667, 0x0B16, 0xBD16, 0xE616, 0xB816, 0x5C16, 0x0116, 0x2E16, 0x6716
+},
+{/*17*/ 0x1700, 0x0017, 0x1717, 0x17B3, 0x17E1, 0x17C8, 0x177E, 0x173F, 0x17FC,
+0x17A7, 0x17FA, 0xB317, 0xE117, 0xC817, 0x7E17, 0x3F17, 0xFC17, 0xA717, 0xFA17
+},
+{/*18*/ 0x1800, 0x0018, 0x1818, 0x180C, 0x1806, 0x1803, 0x18A9, 0x18EC, 0x1823,
+0x1876, 0x18E7, 0x0C18, 0x0618, 0x0318, 0xA918, 0xEC18, 0x2318, 0x7618, 0xE718
+},
+{/*19*/ 0x1900, 0x0019, 0x1919, 0x19B4, 0x195A, 0x192D, 0x196F, 0x198F, 0x19DE,
+0x19FF, 0x197A, 0xB419, 0x5A19, 0x2D19, 0x6F19, 0x8F19, 0xDE19, 0xFF19, 0x7A19
+},
+{/*1A*/ 0x1A00, 0x001A, 0x1A1A, 0x1A0D, 0x1ABE, 0x1A5F, 0x1A54, 0x1A2A, 0x1AA8,
+0x1A15, 0x1AAC, 0x0D1A, 0xBE1A, 0x5F1A, 0x541A, 0x2A1A, 0xA81A, 0x151A, 0xAC1A
+},
+{/*1B*/ 0x1B00, 0x001B, 0x1B1B, 0x1BB5, 0x1BE2, 0x1B71, 0x1B92, 0x1B49, 0x1B55,
+0x1B9C, 0x1B31, 0xB51B, 0xE21B, 0x711B, 0x921B, 0x491B, 0x551B, 0x9C1B, 0x311B
+},
+{/*1C*/ 0x1C00, 0x001C, 0x1C1C, 0x1C0E, 0x1C07, 0x1CBB, 0x1C22, 0x1C11, 0x1C44,
+0x1CB0, 0x1C71, 0x0E1C, 0x071C, 0xBB1C, 0x221C, 0x111C, 0x441C, 0xB01C, 0x711C
+},
+{/*1D*/ 0x1D00, 0x001D, 0x1D1D, 0x1DB6, 0x1D5B, 0x1D95, 0x1DE4, 0x1D72, 0x1DB9,
+0x1D39, 0x1DEC, 0xB61D, 0x5B1D, 0x951D, 0xE41D, 0x721D, 0xB91D, 0x391D, 0xEC1D
+},
+{/*1E*/ 0x1E00, 0x001E, 0x1E1E, 0x1E0F, 0x1EBF, 0x1EE7, 0x1EDF, 0x1ED7, 0x1ECF,
+0x1ED3, 0x1E3A, 0x0F1E, 0xBF1E, 0xE71E, 0xDF1E, 0xD71E, 0xCF1E, 0xD31E, 0x3A1E
+},
+{/*1F*/ 0x1F00, 0x001F, 0x1F1F, 0x1FB7, 0x1FE3, 0x1FC9, 0x1F19, 0x1FB4, 0x1F32,
+0x1F5A, 0x1FA7, 0xB71F, 0xE31F, 0xC91F, 0x191F, 0xB41F, 0x321F, 0x5A1F, 0xA71F
+},
+{/*20*/ 0x2000, 0x0020, 0x2020, 0x2010, 0x2008, 0x2004, 0x20ED, 0x20CE, 0x20AB,
+0x2067, 0x2005, 0x1020, 0x0820, 0x0420, 0xED20, 0xCE20, 0xAB20, 0x6720, 0x0520
+},
+{/*21*/ 0x2100, 0x0021, 0x2121, 0x21A8, 0x2154, 0x212A, 0x212B, 0x21AD, 0x2156,
+0x21EE, 0x2198, 0xA821, 0x5421, 0x2A21, 0x2B21, 0xAD21, 0x5621, 0xEE21, 0x9821
+},
+{/*22*/ 0x2200, 0x0022, 0x2222, 0x2211, 0x22B0, 0x2258, 0x2210, 0x2208, 0x2220,
+0x2204, 0x224E, 0x1122, 0xB022, 0x5822, 0x1022, 0x0822, 0x2022, 0x0422, 0x4E22
+},
+{/*23*/ 0x2300, 0x0023, 0x2323, 0x23A9, 0x23EC, 0x2376, 0x23D6, 0x236B, 0x23DD,
+0x238D, 0x23D3, 0xA923, 0xEC23, 0x7623, 0xD623, 0x6B23, 0xDD23, 0x8D23, 0xD323
+},
+{/*24*/ 0x2400, 0x0024, 0x2424, 0x2412, 0x2409, 0x24BC, 0x2466, 0x2433, 0x24CC,
+0x24A1, 0x2493, 0x1224, 0x0924, 0xBC24, 0x6624, 0x3324, 0xCC24, 0xA124, 0x9324
+},
+{/*25*/ 0x2500, 0x0025, 0x2525, 0x25AA, 0x2555, 0x2592, 0x25A0, 0x2550, 0x2531,
+0x2528, 0x250E, 0xAA25, 0x5525, 0x9225, 0xA025, 0x5025, 0x3125, 0x2825, 0x0E25
+},
+{/*26*/ 0x2600, 0x0026, 0x2626, 0x2613, 0x26B1, 0x26E0, 0x269B, 0x26F5, 0x2647,
+0x26C2, 0x26D8, 0x1326, 0xB126, 0xE026, 0x9B26, 0xF526, 0x4726, 0xC226, 0xD826
+},
+{/*27*/ 0x2700, 0x0027, 0x2727, 0x27AB, 0x27ED, 0x27CE, 0x275D, 0x2796, 0x27BA,
+0x274B, 0x2745, 0xAB27, 0xED27, 0xCE27, 0x5D27, 0x9627, 0xBA27, 0x4B27, 0x4527
+},
+{/*28*/ 0x2800, 0x0028, 0x2828, 0x2814, 0x280A, 0x2805, 0x288A, 0x2845, 0x2865,
+0x289A, 0x2858, 0x1428, 0x0A28, 0x0528, 0x8A28, 0x4528, 0x6528, 0x9A28, 0x5828
+},
+{/*29*/ 0x2900, 0x0029, 0x2929, 0x29AC, 0x2956, 0x292B, 0x294C, 0x2926, 0x2998,
+0x2913, 0x29C5, 0xAC29, 0x5629, 0x2B29, 0x4C29, 0x2629, 0x9829, 0x1329, 0xC529
+},
+{/*2A*/ 0x2A00, 0x002A, 0x2A2A, 0x2A15, 0x2AB2, 0x2A59, 0x2A77, 0x2A83, 0x2AEE,
+0x2AF9, 0x2A13, 0x152A, 0xB22A, 0x592A, 0x772A, 0x832A, 0xEE2A, 0xF92A, 0x132A
+},
+{/*2B*/ 0x2B00, 0x002B, 0x2B2B, 0x2BAD, 0x2BEE, 0x2B77, 0x2BB1, 0x2BE0, 0x2B13,
+0x2B70, 0x2B8E, 0xAD2B, 0xEE2B, 0x772B, 0xB12B, 0xE02B, 0x132B, 0x702B, 0x8E2B
+},
+{/*2C*/ 0x2C00, 0x002C, 0x2C2C, 0x2C16, 0x2C0B, 0x2CBD, 0x2C01, 0x2CB8, 0x2C02,
+0x2C5C, 0x2CCE, 0x162C, 0x0B2C, 0xBD2C, 0x012C, 0xB82C, 0x022C, 0x5C2C, 0xCE2C
+},
+{/*2D*/ 0x2D00, 0x002D, 0x2D2D, 0x2DAE, 0x2D57, 0x2D93, 0x2DC7, 0x2DDB, 0x2DFF,
+0x2DD5, 0x2D53, 0xAE2D, 0x572D, 0x932D, 0xC72D, 0xDB2D, 0xFF2D, 0xD52D, 0x532D
+},
+{/*2E*/ 0x2E00, 0x002E, 0x2E2E, 0x2E17, 0x2EB3, 0x2EE1, 0x2EFC, 0x2E7E, 0x2E89,
+0x2E3F, 0x2E85, 0x172E, 0xB32E, 0xE12E, 0xFC2E, 0x7E2E, 0x892E, 0x3F2E, 0x852E
+},
+{/*2F*/ 0x2F00, 0x002F, 0x2F2F, 0x2FAF, 0x2FEF, 0x2FCF, 0x2F3A, 0x2F1D, 0x2F74,
+0x2FB6, 0x2F18, 0xAF2F, 0xEF2F, 0xCF2F, 0x3A2F, 0x1D2F, 0x742F, 0xB62F, 0x182F
+},
+{/*30*/ 0x3000, 0x0030, 0x3030, 0x3018, 0x300C, 0x3006, 0x3023, 0x30A9, 0x3046,
+0x30EC, 0x30BF, 0x1830, 0x0C30, 0x0630, 0x2330, 0xA930, 0x4630, 0xEC30, 0xBF30
+},
+{/*31*/ 0x3100, 0x0031, 0x3131, 0x31A0, 0x3150, 0x3128, 0x31E5, 0x31CA, 0x31BB,
+0x3165, 0x3122, 0xA031, 0x5031, 0x2831, 0xE531, 0xCA31, 0xBB31, 0x6531, 0x2231
+},
+{/*32*/ 0x3200, 0x0032, 0x3232, 0x3219, 0x32B4, 0x325A, 0x32DE, 0x326F, 0x32CD,
+0x328F, 0x32F4, 0x1932, 0xB432, 0x5A32, 0xDE32, 0x6F32, 0xCD32, 0x8F32, 0xF432
+},
+{/*33*/ 0x3300, 0x0033, 0x3333, 0x33A1, 0x33E8, 0x3374, 0x3318, 0x330C, 0x3330,
+0x3306, 0x3369, 0xA133, 0xE833, 0x7433, 0x1833, 0x0C33, 0x3033, 0x0633, 0x6933
+},
+{/*34*/ 0x3400, 0x0034, 0x3434, 0x341A, 0x340D, 0x34BE, 0x34A8, 0x3454, 0x3421,
+0x342A, 0x3429, 0x1A34, 0x0D34, 0xBE34, 0xA834, 0x5434, 0x2134, 0x2A34, 0x2934
+},
+{/*35*/ 0x3500, 0x0035, 0x3535, 0x35A2, 0x3551, 0x3590, 0x356E, 0x3537, 0x35DC,
+0x35A3, 0x35B4, 0xA235, 0x5135, 0x9035, 0x6E35, 0x3735, 0xDC35, 0xA335, 0xB435
+},
+{/*36*/ 0x3600, 0x0036, 0x3636, 0x361B, 0x36B5, 0x36E2, 0x3655, 0x3692, 0x36AA,
+0x3649, 0x3662, 0x1B36, 0xB536, 0xE236, 0x5536, 0x9236, 0xAA36, 0x4936, 0x6236
+},
+{/*37*/ 0x3700, 0x0037, 0x3737, 0x37A3, 0x37E9, 0x37CC, 0x3793, 0x37F1, 0x3757,
+0x37C0, 0x37FF, 0xA337, 0xE937, 0xCC37, 0x9337, 0xF137, 0x5737, 0xC037, 0xFF37
+},
+{/*38*/ 0x3800, 0x0038, 0x3838, 0x381C, 0x380E, 0x3807, 0x3844, 0x3822, 0x3888,
+0x3811, 0x38E2, 0x1C38, 0x0E38, 0x0738, 0x4438, 0x2238, 0x8838, 0x1138, 0xE238
+},
+{/*39*/ 0x3900, 0x0039, 0x3939, 0x39A4, 0x3952, 0x3929, 0x3982, 0x3941, 0x3975,
+0x3998, 0x397F, 0xA439, 0x5239, 0x2939, 0x8239, 0x4139, 0x7539, 0x9839, 0x7F39
+},
+{/*3A*/ 0x3A00, 0x003A, 0x3A3A, 0x3A1D, 0x3AB6, 0x3A5B, 0x3AB9, 0x3AE4, 0x3A03,
+0x3A72, 0x3AA9, 0x1D3A, 0xB63A, 0x5B3A, 0xB93A, 0xE43A, 0x033A, 0x723A, 0xA93A
+},
+{/*3B*/ 0x3B00, 0x003B, 0x3B3B, 0x3BA5, 0x3BEA, 0x3B75, 0x3B7F, 0x3B87, 0x3BFE,
+0x3BFB, 0x3B34, 0xA53B, 0xEA3B, 0x753B, 0x7F3B, 0x873B, 0xFE3B, 0xFB3B, 0x343B
+},
+{/*3C*/ 0x3C00, 0x003C, 0x3C3C, 0x3C1E, 0x3C0F, 0x3CBF, 0x3CCF, 0x3CDF, 0x3CEF,
+0x3CD7, 0x3C74, 0x1E3C, 0x0F3C, 0xBF3C, 0xCF3C, 0xDF3C, 0xEF3C, 0xD73C, 0x743C
+},
+{/*3D*/ 0x3D00, 0x003D, 0x3D3D, 0x3DA6, 0x3D53, 0x3D91, 0x3D09, 0x3DBC, 0x3D12,
+0x3D5E, 0x3DE9, 0xA63D, 0x533D, 0x913D, 0x093D, 0xBC3D, 0x123D, 0x5E3D, 0xE93D
+},
+{/*3E*/ 0x3E00, 0x003E, 0x3E3E, 0x3E1F, 0x3EB7, 0x3EE3, 0x3E32, 0x3E19, 0x3E64,
+0x3EB4, 0x3E3F, 0x1F3E, 0xB73E, 0xE33E, 0x323E, 0x193E, 0x643E, 0xB43E, 0x3F3E
+},
+{/*3F*/ 0x3F00, 0x003F, 0x3F3F, 0x3FA7, 0x3FEB, 0x3FCD, 0x3FF4, 0x3F7A, 0x3F99,
+0x3F3D, 0x3FA2, 0xA73F, 0xEB3F, 0xCD3F, 0xF43F, 0x7A3F, 0x993F, 0x3D3F, 0xA23F
+},
+{/*40*/ 0x4000, 0x0040, 0x4040, 0x4020, 0x4010, 0x4008, 0x40AB, 0x40ED, 0x4027,
+0x40CE, 0x400A, 0x2040, 0x1040, 0x0840, 0xAB40, 0xED40, 0x2740, 0xCE40, 0x0A40
+},
+{/*41*/ 0x4100, 0x0041, 0x4141, 0x4198, 0x414C, 0x4126, 0x416D, 0x418E, 0x41DA,
+0x4147, 0x4197, 0x9841, 0x4C41, 0x2641, 0x6D41, 0x8E41, 0xDA41, 0x4741, 0x9741
+},
+{/*42*/ 0x4200, 0x0042, 0x4242, 0x4221, 0x42A8, 0x4254, 0x4256, 0x422B, 0x42AC,
+0x42AD, 0x4241, 0x2142, 0xA842, 0x5442, 0x5642, 0x2B42, 0xAC42, 0xAD42, 0x4142
+},
+{/*43*/ 0x4300, 0x0043, 0x4343, 0x4399, 0x43F4, 0x437A, 0x4390, 0x4348, 0x4351,
+0x4324, 0x43DC, 0x9943, 0xF443, 0x7A43, 0x9043, 0x4843, 0x5143, 0x2443, 0xDC43
+},
+{/*44*/ 0x4400, 0x0044, 0x4444, 0x4422, 0x4411, 0x44B0, 0x4420, 0x4410, 0x4440,
+0x4408, 0x449C, 0x2244, 0x1144, 0xB044, 0x2044, 0x1044, 0x4044, 0x0844, 0x9C44
+},
+{/*45*/ 0x4500, 0x0045, 0x4545, 0x459A, 0x454D, 0x459E, 0x45E6, 0x4573, 0x45BD,
+0x4581, 0x4501, 0x9A45, 0x4D45, 0x9E45, 0xE645, 0x7345, 0xBD45, 0x8145, 0x0145
+},
+{/*46*/ 0x4600, 0x0046, 0x4646, 0x4623, 0x46A9, 0x46EC, 0x46DD, 0x46D6, 0x46CB,
+0x466B, 0x46D7, 0x2346, 0xA946, 0xEC46, 0xDD46, 0xD646, 0xCB46, 0x6B46, 0xD746
+},
+{/*47*/ 0x4700, 0x0047, 0x4747, 0x479B, 0x47F5, 0x47C2, 0x471B, 0x47B5, 0x4736,
+0x47E2, 0x474A, 0x9B47, 0xF547, 0xC247, 0x1B47, 0xB547, 0x3647, 0xE247, 0x4A47
+},
+{/*48*/ 0x4800, 0x0048, 0x4848, 0x4824, 0x4812, 0x4809, 0x48CC, 0x4866, 0x48E9,
+0x4833, 0x4857, 0x2448, 0x1248, 0x0948, 0xCC48, 0x6648, 0xE948, 0x3348, 0x5748
+},
+{/*49*/ 0x4900, 0x0049, 0x4949, 0x499C, 0x494E, 0x4927, 0x490A, 0x4905, 0x4914,
+0x49BA, 0x49CA, 0x9C49, 0x4E49, 0x2749, 0x0A49, 0x0549, 0x1449, 0xBA49, 0xCA49
+},
+{/*4A*/ 0x4A00, 0x004A, 0x4A4A, 0x4A25, 0x4AAA, 0x4A55, 0x4A31, 0x4AA0, 0x4A62,
+0x4A50, 0x4A1C, 0x254A, 0xAA4A, 0x554A, 0x314A, 0xA04A, 0x624A, 0x504A, 0x1C4A
+},
+{/*4B*/ 0x4B00, 0x004B, 0x4B4B, 0x4B9D, 0x4BF6, 0x4B7B, 0x4BF7, 0x4BC3, 0x4B9F,
+0x4BD9, 0x4B81, 0x9D4B, 0xF64B, 0x7B4B, 0xF74B, 0xC34B, 0x9F4B, 0xD94B, 0x814B
+},
+{/*4C*/ 0x4C00, 0x004C, 0x4C4C, 0x4C26, 0x4C13, 0x4CB1, 0x4C47, 0x4C9B, 0x4C8E,
+0x4CF5, 0x4CC1, 0x264C, 0x134C, 0xB14C, 0x474C, 0x9B4C, 0x8E4C, 0xF54C, 0xC14C
+},
+{/*4D*/ 0x4D00, 0x004D, 0x4D4D, 0x4D9E, 0x4D4F, 0x4D9F, 0x4D81, 0x4DF8, 0x4D73,
+0x4D7C, 0x4D5C, 0x9E4D, 0x4F4D, 0x9F4D, 0x814D, 0xF84D, 0x734D, 0x7C4D, 0x5C4D
+},
+{/*4E*/ 0x4E00, 0x004E, 0x4E4E, 0x4E27, 0x4EAB, 0x4EED, 0x4EBA, 0x4E5D, 0x4E05,
+0x4E96, 0x4E8A, 0x274E, 0xAB4E, 0xED4E, 0xBA4E, 0x5D4E, 0x054E, 0x964E, 0x8A4E
+},
+{/*4F*/ 0x4F00, 0x004F, 0x4F4F, 0x4F9F, 0x4FF7, 0x4FC3, 0x4F7C, 0x4F3E, 0x4FF8,
+0x4F1F, 0x4F17, 0x9F4F, 0xF74F, 0xC34F, 0x7C4F, 0x3E4F, 0xF84F, 0x1F4F, 0x174F
+},
+{/*50*/ 0x5000, 0x0050, 0x5050, 0x5028, 0x5014, 0x500A, 0x5065, 0x508A, 0x50CA,
+0x5045, 0x50B0, 0x2850, 0x1450, 0x0A50, 0x6550, 0x8A50, 0xCA50, 0x4550, 0xB050
+},
+{/*51*/ 0x5100, 0x0051, 0x5151, 0x5190, 0x5148, 0x5124, 0x51A3, 0x51E9, 0x5137,
+0x51CC, 0x512D, 0x9051, 0x4851, 0x2451, 0xA351, 0xE951, 0x3751, 0xCC51, 0x2D51
+},
+{/*52*/ 0x5200, 0x0052, 0x5252, 0x5229, 0x52AC, 0x5256, 0x5298, 0x524C, 0x5241,
+0x5226, 0x52FB, 0x2952, 0xAC52, 0x5652, 0x9852, 0x4C52, 0x4152, 0x2652, 0xFB52
+},
+{/*53*/ 0x5300, 0x0053, 0x5353, 0x5391, 0x53F0, 0x5378, 0x535E, 0x532F, 0x53BC,
+0x53AF, 0x5366, 0x9153, 0xF053, 0x7853, 0x5E53, 0x2F53, 0xBC53, 0xAF53, 0x6653
+},
+{/*54*/ 0x5400, 0x0054, 0x5454, 0x542A, 0x5415, 0x54B2, 0x54EE, 0x5477, 0x54AD,
+0x5483, 0x5426, 0x2A54, 0x1554, 0xB254, 0xEE54, 0x7754, 0xAD54, 0x8354, 0x2654
+},
+{/*55*/ 0x5500, 0x0055, 0x5555, 0x5592, 0x5549, 0x559C, 0x5528, 0x5514, 0x5550,
+0x550A, 0x55BB, 0x9255, 0x4955, 0x9C55, 0x2855, 0x1455, 0x5055, 0x0A55, 0xBB55
+},
+{/*56*/ 0x5600, 0x0056, 0x5656, 0x562B, 0x56AD, 0x56EE, 0x5613, 0x56B1, 0x5626,
+0x56E0, 0x566D, 0x2B56, 0xAD56, 0xEE56, 0x1356, 0xB156, 0x2656, 0xE056, 0x6D56
+},
+{/*57*/ 0x5700, 0x0057, 0x5757, 0x5793, 0x57F1, 0x57C0, 0x57D5, 0x57D2, 0x57DB,
+0x5769, 0x57F0, 0x9357, 0xF157, 0xC057, 0xD557, 0xD257, 0xDB57, 0x6957, 0xF057
+},
+{/*58*/ 0x5800, 0x0058, 0x5858, 0x582C, 0x5816, 0x580B, 0x5802, 0x5801, 0x5804,
+0x58B8, 0x58ED, 0x2C58, 0x1658, 0x0B58, 0x0258, 0x0158, 0x0458, 0xB858, 0xED58
+},
+{/*59*/ 0x5900, 0x0059, 0x5959, 0x5994, 0x594A, 0x5925, 0x59C4, 0x5962, 0x59F9,
+0x5931, 0x5970, 0x9459, 0x4A59, 0x2559, 0xC459, 0x6259, 0xF959, 0x3159, 0x7059
+},
+{/*5A*/ 0x5A00, 0x005A, 0x5A5A, 0x5A2D, 0x5AAE, 0x5A57, 0x5AFF, 0x5AC7, 0x5A8F,
+0x5ADB, 0x5AA6, 0x2D5A, 0xAE5A, 0x575A, 0xFF5A, 0xC75A, 0x8F5A, 0xDB5A, 0xA65A
+},
+{/*5B*/ 0x5B00, 0x005B, 0x5B5B, 0x5B95, 0x5BF2, 0x5B79, 0x5B39, 0x5BA4, 0x5B72,
+0x5B52, 0x5B3B, 0x955B, 0xF25B, 0x795B, 0x395B, 0xA45B, 0x725B, 0x525B, 0x3B5B
+},
+{/*5C*/ 0x5C00, 0x005C, 0x5C5C, 0x5C2E, 0x5C17, 0x5CB3, 0x5C89, 0x5CFC, 0x5C63,
+0x5C7E, 0x5C7B, 0x2E5C, 0x175C, 0xB35C, 0x895C, 0xFC5C, 0x635C, 0x7E5C, 0x7B5C
+},
+{/*5D*/ 0x5D00, 0x005D, 0x5D5D, 0x5D96, 0x5D4B, 0x5D9D, 0x5D4F, 0x5D9F, 0x5D9E,
+0x5DF7, 0x5DE6, 0x965D, 0x4B5D, 0x9D5D, 0x4F5D, 0x9F5D, 0x9E5D, 0xF75D, 0xE65D
+},
+{/*5E*/ 0x5E00, 0x005E, 0x5E5E, 0x5E2F, 0x5EAF, 0x5EEF, 0x5E74, 0x5E3A, 0x5EE8,
+0x5E1D, 0x5E30, 0x2F5E, 0xAF5E, 0xEF5E, 0x745E, 0x3A5E, 0xE85E, 0x1D5E, 0x305E
+},
+{/*5F*/ 0x5F00, 0x005F, 0x5F5F, 0x5F97, 0x5FF3, 0x5FC1, 0x5FB2, 0x5F59, 0x5F15,
+0x5F94, 0x5FAD, 0x975F, 0xF35F, 0xC15F, 0xB25F, 0x595F, 0x155F, 0x945F, 0xAD5F
+},
+{/*60*/ 0x6000, 0x0060, 0x6060, 0x6030, 0x6018, 0x600C, 0x6046, 0x6023, 0x608C,
+0x60A9, 0x600F, 0x3060, 0x1860, 0x0C60, 0x4660, 0x2360, 0x8C60, 0xA960, 0x0F60
+},
+{/*61*/ 0x6100, 0x0061, 0x6161, 0x6188, 0x6144, 0x6122, 0x6180, 0x6140, 0x6171,
+0x6120, 0x6192, 0x8861, 0x4461, 0x2261, 0x8061, 0x4061, 0x7161, 0x2061, 0x9261
+},
+{/*62*/ 0x6200, 0x0062, 0x6262, 0x6231, 0x62A0, 0x6250, 0x62BB, 0x62E5, 0x6207,
+0x62CA, 0x6244, 0x3162, 0xA062, 0x5062, 0xBB62, 0xE562, 0x0762, 0xCA62, 0x4462
+},
+{/*63*/ 0x6300, 0x0063, 0x6363, 0x6389, 0x63FC, 0x637E, 0x637D, 0x6386, 0x63FA,
+0x6343, 0x63D9, 0x8963, 0xFC63, 0x7E63, 0x7D63, 0x8663, 0xFA63, 0x4363, 0xD963
+},
+{/*64*/ 0x6400, 0x0064, 0x6464, 0x6432, 0x6419, 0x64B4, 0x64CD, 0x64DE, 0x64EB,
+0x646F, 0x6499, 0x3264, 0x1964, 0xB464, 0xCD64, 0xDE64, 0xEB64, 0x6F64, 0x9964
+},
+{/*65*/ 0x6500, 0x0065, 0x6565, 0x658A, 0x6545, 0x659A, 0x650B, 0x65BD, 0x6516,
+0x65E6, 0x6504, 0x8A65, 0x4565, 0x9A65, 0x0B65, 0xBD65, 0x1665, 0xE665, 0x0465
+},
+{/*66*/ 0x6600, 0x0066, 0x6666, 0x6633, 0x66A1, 0x66E8, 0x6630, 0x6618, 0x6660,
+0x660C, 0x66D2, 0x3366, 0xA166, 0xE866, 0x3066, 0x1866, 0x6066, 0x0C66, 0xD266
+},
+{/*67*/ 0x6700, 0x0067, 0x6767, 0x678B, 0x67FD, 0x67C6, 0x67F6, 0x677B, 0x679D,
+0x6785, 0x674F, 0x8B67, 0xFD67, 0xC667, 0xF667, 0x7B67, 0x9D67, 0x8567, 0x4F67
+},
+{/*68*/ 0x6800, 0x0068, 0x6868, 0x6834, 0x681A, 0x680D, 0x6821, 0x68A8, 0x6842,
+0x6854, 0x6852, 0x3468, 0x1A68, 0x0D68, 0x2168, 0xA868, 0x4268, 0x5468, 0x5268
+},
+{/*69*/ 0x6900, 0x0069, 0x6969, 0x698C, 0x6946, 0x6923, 0x69E7, 0x69CB, 0x69BF,
+0x69DD, 0x69CF, 0x8C69, 0x4669, 0x2369, 0xE769, 0xCB69, 0xBF69, 0xDD69, 0xCF69
+},
+{/*6A*/ 0x6A00, 0x006A, 0x6A6A, 0x6A35, 0x6AA2, 0x6A51, 0x6ADC, 0x6A6E, 0x6AC9,
+0x6A37, 0x6A19, 0x356A, 0xA26A, 0x516A, 0xDC6A, 0x6E6A, 0xC96A, 0x376A, 0x196A
+},
+{/*6B*/ 0x6B00, 0x006B, 0x6B6B, 0x6B8D, 0x6BFE, 0x6B7F, 0x6B1A, 0x6B0D, 0x6B34,
+0x6BBE, 0x6B84, 0x8D6B, 0xFE6B, 0x7F6B, 0x1A6B, 0x0D6B, 0x346B, 0xBE6B, 0x846B
+},
+{/*6C*/ 0x6C00, 0x006C, 0x6C6C, 0x6C36, 0x6C1B, 0x6CB5, 0x6CAA, 0x6C55, 0x6C25,
+0x6C92, 0x6CC4, 0x366C, 0x1B6C, 0xB56C, 0xAA6C, 0x556C, 0x256C, 0x926C, 0xC46C
+},
+{/*6D*/ 0x6D00, 0x006D, 0x6D6D, 0x6D8E, 0x6D47, 0x6D9B, 0x6D6C, 0x6D36, 0x6DD8,
+0x6D1B, 0x6D59, 0x8E6D, 0x476D, 0x9B6D, 0x6C6D, 0x366D, 0xD86D, 0x1B6D, 0x596D
+},
+{/*6E*/ 0x6E00, 0x006E, 0x6E6E, 0x6E37, 0x6EA3, 0x6EE9, 0x6E57, 0x6E93, 0x6EAE,
+0x6EF1, 0x6E8F, 0x376E, 0xA36E, 0xE96E, 0x576E, 0x936E, 0xAE6E, 0xF16E, 0x8F6E
+},
+{/*6F*/ 0x6F00, 0x006F, 0x6F6F, 0x6F8F, 0x6FFF, 0x6FC7, 0x6F91, 0x6FF0, 0x6F53,
+0x6F78, 0x6F12, 0x8F6F, 0xFF6F, 0xC76F, 0x916F, 0xF06F, 0x536F, 0x786F, 0x126F
+},
+{/*70*/ 0x7000, 0x0070, 0x7070, 0x7038, 0x701C, 0x700E, 0x7088, 0x7044, 0x7061,
+0x7022, 0x70B5, 0x3870, 0x1C70, 0x0E70, 0x8870, 0x4470, 0x6170, 0x2270, 0xB570
+},
+{/*71*/ 0x7100, 0x0071, 0x7171, 0x7180, 0x7140, 0x7120, 0x714E, 0x7127, 0x719C,
+0x71AB, 0x7128, 0x8071, 0x4071, 0x2071, 0x4E71, 0x2771, 0x9C71, 0xAB71, 0x2871
+},
+{/*72*/ 0x7200, 0x0072, 0x7272, 0x7239, 0x72A4, 0x7252, 0x7275, 0x7282, 0x72EA,
+0x7241, 0x72FE, 0x3972, 0xA472, 0x5272, 0x7572, 0x8272, 0xEA72, 0x4172, 0xFE72
+},
+{/*73*/ 0x7300, 0x0073, 0x7373, 0x7381, 0x73F8, 0x737C, 0x73B3, 0x73E1, 0x7317,
+0x73C8, 0x7363, 0x8173, 0xF873, 0x7C73, 0xB373, 0xE173, 0x1773, 0xC873, 0x6373
+},
+{/*74*/ 0x7400, 0x0074, 0x7474, 0x743A, 0x741D, 0x74B6, 0x7403, 0x74B9, 0x7406,
+0x74E4, 0x7423, 0x3A74, 0x1D74, 0xB674, 0x0374, 0xB974, 0x0674, 0xE474, 0x2374
+},
+{/*75*/ 0x7500, 0x0075, 0x7575, 0x7582, 0x7541, 0x7598, 0x75C5, 0x75DA, 0x75FB,
+0x756D, 0x75BE, 0x8275, 0x4175, 0x9875, 0xC575, 0xDA75, 0xFB75, 0x6D75, 0xBE75
+},
+{/*76*/ 0x7600, 0x0076, 0x7676, 0x763B, 0x76A5, 0x76EA, 0x76FE, 0x767F, 0x768D,
+0x7687, 0x7668, 0x3B76, 0xA576, 0xEA76, 0xFE76, 0x7F76, 0x8D76, 0x8776, 0x6876
+},
+{/*77*/ 0x7700, 0x0077, 0x7777, 0x7783, 0x77F9, 0x77C4, 0x7738, 0x771C, 0x7770,
+0x770E, 0x77F5, 0x8377, 0xF977, 0xC477, 0x3877, 0x1C77, 0x7077, 0x0E77, 0xF577
+},
+{/*78*/ 0x7800, 0x0078, 0x7878, 0x783C, 0x781E, 0x780F, 0x78EF, 0x78CF, 0x78AF,
+0x78DF, 0x78E8, 0x3C78, 0x1E78, 0x0F78, 0xEF78, 0xCF78, 0xAF78, 0xDF78, 0xE878
+},
+{/*79*/ 0x7900, 0x0079, 0x7979, 0x7984, 0x7942, 0x7921, 0x7929, 0x79AC, 0x7952,
+0x7956, 0x7975, 0x8479, 0x4279, 0x2179, 0x2979, 0xAC79, 0x5279, 0x5679, 0x7579
+},
+{/*7A*/ 0x7A00, 0x007A, 0x7A7A, 0x7A3D, 0x7AA6, 0x7A53, 0x7A12, 0x7A09, 0x7A24,
+0x7ABC, 0x7AA3, 0x3D7A, 0xA67A, 0x537A, 0x127A, 0x097A, 0x247A, 0xBC7A, 0xA37A
+},
+{/*7B*/ 0x7B00, 0x007B, 0x7B7B, 0x7B85, 0x7BFA, 0x7B7D, 0x7BD4, 0x7B6A, 0x7BD9,
+0x7B35, 0x7B3E, 0x857B, 0xFA7B, 0x7D7B, 0xD47B, 0x6A7B, 0xD97B, 0x357B, 0x3E7B
+},
+{/*7C*/ 0x7C00, 0x007C, 0x7C7C, 0x7C3E, 0x7C1F, 0x7CB7, 0x7C64, 0x7C32, 0x7CC8,
+0x7C19, 0x7C7E, 0x3E7C, 0x1F7C, 0xB77C, 0x647C, 0x327C, 0xC87C, 0x197C, 0x7E7C
+},
+{/*7D*/ 0x7D00, 0x007D, 0x7D7D, 0x7D86, 0x7D43, 0x7D99, 0x7DA2, 0x7D51, 0x7D35,
+0x7D90, 0x7DE3, 0x867D, 0x437D, 0x997D, 0xA27D, 0x517D, 0x357D, 0x907D, 0xE37D
+},
+{/*7E*/ 0x7E00, 0x007E, 0x7E7E, 0x7E3F, 0x7EA7, 0x7EEB, 0x7E99, 0x7EF4, 0x7E43,
+0x7E7A, 0x7E35, 0x3F7E, 0xA77E, 0xEB7E, 0x997E, 0xF47E, 0x437E, 0x7A7E, 0x357E
+},
+{/*7F*/ 0x7F00, 0x007F, 0x7F7F, 0x7F87, 0x7FFB, 0x7FC5, 0x7F5F, 0x7F97, 0x7FBE,
+0x7FF3, 0x7FA8, 0x877F, 0xFB7F, 0xC57F, 0x5F7F, 0x977F, 0xBE7F, 0xF37F, 0xA87F
+},
+{/*80*/ 0x8000, 0x0080, 0x8080, 0x8040, 0x8020, 0x8010, 0x8027, 0x80AB, 0x804E,
+0x80ED, 0x8014, 0x4080, 0x2080, 0x1080, 0x2780, 0xAB80, 0x4E80, 0xED80, 0x1480
+},
+{/*81*/ 0x8100, 0x0081, 0x8181, 0x81F8, 0x817C, 0x813E, 0x81E1, 0x81C8, 0x81B3,
+0x8164, 0x8189, 0xF881, 0x7C81, 0x3E81, 0xE181, 0xC881, 0xB381, 0x6481, 0x8981
+},
+{/*82*/ 0x8200, 0x0082, 0x8282, 0x8241, 0x8298, 0x824C, 0x82DA, 0x826D, 0x82C5,
+0x828E, 0x825F, 0x4182, 0x9882, 0x4C82, 0xDA82, 0x6D82, 0xC582, 0x8E82, 0x5F82
+},
+{/*83*/ 0x8300, 0x0083, 0x8383, 0x83F9, 0x83C4, 0x8362, 0x831C, 0x830E, 0x8338,
+0x8307, 0x83C2, 0xF983, 0xC483, 0x6283, 0x1C83, 0x0E83, 0x3883, 0x0783, 0xC283
+},
+{/*84*/ 0x8400, 0x0084, 0x8484, 0x8442, 0x8421, 0x84A8, 0x84AC, 0x8456, 0x8429,
+0x842B, 0x8482, 0x4284, 0x2184, 0xA884, 0xAC84, 0x5684, 0x2984, 0x2B84, 0x8284
+},
+{/*85*/ 0x8500, 0x0085, 0x8585, 0x85FA, 0x857D, 0x8586, 0x856A, 0x8535, 0x85D4,
+0x85A2, 0x851F, 0xFA85, 0x7D85, 0x8685, 0x6A85, 0x3585, 0xD485, 0xA285, 0x1F85
+},
+{/*86*/ 0x8600, 0x0086, 0x8686, 0x8643, 0x8699, 0x86F4, 0x8651, 0x8690, 0x86A2,
+0x8648, 0x86C9, 0x4386, 0x9986, 0xF486, 0x5186, 0x9086, 0xA286, 0x4886, 0xC986
+},
+{/*87*/ 0x8700, 0x0087, 0x8787, 0x87FB, 0x87C5, 0x87DA, 0x8797, 0x87F3, 0x875F,
+0x87C1, 0x8754, 0xFB87, 0xC587, 0xDA87, 0x9787, 0xF387, 0x5F87, 0xC187, 0x5487
+},
+{/*88*/ 0x8800, 0x0088, 0x8888, 0x8844, 0x8822, 0x8811, 0x8840, 0x8820, 0x8880,
+0x8810, 0x8849, 0x4488, 0x2288, 0x1188, 0x4088, 0x2088, 0x8088, 0x1088, 0x4988
+},
+{/*89*/ 0x8900, 0x0089, 0x8989, 0x89FC, 0x897E, 0x893F, 0x8986, 0x8943, 0x897D,
+0x8999, 0x89D4, 0xFC89, 0x7E89, 0x3F89, 0x8689, 0x4389, 0x7D89, 0x9989, 0xD489
+},
+{/*8A*/ 0x8A00, 0x008A, 0x8A8A, 0x8A45, 0x8A9A, 0x8A4D, 0x8ABD, 0x8AE6, 0x8A0B,
+0x8A73, 0x8A02, 0x458A, 0x9A8A, 0x4D8A, 0xBD8A, 0xE68A, 0x0B8A, 0x738A, 0x028A
+},
+{/*8B*/ 0x8B00, 0x008B, 0x8B8B, 0x8BFD, 0x8BC6, 0x8B63, 0x8B7B, 0x8B85, 0x8BF6,
+0x8BFA, 0x8B9F, 0xFD8B, 0xC68B, 0x638B, 0x7B8B, 0x858B, 0xF68B, 0xFA8B, 0x9F8B
+},
+{/*8C*/ 0x8C00, 0x008C, 0x8C8C, 0x8C46, 0x8C23, 0x8CA9, 0x8CCB, 0x8CDD, 0x8CE7,
+0x8CD6, 0x8CDF, 0x468C, 0x238C, 0xA98C, 0xCB8C, 0xDD8C, 0xE78C, 0xD68C, 0xDF8C
+},
+{/*8D*/ 0x8D00, 0x008D, 0x8D8D, 0x8DFE, 0x8D7F, 0x8D87, 0x8D0D, 0x8DBE, 0x8D1A,
+0x8D5F, 0x8D42, 0xFE8D, 0x7F8D, 0x878D, 0x0D8D, 0xBE8D, 0x1A8D, 0x5F8D, 0x428D
+},
+{/*8E*/ 0x8E00, 0x008E, 0x8E8E, 0x8E47, 0x8E9B, 0x8EF5, 0x8E36, 0x8E1B, 0x8E6C,
+0x8EB5, 0x8E94, 0x478E, 0x9B8E, 0xF58E, 0x368E, 0x1B8E, 0x6C8E, 0xB58E, 0x948E
+},
+{/*8F*/ 0x8F00, 0x008F, 0x8F8F, 0x8FFF, 0x8FC7, 0x8FDB, 0x8FF0, 0x8F78, 0x8F91,
+0x8F3C, 0x8F09, 0xFF8F, 0xC78F, 0xDB8F, 0xF08F, 0x788F, 0x918F, 0x3C8F, 0x098F
+},
+{/*90*/ 0x9000, 0x0090, 0x9090, 0x9048, 0x9024, 0x9012, 0x90E9, 0x90CC, 0x90A3,
+0x9066, 0x90AE, 0x4890, 0x2490, 0x1290, 0xE990, 0xCC90, 0xA390, 0x6690, 0xAE90
+},
+{/*91*/ 0x9100, 0x0091, 0x9191, 0x91F0, 0x9178, 0x913C, 0x912F, 0x91AF, 0x915E,
+0x91EF, 0x9133, 0xF091, 0x7891, 0x3C91, 0x2F91, 0xAF91, 0x5E91, 0xEF91, 0x3391
+},
+{/*92*/ 0x9200, 0x0092, 0x9292, 0x9249, 0x929C, 0x924E, 0x9214, 0x920A, 0x9228,
+0x9205, 0x92E5, 0x4992, 0x9C92, 0x4E92, 0x1492, 0x0A92, 0x2892, 0x0592, 0xE592
+},
+{/*93*/ 0x9300, 0x0093, 0x9393, 0x93F1, 0x93C0, 0x9360, 0x93D2, 0x9369, 0x93D5,
+0x938C, 0x9378, 0xF193, 0xC093, 0x6093, 0xD293, 0x6993, 0xD593, 0x8C93, 0x7893
+},
+{/*94*/ 0x9400, 0x0094, 0x9494, 0x944A, 0x9425, 0x94AA, 0x9462, 0x9431, 0x94C4,
+0x94A0, 0x9438, 0x4A94, 0x2594, 0xAA94, 0x6294, 0x3194, 0xC494, 0xA094, 0x3894
+},
+{/*95*/ 0x9500, 0x0095, 0x9595, 0x95F2, 0x9579, 0x9584, 0x95A4, 0x9552, 0x9539,
+0x9529, 0x95A5, 0xF295, 0x7995, 0x8495, 0xA495, 0x5295, 0x3995, 0x2995, 0xA595
+},
+{/*96*/ 0x9600, 0x0096, 0x9696, 0x964B, 0x969D, 0x96F6, 0x969F, 0x96F7, 0x964F,
+0x96C3, 0x9673, 0x4B96, 0x9D96, 0xF696, 0x9F96, 0xF796, 0x4F96, 0xC396, 0x7396
+},
+{/*97*/ 0x9700, 0x0097, 0x9797, 0x97F3, 0x97C1, 0x97D8, 0x9759, 0x9794, 0x97B2,
+0x974A, 0x97EE, 0xF397, 0xC197, 0xD897, 0x5997, 0x9497, 0xB297, 0x4A97, 0xEE97
+},
+{/*98*/ 0x9800, 0x0098, 0x9898, 0x984C, 0x9826, 0x9813, 0x988E, 0x9847, 0x986D,
+0x989B, 0x98F3, 0x4C98, 0x2698, 0x1398, 0x8E98, 0x4798, 0x6D98, 0x9B98, 0xF398
+},
+{/*99*/ 0x9900, 0x0099, 0x9999, 0x99F4, 0x997A, 0x993D, 0x9948, 0x9924, 0x9990,
+0x9912, 0x996E, 0xF499, 0x7A99, 0x3D99, 0x4899, 0x2499, 0x9099, 0x1299, 0x6E99
+},
+{/*9A*/ 0x9A00, 0x009A, 0x9A9A, 0x9A4D, 0x9A9E, 0x9A4F, 0x9A73, 0x9A81, 0x9AE6,
+0x9AF8, 0x9AB8, 0x4D9A, 0x9E9A, 0x4F9A, 0x739A, 0x819A, 0xE69A, 0xF89A, 0xB89A
+},
+{/*9B*/ 0x9B00, 0x009B, 0x9B9B, 0x9BF5, 0x9BC2, 0x9B61, 0x9BB5, 0x9BE2, 0x9B1B,
+0x9B71, 0x9B25, 0xF59B, 0xC29B, 0x619B, 0xB59B, 0xE29B, 0x1B9B, 0x719B, 0x259B
+},
+{/*9C*/ 0x9C00, 0x009C, 0x9C9C, 0x9C4E, 0x9C27, 0x9CAB, 0x9C05, 0x9CBA, 0x9C0A,
+0x9C5D, 0x9C65, 0x4E9C, 0x279C, 0xAB9C, 0x059C, 0xBA9C, 0x0A9C, 0x5D9C, 0x659C
+},
+{/*9D*/ 0x9D00, 0x009D, 0x9D9D, 0x9DF6, 0x9D7B, 0x9D85, 0x9DC3, 0x9DD9, 0x9DF7,
+0x9DD4, 0x9DF8, 0xF69D, 0x7B9D, 0x859D, 0xC39D, 0xD99D, 0xF79D, 0xD49D, 0xF89D
+},
+{/*9E*/ 0x9E00, 0x009E, 0x9E9E, 0x9E4F, 0x9E9F, 0x9EF7, 0x9EF8, 0x9E7C, 0x9E81,
+0x9E3E, 0x9E2E, 0x4F9E, 0x9F9E, 0xF79E, 0xF89E, 0x7C9E, 0x819E, 0x3E9E, 0x2E9E
+},
+{/*9F*/ 0x9F00, 0x009F, 0x9F9F, 0x9FF7, 0x9FC3, 0x9FD9, 0x9F3E, 0x9F1F, 0x9F7C,
+0x9FB7, 0x9FB3, 0xF79F, 0xC39F, 0xD99F, 0x3E9F, 0x1F9F, 0x7C9F, 0xB79F, 0xB39F
+},
+{/*A0*/ 0xA000, 0x00A0, 0xA0A0, 0xA050, 0xA028, 0xA014, 0xA0CA, 0xA065, 0xA0E5,
+0xA08A, 0xA011, 0x50A0, 0x28A0, 0x14A0, 0xCAA0, 0x65A0, 0xE5A0, 0x8AA0, 0x11A0
+},
+{/*A1*/ 0xA100, 0x00A1, 0xA1A1, 0xA1E8, 0xA174, 0xA13A, 0xA10C, 0xA106, 0xA118,
+0xA103, 0xA18C, 0xE8A1, 0x74A1, 0x3AA1, 0x0CA1, 0x06A1, 0x18A1, 0x03A1, 0x8CA1
+},
+{/*A2*/ 0xA200, 0x00A2, 0xA2A2, 0xA251, 0xA290, 0xA248, 0xA237, 0xA2A3, 0xA26E,
+0xA2E9, 0xA25A, 0x51A2, 0x90A2, 0x48A2, 0x37A2, 0xA3A2, 0x6EA2, 0xE9A2, 0x5AA2
+},
+{/*A3*/ 0xA300, 0x00A3, 0xA3A3, 0xA3E9, 0xA3CC, 0xA366, 0xA3F1, 0xA3C0, 0xA393,
+0xA360, 0xA3C7, 0xE9A3, 0xCCA3, 0x66A3, 0xF1A3, 0xC0A3, 0x93A3, 0x60A3, 0xC7A3
+},
+{/*A4*/ 0xA400, 0x00A4, 0xA4A4, 0xA452, 0xA429, 0xA4AC, 0xA441, 0xA498, 0xA482,
+0xA44C, 0xA487, 0x52A4, 0x29A4, 0xACA4, 0x41A4, 0x98A4, 0x82A4, 0x4CA4, 0x87A4
+},
+{/*A5*/ 0xA500, 0x00A5, 0xA5A5, 0xA5EA, 0xA575, 0xA582, 0xA587, 0xA5FB, 0xA57F,
+0xA5C5, 0xA51A, 0xEAA5, 0x75A5, 0x82A5, 0x87A5, 0xFBA5, 0x7FA5, 0xC5A5, 0x1AA5
+},
+{/*A6*/ 0xA600, 0x00A6, 0xA6A6, 0xA653, 0xA691, 0xA6F0, 0xA6BC, 0xA65E, 0xA609,
+0xA62F, 0xA6CC, 0x53A6, 0x91A6, 0xF0A6, 0xBCA6, 0x5EA6, 0x09A6, 0x2FA6, 0xCCA6
+},
+{/*A7*/ 0xA700, 0x00A7, 0xA7A7, 0xA7EB, 0xA7CD, 0xA7DE, 0xA77A, 0xA73D, 0xA7F4,
+0xA7A6, 0xA751, 0xEBA7, 0xCDA7, 0xDEA7, 0x7AA7, 0x3DA7, 0xF4A7, 0xA6A7, 0x51A7
+},
+{/*A8*/ 0xA800, 0x00A8, 0xA8A8, 0xA854, 0xA82A, 0xA815, 0xA8AD, 0xA8EE, 0xA82B,
+0xA877, 0xA84C, 0x54A8, 0x2AA8, 0x15A8, 0xADA8, 0xEEA8, 0x2BA8, 0x77A8, 0x4CA8
+},
+{/*A9*/ 0xA900, 0x00A9, 0xA9A9, 0xA9EC, 0xA976, 0xA93B, 0xA96B, 0xA98D, 0xA9D6,
+0xA9FE, 0xA9D1, 0xECA9, 0x76A9, 0x3BA9, 0x6BA9, 0x8DA9, 0xD6A9, 0xFEA9, 0xD1A9
+},
+{/*AA*/ 0xAA00, 0x00AA, 0xAAAA, 0xAA55, 0xAA92, 0xAA49, 0xAA50, 0xAA28, 0xAAA0,
+0xAA14, 0xAA07, 0x55AA, 0x92AA, 0x49AA, 0x50AA, 0x28AA, 0xA0AA, 0x14AA, 0x07AA
+},
+{/*AB*/ 0xAB00, 0x00AB, 0xABAB, 0xABED, 0xABCE, 0xAB67, 0xAB96, 0xAB4B, 0xAB5D,
+0xAB9D, 0xAB9A, 0xEDAB, 0xCEAB, 0x67AB, 0x96AB, 0x4BAB, 0x5DAB, 0x9DAB, 0x9AAB
+},
+{/*AC*/ 0xAC00, 0x00AC, 0xACAC, 0xAC56, 0xAC2B, 0xACAD, 0xAC26, 0xAC13, 0xAC4C,
+0xACB1, 0xACDA, 0x56AC, 0x2BAC, 0xADAC, 0x26AC, 0x13AC, 0x4CAC, 0xB1AC, 0xDAAC
+},
+{/*AD*/ 0xAD00, 0x00AD, 0xADAD, 0xADEE, 0xAD77, 0xAD83, 0xADE0, 0xAD70, 0xADB1,
+0xAD38, 0xAD47, 0xEEAD, 0x77AD, 0x83AD, 0xE0AD, 0x70AD, 0xB1AD, 0x38AD, 0x47AD
+},
+{/*AE*/ 0xAE00, 0x00AE, 0xAEAE, 0xAE57, 0xAE93, 0xAEF1, 0xAEDB, 0xAED5, 0xAEC7,
+0xAED2, 0xAE91, 0x57AE, 0x93AE, 0xF1AE, 0xDBAE, 0xD5AE, 0xC7AE, 0xD2AE, 0x91AE
+},
+{/*AF*/ 0xAF00, 0x00AF, 0xAFAF, 0xAFEF, 0xAFCF, 0xAFDF, 0xAF1D, 0xAFB6, 0xAF3A,
+0xAF5B, 0xAF0C, 0xEFAF, 0xCFAF, 0xDFAF, 0x1DAF, 0xB6AF, 0x3AAF, 0x5BAF, 0x0CAF
+},
+{/*B0*/ 0xB000, 0x00B0, 0xB0B0, 0xB058, 0xB02C, 0xB016, 0xB004, 0xB002, 0xB008,
+0xB001, 0xB0AB, 0x58B0, 0x2CB0, 0x16B0, 0x04B0, 0x02B0, 0x08B0, 0x01B0, 0xABB0
+},
+{/*B1*/ 0xB100, 0x00B1, 0xB1B1, 0xB1E0, 0xB170, 0xB138, 0xB1C2, 0xB161, 0xB1F5,
+0xB188, 0xB136, 0xE0B1, 0x70B1, 0x38B1, 0xC2B1, 0x61B1, 0xF5B1, 0x88B1, 0x36B1
+},
+{/*B2*/ 0xB200, 0x00B2, 0xB2B2, 0xB259, 0xB294, 0xB24A, 0xB2F9, 0xB2C4, 0xB283,
+0xB262, 0xB2E0, 0x59B2, 0x94B2, 0x4AB2, 0xF9B2, 0xC4B2, 0x83B2, 0x62B2, 0xE0B2
+},
+{/*B3*/ 0xB300, 0x00B3, 0xB3B3, 0xB3E1, 0xB3C8, 0xB364, 0xB33F, 0xB3A7, 0xB37E,
+0xB3EB, 0xB37D, 0xE1B3, 0xC8B3, 0x64B3, 0x3FB3, 0xA7B3, 0x7EB3, 0xEBB3, 0x7DB3
+},
+{/*B4*/ 0xB400, 0x00B4, 0xB4B4, 0xB45A, 0xB42D, 0xB4AE, 0xB48F, 0xB4FF, 0xB46F,
+0xB4C7, 0xB43D, 0x5AB4, 0x2DB4, 0xAEB4, 0x8FB4, 0xFFB4, 0x6FB4, 0xC7B4, 0x3DB4
+},
+{/*B5*/ 0xB500, 0x00B5, 0xB5B5, 0xB5E2, 0xB571, 0xB580, 0xB549, 0xB59C, 0xB592,
+0xB54E, 0xB5A0, 0xE2B5, 0x71B5, 0x80B5, 0x49B5, 0x9CB5, 0x92B5, 0x4EB5, 0xA0B5
+},
+{/*B6*/ 0xB600, 0x00B6, 0xB6B6, 0xB65B, 0xB695, 0xB6F2, 0xB672, 0xB639, 0xB6E4,
+0xB6A4, 0xB676, 0x5BB6, 0x95B6, 0xF2B6, 0x72B6, 0x39B6, 0xE4B6, 0xA4B6, 0x76B6
+},
+{/*B7*/ 0xB700, 0x00B7, 0xB7B7, 0xB7E3, 0xB7C9, 0xB7DC, 0xB7B4, 0xB75A, 0xB719,
+0xB72D, 0xB7EB, 0xE3B7, 0xC9B7, 0xDCB7, 0xB4B7, 0x5AB7, 0x19B7, 0x2DB7, 0xEBB7
+},
+{/*B8*/ 0xB800, 0x00B8, 0xB8B8, 0xB85C, 0xB82E, 0xB817, 0xB863, 0xB889, 0xB8C6,
+0xB8FC, 0xB8F6, 0x5CB8, 0x2EB8, 0x17B8, 0x63B8, 0x89B8, 0xC6B8, 0xFCB8, 0xF6B8
+},
+{/*B9*/ 0xB900, 0x00B9, 0xB9B9, 0xB9E4, 0xB972, 0xB939, 0xB9A5, 0xB9EA, 0xB93B,
+0xB975, 0xB96B, 0xE4B9, 0x72B9, 0x39B9, 0xA5B9, 0xEAB9, 0x3BB9, 0x75B9, 0x6BB9
+},
+{/*BA*/ 0xBA00, 0x00BA, 0xBABA, 0xBA5D, 0xBA96, 0xBA4B, 0xBA9E, 0xBA4F, 0xBA4D,
+0xBA9F, 0xBABD, 0x5DBA, 0x96BA, 0x4BBA, 0x9EBA, 0x4FBA, 0x4DBA, 0x9FBA, 0xBDBA
+},
+{/*BB*/ 0xBB00, 0x00BB, 0xBBBB, 0xBBE5, 0xBBCA, 0xBB65, 0xBB58, 0xBB2C, 0xBBB0,
+0xBB16, 0xBB20, 0xE5BB, 0xCABB, 0x65BB, 0x58BB, 0x2CBB, 0xB0BB, 0x16BB, 0x20BB
+},
+{/*BC*/ 0xBC00, 0x00BC, 0xBCBC, 0xBC5E, 0xBC2F, 0xBCAF, 0xBCE8, 0xBC74, 0xBCA1,
+0xBC3A, 0xBC60, 0x5EBC, 0x2FBC, 0xAFBC, 0xE8BC, 0x74BC, 0xA1BC, 0x3ABC, 0x60BC
+},
+{/*BD*/ 0xBD00, 0x00BD, 0xBDBD, 0xBDE6, 0xBD73, 0xBD81, 0xBD2E, 0xBD17, 0xBD5C,
+0xBDB3, 0xBDFD, 0xE6BD, 0x73BD, 0x81BD, 0x2EBD, 0x17BD, 0x5CBD, 0xB3BD, 0xFDBD
+},
+{/*BE*/ 0xBE00, 0x00BE, 0xBEBE, 0xBE5F, 0xBE97, 0xBEF3, 0xBE15, 0xBEB2, 0xBE2A,
+0xBE59, 0xBE2B, 0x5FBE, 0x97BE, 0xF3BE, 0x15BE, 0xB2BE, 0x2ABE, 0x59BE, 0x2BBE
+},
+{/*BF*/ 0xBF00, 0x00BF, 0xBFBF, 0xBFE7, 0xBFCB, 0xBFDD, 0xBFD3, 0xBFD1, 0xBFD7,
+0xBFD0, 0xBFB6, 0xE7BF, 0xCBBF, 0xDDBF, 0xD3BF, 0xD1BF, 0xD7BF, 0xD0BF, 0xB6BF
+},
+{/*C0*/ 0xC000, 0x00C0, 0xC0C0, 0xC060, 0xC030, 0xC018, 0xC08C, 0xC046, 0xC069,
+0xC023, 0xC01E, 0x60C0, 0x30C0, 0x18C0, 0x8CC0, 0x46C0, 0x69C0, 0x23C0, 0x1EC0
+},
+{/*C1*/ 0xC100, 0x00C1, 0xC1C1, 0xC1D8, 0xC16C, 0xC136, 0xC14A, 0xC125, 0xC194,
+0xC1AA, 0xC183, 0xD8C1, 0x6CC1, 0x36C1, 0x4AC1, 0x25C1, 0x94C1, 0xAAC1, 0x83C1
+},
+{/*C2*/ 0xC200, 0x00C2, 0xC2C2, 0xC261, 0xC288, 0xC244, 0xC271, 0xC280, 0xC2E2,
+0xC240, 0xC255, 0x61C2, 0x88C2, 0x44C2, 0x71C2, 0x80C2, 0xE2C2, 0x40C2, 0x55C2
+},
+{/*C3*/ 0xC300, 0x00C3, 0xC3C3, 0xC3D9, 0xC3D4, 0xC36A, 0xC3B7, 0xC3E3, 0xC31F,
+0xC3C9, 0xC3C8, 0xD9C3, 0xD4C3, 0x6AC3, 0xB7C3, 0xE3C3, 0x1FC3, 0xC9C3, 0xC8C3
+},
+{/*C4*/ 0xC400, 0x00C4, 0xC4C4, 0xC462, 0xC431, 0xC4A0, 0xC407, 0xC4BB, 0xC40E,
+0xC4E5, 0xC488, 0x62C4, 0x31C4, 0xA0C4, 0x07C4, 0xBBC4, 0x0EC4, 0xE5C4, 0x88C4
+},
+{/*C5*/ 0xC500, 0x00C5, 0xC5C5, 0xC5DA, 0xC56D, 0xC58E, 0xC5C1, 0xC5D8, 0xC5F3,
+0xC56C, 0xC515, 0xDAC5, 0x6DC5, 0x8EC5, 0xC1C5, 0xD8C5, 0xF3C5, 0x6CC5, 0x15C5
+},
+{/*C6*/ 0xC600, 0x00C6, 0xC6C6, 0xC663, 0xC689, 0xC6FC, 0xC6FA, 0xC67D, 0xC685,
+0xC686, 0xC6C3, 0x63C6, 0x89C6, 0xFCC6, 0xFAC6, 0x7DC6, 0x85C6, 0x86C6, 0xC3C6
+},
+{/*C7*/ 0xC700, 0x00C7, 0xC7C7, 0xC7DB, 0xC7D5, 0xC7D2, 0xC73C, 0xC71E, 0xC778,
+0xC70F, 0xC75E, 0xDBC7, 0xD5C7, 0xD2C7, 0x3CC7, 0x1EC7, 0x78C7, 0x0FC7, 0x5EC7
+},
+{/*C8*/ 0xC800, 0x00C8, 0xC8C8, 0xC864, 0xC832, 0xC819, 0xC8EB, 0xC8CD, 0xC8A7,
+0xC8DE, 0xC843, 0x64C8, 0x32C8, 0x19C8, 0xEBC8, 0xCDC8, 0xA7C8, 0xDEC8, 0x43C8
+},
+{/*C9*/ 0xC900, 0x00C9, 0xC9C9, 0xC9DC, 0xC96E, 0xC937, 0xC92D, 0xC9AE, 0xC95A,
+0xC957, 0xC9DE, 0xDCC9, 0x6EC9, 0x37C9, 0x2DC9, 0xAEC9, 0x5AC9, 0x57C9, 0xDEC9
+},
+{/*CA*/ 0xCA00, 0x00CA, 0xCACA, 0xCA65, 0xCA8A, 0xCA45, 0xCA16, 0xCA0B, 0xCA2C,
+0xCABD, 0xCA08, 0x65CA, 0x8ACA, 0x45CA, 0x16CA, 0x0BCA, 0x2CCA, 0xBDCA, 0x08CA
+},
+{/*CB*/ 0xCB00, 0x00CB, 0xCBCB, 0xCBDD, 0xCBD6, 0xCB6B, 0xCBD0, 0xCB68, 0xCBD1,
+0xCB34, 0xCB95, 0xDDCB, 0xD6CB, 0x6BCB, 0xD0CB, 0x68CB, 0xD1CB, 0x34CB, 0x95CB
+},
+{/*CC*/ 0xCC00, 0x00CC, 0xCCCC, 0xCC66, 0xCC33, 0xCCA1, 0xCC60, 0xCC30, 0xCCC0,
+0xCC18, 0xCCD5, 0x66CC, 0x33CC, 0xA1CC, 0x60CC, 0x30CC, 0xC0CC, 0x18CC, 0xD5CC
+},
+{/*CD*/ 0xCD00, 0x00CD, 0xCDCD, 0xCDDE, 0xCD6F, 0xCD8F, 0xCDA6, 0xCD53, 0xCD3D,
+0xCD91, 0xCD48, 0xDECD, 0x6FCD, 0x8FCD, 0xA6CD, 0x53CD, 0x3DCD, 0x91CD, 0x48CD
+},
+{/*CE*/ 0xCE00, 0x00CE, 0xCECE, 0xCE67, 0xCE8B, 0xCEFD, 0xCE9D, 0xCEF6, 0xCE4B,
+0xCE7B, 0xCE9E, 0x67CE, 0x8BCE, 0xFDCE, 0x9DCE, 0xF6CE, 0x4BCE, 0x7BCE, 0x9ECE
+},
+{/*CF*/ 0xCF00, 0x00CF, 0xCFCF, 0xCFDF, 0xCFD7, 0xCFD3, 0xCF5B, 0xCF95, 0xCFB6,
+0xCFF2, 0xCF03, 0xDFCF, 0xD7CF, 0xD3CF, 0x5BCF, 0x95CF, 0xB6CF, 0xF2CF, 0x03CF
+},
+{/*D0*/ 0xD000, 0x00D0, 0xD0D0, 0xD068, 0xD034, 0xD01A, 0xD042, 0xD021, 0xD084,
+0xD0A8, 0xD0A4, 0x68D0, 0x34D0, 0x1AD0, 0x42D0, 0x21D0, 0x84D0, 0xA8D0, 0xA4D0
+},
+{/*D1*/ 0xD100, 0x00D1, 0xD1D1, 0xD1D0, 0xD168, 0xD134, 0xD184, 0xD142, 0xD179,
+0xD121, 0xD139, 0xD0D1, 0x68D1, 0x34D1, 0x84D1, 0x42D1, 0x79D1, 0x21D1, 0x39D1
+},
+{/*D2*/ 0xD200, 0x00D2, 0xD2D2, 0xD269, 0xD28C, 0xD246, 0xD2BF, 0xD2E7, 0xD20F,
+0xD2CB, 0xD2EF, 0x69D2, 0x8CD2, 0x46D2, 0xBFD2, 0xE7D2, 0x0FD2, 0xCBD2, 0xEFD2
+},
+{/*D3*/ 0xD300, 0x00D3, 0xD3D3, 0xD3D1, 0xD3D0, 0xD368, 0xD379, 0xD384, 0xD3F2,
+0xD342, 0xD372, 0xD1D3, 0xD0D3, 0x68D3, 0x79D3, 0x84D3, 0xF2D3, 0x42D3, 0x72D3
+},
+{/*D4*/ 0xD400, 0x00D4, 0xD4D4, 0xD46A, 0xD435, 0xD4A2, 0xD4C9, 0xD4DC, 0xD4E3,
+0xD46E, 0xD432, 0x6AD4, 0x35D4, 0xA2D4, 0xC9D4, 0xDCD4, 0xE3D4, 0x6ED4, 0x32D4
+},
+{/*D5*/ 0xD500, 0x00D5, 0xD5D5, 0xD5D2, 0xD569, 0xD58C, 0xD50F, 0xD5BF, 0xD51E,
+0xD5E7, 0xD5AF, 0xD2D5, 0x69D5, 0x8CD5, 0x0FD5, 0xBFD5, 0x1ED5, 0xE7D5, 0xAFD5
+},
+{/*D6*/ 0xD600, 0x00D6, 0xD6D6, 0xD66B, 0xD68D, 0xD6FE, 0xD634, 0xD61A, 0xD668,
+0xD60D, 0xD679, 0x6BD6, 0x8DD6, 0xFED6, 0x34D6, 0x1AD6, 0x68D6, 0x0DD6, 0x79D6
+},
+{/*D7*/ 0xD700, 0x00D7, 0xD7D7, 0xD7D3, 0xD7D1, 0xD7D0, 0xD7F2, 0xD779, 0xD795,
+0xD784, 0xD7E4, 0xD3D7, 0xD1D7, 0xD0D7, 0xF2D7, 0x79D7, 0x95D7, 0x84D7, 0xE4D7
+},
+{/*D8*/ 0xD800, 0x00D8, 0xD8D8, 0xD86C, 0xD836, 0xD81B, 0xD825, 0xD8AA, 0xD84A,
+0xD855, 0xD8F9, 0x6CD8, 0x36D8, 0x1BD8, 0x25D8, 0xAAD8, 0x4AD8, 0x55D8, 0xF9D8
+},
+{/*D9*/ 0xD900, 0x00D9, 0xD9D9, 0xD9D4, 0xD96A, 0xD935, 0xD9E3, 0xD9C9, 0xD9B7,
+0xD9DC, 0xD964, 0xD4D9, 0x6AD9, 0x35D9, 0xE3D9, 0xC9D9, 0xB7D9, 0xDCD9, 0x64D9
+},
+{/*DA*/ 0xDA00, 0x00DA, 0xDADA, 0xDA6D, 0xDA8E, 0xDA47, 0xDAD8, 0xDA6C, 0xDAC1,
+0xDA36, 0xDAB2, 0x6DDA, 0x8EDA, 0x47DA, 0xD8DA, 0x6CDA, 0xC1DA, 0x36DA, 0xB2DA
+},
+{/*DB*/ 0xDB00, 0x00DB, 0xDBDB, 0xDBD5, 0xDBD2, 0xDB69, 0xDB1E, 0xDB0F, 0xDB3C,
+0xDBBF, 0xDB2F, 0xD5DB, 0xD2DB, 0x69DB, 0x1EDB, 0x0FDB, 0x3CDB, 0xBFDB, 0x2FDB
+},
+{/*DC*/ 0xDC00, 0x00DC, 0xDCDC, 0xDC6E, 0xDC37, 0xDCA3, 0xDCAE, 0xDC57, 0xDC2D,
+0xDC93, 0xDC6F, 0x6EDC, 0x37DC, 0xA3DC, 0xAEDC, 0x57DC, 0x2DDC, 0x93DC, 0x6FDC
+},
+{/*DD*/ 0xDD00, 0x00DD, 0xDDDD, 0xDDD6, 0xDD6B, 0xDD8D, 0xDD68, 0xDD34, 0xDDD0,
+0xDD1A, 0xDDF2, 0xD6DD, 0x6BDD, 0x8DDD, 0x68DD, 0x34DD, 0xD0DD, 0x1ADD, 0xF2DD
+},
+{/*DE*/ 0xDE00, 0x00DE, 0xDEDE, 0xDE6F, 0xDE8F, 0xDEFF, 0xDE53, 0xDE91, 0xDEA6,
+0xDEF0, 0xDE24, 0x6FDE, 0x8FDE, 0xFFDE, 0x53DE, 0x91DE, 0xA6DE, 0xF0DE, 0x24DE
+},
+{/*DF*/ 0xDF00, 0x00DF, 0xDFDF, 0xDFD7, 0xDFD3, 0xDFD1, 0xDF95, 0xDFF2, 0xDF5B,
+0xDF79, 0xDFB9, 0xD7DF, 0xD3DF, 0xD1DF, 0x95DF, 0xF2DF, 0x5BDF, 0x79DF, 0xB9DF
+},
+{/*E0*/ 0xE000, 0x00E0, 0xE0E0, 0xE070, 0xE038, 0xE01C, 0xE061, 0xE088, 0xE0C2,
+0xE044, 0xE01B, 0x70E0, 0x38E0, 0x1CE0, 0x61E0, 0x88E0, 0xC2E0, 0x44E0, 0x1BE0
+},
+{/*E1*/ 0xE100, 0x00E1, 0xE1E1, 0xE1C8, 0xE164, 0xE132, 0xE1A7, 0xE1EB, 0xE13F,
+0xE1CD, 0xE186, 0xC8E1, 0x64E1, 0x32E1, 0xA7E1, 0xEBE1, 0x3FE1, 0xCDE1, 0x86E1
+},
+{/*E2*/ 0xE200, 0x00E2, 0xE2E2, 0xE271, 0xE280, 0xE240, 0xE29C, 0xE24E, 0xE249,
+0xE227, 0xE250, 0x71E2, 0x80E2, 0x40E2, 0x9CE2, 0x4EE2, 0x49E2, 0x27E2, 0x50E2
+},
+{/*E3*/ 0xE300, 0x00E3, 0xE3E3, 0xE3C9, 0xE3DC, 0xE36E, 0xE35A, 0xE32D, 0xE3B4,
+0xE3AE, 0xE3CD, 0xC9E3, 0xDCE3, 0x6EE3, 0x5AE3, 0x2DE3, 0xB4E3, 0xAEE3, 0xCDE3
+},
+{/*E4*/ 0xE400, 0x00E4, 0xE4E4, 0xE472, 0xE439, 0xE4A4, 0xE4EA, 0xE475, 0xE4A5,
+0xE482, 0xE48D, 0x72E4, 0x39E4, 0xA4E4, 0xEAE4, 0x75E4, 0xA5E4, 0x82E4, 0x8DE4
+},
+{/*E5*/ 0xE500, 0x00E5, 0xE5E5, 0xE5CA, 0xE565, 0xE58A, 0xE52C, 0xE516, 0xE558,
+0xE50B, 0xE510, 0xCAE5, 0x65E5, 0x8AE5, 0x2CE5, 0x16E5, 0x58E5, 0x0BE5, 0x10E5
+},
+{/*E6*/ 0xE600, 0x00E6, 0xE6E6, 0xE673, 0xE681, 0xE6F8, 0xE617, 0xE6B3, 0xE62E,
+0xE6E1, 0xE6C6, 0x73E6, 0x81E6, 0xF8E6, 0x17E6, 0xB3E6, 0x2EE6, 0xE1E6, 0xC6E6
+},
+{/*E7*/ 0xE700, 0x00E7, 0xE7E7, 0xE7CB, 0xE7DD, 0xE7D6, 0xE7D1, 0xE7D0, 0xE7D3,
+0xE768, 0xE75B, 0xCBE7, 0xDDE7, 0xD6E7, 0xD1E7, 0xD0E7, 0xD3E7, 0x68E7, 0x5BE7
+},
+{/*E8*/ 0xE800, 0x00E8, 0xE8E8, 0xE874, 0xE83A, 0xE81D, 0xE806, 0xE803, 0xE80C,
+0xE8B9, 0xE846, 0x74E8, 0x3AE8, 0x1DE8, 0x06E8, 0x03E8, 0x0CE8, 0xB9E8, 0x46E8
+},
+{/*E9*/ 0xE900, 0x00E9, 0xE9E9, 0xE9CC, 0xE966, 0xE933, 0xE9C0, 0xE960, 0xE9F1,
+0xE930, 0xE9DB, 0xCCE9, 0x66E9, 0x33E9, 0xC0E9, 0x60E9, 0xF1E9, 0x30E9, 0xDBE9
+},
+{/*EA*/ 0xEA00, 0x00EA, 0xEAEA, 0xEA75, 0xEA82, 0xEA41, 0xEAFB, 0xEAC5, 0xEA87,
+0xEADA, 0xEA0D, 0x75EA, 0x82EA, 0x41EA, 0xFBEA, 0xC5EA, 0x87EA, 0xDAEA, 0x0DEA
+},
+{/*EB*/ 0xEB00, 0x00EB, 0xEBEB, 0xEBCD, 0xEBDE, 0xEB6F, 0xEB3D, 0xEBA6, 0xEB7A,
+0xEB53, 0xEB90, 0xCDEB, 0xDEEB, 0x6FEB, 0x3DEB, 0xA6EB, 0x7AEB, 0x53EB, 0x90EB
+},
+{/*EC*/ 0xEC00, 0x00EC, 0xECEC, 0xEC76, 0xEC3B, 0xECA5, 0xEC8D, 0xECFE, 0xEC6B,
+0xEC7F, 0xECD0, 0x76EC, 0x3BEC, 0xA5EC, 0x8DEC, 0xFEEC, 0x6BEC, 0x7FEC, 0xD0EC
+},
+{/*ED*/ 0xED00, 0x00ED, 0xEDED, 0xEDCE, 0xED67, 0xED8B, 0xED4B, 0xED9D, 0xED96,
+0xEDF6, 0xED4D, 0xCEED, 0x67ED, 0x8BED, 0x4BED, 0x9DED, 0x96ED, 0xF6ED, 0x4DED
+},
+{/*EE*/ 0xEE00, 0x00EE, 0xEEEE, 0xEE77, 0xEE83, 0xEEF9, 0xEE70, 0xEE38, 0xEEE0,
+0xEE1C, 0xEE9B, 0x77EE, 0x83EE, 0xF9EE, 0x70EE, 0x38EE, 0xE0EE, 0x1CEE, 0x9BEE
+},
+{/*EF*/ 0xEF00, 0x00EF, 0xEFEF, 0xEFCF, 0xEFDF, 0xEFD7, 0xEFB6, 0xEF5B, 0xEF1D,
+0xEF95, 0xEF06, 0xCFEF, 0xDFEF, 0xD7EF, 0xB6EF, 0x5BEF, 0x1DEF, 0x95EF, 0x06EF
+},
+{/*F0*/ 0xF000, 0x00F0, 0xF0F0, 0xF078, 0xF03C, 0xF01E, 0xF0AF, 0xF0EF, 0xF02F,
+0xF0CF, 0xF0A1, 0x78F0, 0x3CF0, 0x1EF0, 0xAFF0, 0xEFF0, 0x2FF0, 0xCFF0, 0xA1F0
+},
+{/*F1*/ 0xF100, 0x00F1, 0xF1F1, 0xF1C0, 0xF160, 0xF130, 0xF169, 0xF18C, 0xF1D2,
+0xF146, 0xF13C, 0xC0F1, 0x60F1, 0x30F1, 0x69F1, 0x8CF1, 0xD2F1, 0x46F1, 0x3CF1
+},
+{/*F2*/ 0xF200, 0x00F2, 0xF2F2, 0xF279, 0xF284, 0xF242, 0xF252, 0xF229, 0xF2A4,
+0xF2AC, 0xF2EA, 0x79F2, 0x84F2, 0x42F2, 0x52F2, 0x29F2, 0xA4F2, 0xACF2, 0xEAF2
+},
+{/*F3*/ 0xF300, 0x00F3, 0xF3F3, 0xF3C1, 0xF3D8, 0xF36C, 0xF394, 0xF34A, 0xF359,
+0xF325, 0xF377, 0xC1F3, 0xD8F3, 0x6CF3, 0x94F3, 0x4AF3, 0x59F3, 0x25F3, 0x77F3
+},
+{/*F4*/ 0xF400, 0x00F4, 0xF4F4, 0xF47A, 0xF43D, 0xF4A6, 0xF424, 0xF412, 0xF448,
+0xF409, 0xF437, 0x7AF4, 0x3DF4, 0xA6F4, 0x24F4, 0x12F4, 0x48F4, 0x09F4, 0x37F4
+},
+{/*F5*/ 0xF500, 0x00F5, 0xF5F5, 0xF5C2, 0xF561, 0xF588, 0xF5E2, 0xF571, 0xF5B5,
+0xF580, 0xF5AA, 0xC2F5, 0x61F5, 0x88F5, 0xE2F5, 0x71F5, 0xB5F5, 0x80F5, 0xAAF5
+},
+{/*F6*/ 0xF600, 0x00F6, 0xF6F6, 0xF67B, 0xF685, 0xF6FA, 0xF6D9, 0xF6D4, 0xF6C3,
+0xF66A, 0xF67C, 0x7BF6, 0x85F6, 0xFAF6, 0xD9F6, 0xD4F6, 0xC3F6, 0x6AF6, 0x7CF6
+},
+{/*F7*/ 0xF700, 0x00F7, 0xF7F7, 0xF7C3, 0xF7D9, 0xF7D4, 0xF71F, 0xF7B7, 0xF73E,
+0xF7E3, 0xF7E1, 0xC3F7, 0xD9F7, 0xD4F7, 0x1FF7, 0xB7F7, 0x3EF7, 0xE3F7, 0xE1F7
+},
+{/*F8*/ 0xF800, 0x00F8, 0xF8F8, 0xF87C, 0xF83E, 0xF81F, 0xF8C8, 0xF864, 0xF8E1,
+0xF832, 0xF8FC, 0x7CF8, 0x3EF8, 0x1FF8, 0xC8F8, 0x64F8, 0xE1F8, 0x32F8, 0xFCF8
+},
+{/*F9*/ 0xF900, 0x00F9, 0xF9F9, 0xF9C4, 0xF962, 0xF931, 0xF90E, 0xF907, 0xF91C,
+0xF9BB, 0xF961, 0xC4F9, 0x62F9, 0x31F9, 0x0EF9, 0x07F9, 0x1CF9, 0xBBF9, 0x61F9
+},
+{/*FA*/ 0xFA00, 0x00FA, 0xFAFA, 0xFA7D, 0xFA86, 0xFA43, 0xFA35, 0xFAA2, 0xFA6A,
+0xFA51, 0xFAB7, 0x7DFA, 0x86FA, 0x43FA, 0x35FA, 0xA2FA, 0x6AFA, 0x51FA, 0xB7FA
+},
+{/*FB*/ 0xFB00, 0x00FB, 0xFBFB, 0xFBC5, 0xFBDA, 0xFB6D, 0xFBF3, 0xFBC1, 0xFB97,
+0xFBD8, 0xFB2A, 0xC5FB, 0xDAFB, 0x6DFB, 0xF3FB, 0xC1FB, 0x97FB, 0xD8FB, 0x2AFB
+},
+{/*FC*/ 0xFC00, 0x00FC, 0xFCFC, 0xFC7E, 0xFC3F, 0xFCA7, 0xFC43, 0xFC99, 0xFC86,
+0xFCF4, 0xFC6A, 0x7EFC, 0x3FFC, 0xA7FC, 0x43FC, 0x99FC, 0x86FC, 0xF4FC, 0x6AFC
+},
+{/*FD*/ 0xFD00, 0x00FD, 0xFDFD, 0xFDC6, 0xFD63, 0xFD89, 0xFD85, 0xFDFA, 0xFD7B,
+0xFD7D, 0xFDF7, 0xC6FD, 0x63FD, 0x89FD, 0x85FD, 0xFAFD, 0x7BFD, 0x7DFD, 0xF7FD
+},
+{/*FE*/ 0xFE00, 0x00FE, 0xFEFE, 0xFE7F, 0xFE87, 0xFEFB, 0xFEBE, 0xFE5F, 0xFE0D,
+0xFE97, 0xFE21, 0x7FFE, 0x87FE, 0xFBFE, 0xBEFE, 0x5FFE, 0x0DFE, 0x97FE, 0x21FE
+},
+{/*FF*/ 0xFF00, 0x00FF, 0xFFFF, 0xFFC7, 0xFFDB, 0xFFD5, 0xFF78, 0xFF3C, 0xFFF0,
+0xFF1E, 0xFFBC, 0xC7FF, 0xDBFF, 0xD5FF, 0x78FF, 0x3CFF, 0xF0FF, 0x1EFF, 0xBCF
+}
+};
+
+/*
+ * map_syndrome_byte_to_channel
+ *
+ * use the 'row' as a search key (row entry) to being the
+ * scan, looking for a match with 'syndrome'
+ *
+ * Symbols 00h-07h map to data bits 0-63 (channel 0)
+ * Symbols 08h-0Fh map to data bits 62-127 (channel 1)
+ *
+ * Columns 11-18 contain Symbols 00h-07h  (channel 0)
+ * Columns 3-10  contain Symbols 08h-0Fh  (channel 1)
+ * Columns 0-2   contain the ECC Symbols
+ *
+ * Columns 0   1   2   3  4  5  6  7  8  9  10 11 12 13 14 15 16 17 18
+ * Symbols 12h 11h 10h Fh Eh Dh Ch Bh Ah 9h 8h 7h 6h 5h 4h 3h 2h 1h 0h
+ *
+ * return:
+ *	channel number OR
+ *	-1 if not found
+ */
+static int map_syndrome_byte_to_channel(unsigned int syndrome,
+					unsigned int row)
+{
+	int channel = -1;
+	int column;
+
+	/* Scan all rows, looking for syndrome, or end of table */
+	for (column = 0; column < NUMBER_X8_COLS; column++) {
+		if (x8_chipkill_syndromes[row][column] == syndrome) {
+			if ((column >= 11) && (column <= 18))
+				channel = 0;
+			else if ((column >= 3) && (column <= 10))
+				channel = 1;
+			break;
+		}
+	}
+
+	return channel;
+}
+
+/*
+ * get_channel_from_x8_syndrome
+ *
+ *	given the syndrome argument, scan each of the channel tables
+ *	looking for a syndrome match. Depending on which table it is
+ *	found, return the channel number
+ */
+static int get_channel_from_x8_syndrome(unsigned short syndrome)
+{
+	int channel;
+
+	debugf0("%s()\n", __func__);
+
+	/* Try using lower syndrome byte as (row) search key */
+	channel = map_syndrome_byte_to_channel(syndrome, syndrome & 0xFF);
+	if (channel < 0) {
+		/* Try using upper syndrome byte as (row) search key */
+		channel = map_syndrome_byte_to_channel(syndrome,
+						(syndrome >> 16) & 0xFF);
+	}
+
+	debugf0("%s(): syndrome(%x) %s\n", __func__, syndrome,
+		channel < 0 ? "NOT found" : " FOUND");
+
+	return channel;
+}
+
+/*
+ * toString tables for the various error decoding values
+ */
+static const char *tt_msgs[] = {        /* transaction type */
+	"instruction",
+	"data",
+	"generic",
+	"reserved"
+};
+
+static const char *ll_msgs[] = {	/* cache level */
+	"L0",
+	"L1",
+	"L2",
+	"L3/generic"
+};
+
+static const char *rrrr_msgs[] = {
+	"generic",
+	"generic read",
+	"generic write",
+	"data read",
+	"data write",
+	"inst fetch",
+	"prefetch",
+	"evict",
+	"snoop",
+	"reserved RRRR= 9",
+	"reserved RRRR= 10",
+	"reserved RRRR= 11",
+	"reserved RRRR= 12",
+	"reserved RRRR= 13",
+	"reserved RRRR= 14",
+	"reserved RRRR= 15"
+};
+
+static const char *pp_msgs[] = {	/* participating processor */
+	"local node originated (SRC)",
+	"local node responded to request (RES)",
+	"local node observed as 3rd party (OBS)",
+	"generic"
+};
+
+static const char *to_msgs[] = {
+	"no timeout",
+	"timed out"
+};
+
+static const char *ii_msgs[] = {	/* memory or i/o */
+	"mem access",
+	"reserved",
+	"i/o access",
+	"generic"
+};
+
+/* Map the 5 bits of Extended Error code toString table
+ *	enhanced to support the new F10 error codes
+ *	esp, with L3 cache errors
+ */
+static const char *ext_msgs[] = {	/* extended error */
+	"K8 ECC error/F10 reserved",	/* 0_0000b */
+	"CRC error",			/* 0_0001b */
+	"sync error",			/* 0_0010b */
+	"mst abort",			/* 0_0011b */
+	"tgt abort",			/* 0_0100b */
+	"GART error",			/* 0_0101b */
+	"RMW error",			/* 0_0110b */
+	"Wdog timer error",		/* 0_0111b */
+	"F10-ECC/K8-Chipkill error",	/* 0_1000b */
+	"DEV Error",			/* 0_1001b */
+	"Link Data error",		/* 0_1010b */
+	"Link or L3 Protocol error",	/* 0_1011b */
+	"NB Array error",		/* 0_1100b */
+	"DRAM Parity error",		/* 0_1101b */
+	"Link Retry/GART Table Walk/DEV Table Walk error", /* 0_1110b */
+	"Res 0x0ff error",		/* 0_1111b */
+	"Res 0x100 error",		/* 1_0000b */
+	"Res 0x101 error",		/* 1_0001b */
+	"Res 0x102 error",		/* 1_0010b */
+	"Res 0x103 error",		/* 1_0011b */
+	"Res 0x104 error",		/* 1_0100b */
+	"Res 0x105 error",		/* 1_0101b */
+	"Res 0x106 error",		/* 1_0110b */
+	"Res 0x107 error",		/* 1_0111b */
+	"Res 0x108 error",		/* 1_1000b */
+	"Res 0x109 error",		/* 1_1001b */
+	"Res 0x10A error",		/* 1_1010b */
+	"Res 0x10B error",		/* 1_1011b */
+	"L3 Cache Data error",		/* 1_1100b */
+	"L3 CacheTag error",		/* 1_1101b */
+	"L3 Cache LRU error",		/* 1_1110b */
+	"Res 0x1FF error"		/* 1_1111b */
+};
+
+static const char *htlink_msgs[] = {
+	"none",
+	"1",
+	"2",
+	"1 2",
+	"3",
+	"1 3",
+	"2 3",
+	"1 2 3"
+};
+
+/*
+ * The DCSB and DCSM registers differ between Rev E and Rev F CPUs
+ * The following several functions intialize and extract information
+ * from this registers
+ */
+
+/*
+ * k8_set_dctBase_dctMask_rev_specific(pvt)
+ *
+ *	NOTE: CPU Revision Dependent code: Rev E and Rev F
+ *
+ *	Set the DCSB and DCSM mask values depending on the
+ *	CPU revision value.
+ *	Also set the shift factor for the DCSB and DCSM values
+ *
+ *	member dcs_mask_notused, REV E:
+ *
+ *	To find the max InputAddr for the csrow, start with the base
+ *	address and set all bits that are "don't care" bits in the test at
+ *	the start of section 3.5.4 (p. 84).
+ *
+ *	The "don't care" bits are all set bits in the mask and
+ *	all bits in the gaps between bit ranges [35-25] and [19-13].
+ *	The value REV_E_DCS_NOTUSED_BITS represents bits [24-20] and [12-0],
+ *	which are all bits in the above-mentioned gaps.
+ *
+ *	member dcs_mask_notused, REV F:
+ *
+ *	To find the max InputAddr for the csrow, start with the base
+ *	address and set all bits that are "don't care" bits in the test at
+ *	the start of NPT section 4.5.4 (p. 87).
+ *
+ *	The "don't care" bits are all set bits in the mask and
+ *	all bits in the gaps between bit ranges [36-27] and [21-13].
+ *	The value REV_F_DCS_NOTUSED_BITS represents bits [26-22] and [12-0],
+ *	which are all bits in the above-mentioned gaps.
+ */
+static void k8_set_dctBase_dctMask_rev_specific(struct amd64_pvt *pvt)
+{
+	if (pvt->ext_model >= OPTERON_CPU_REV_F) {
+		pvt->dcsb_mask = REV_F_DCSB_BASE_BITS;
+		pvt->dcsm_mask = REV_F_DCSM_MASK_BITS;
+		pvt->dcs_mask_notused = REV_F_DCS_NOTUSED_BITS;
+		pvt->dcs_shift = REV_F_DCS_SHIFT;
+		pvt->dcsm_shift_bit = REV_F_DCSM_SHIFT;
+		pvt->num_dcsm = REV_F_DCSM_COUNT;
+	} else {
+		pvt->dcsb_mask = REV_E_DCSB_BASE_BITS;
+		pvt->dcsm_mask = REV_E_DCSM_MASK_BITS;
+		pvt->dcs_mask_notused = REV_E_DCS_NOTUSED_BITS;
+		pvt->dcs_shift = REV_E_DCS_SHIFT;
+		pvt->dcsm_shift_bit = REV_E_DCSM_SHIFT;
+		pvt->num_dcsm = REV_E_DCSM_COUNT;
+	}
+}
+
+/*
+ * f10_set_dctBase_dctMask_rev_specific
+ *
+ *	setup F10 values of masks, shifts and counts for this family of CPUs
+ *	concerning the DRAM BASE and MASK registers
+ */
+static void f10_set_dctBase_dctMask_rev_specific(struct amd64_pvt *pvt)
+{
+		pvt->dcsb_mask = REV_F10_DCSB_BASE_BITS;
+		pvt->dcsm_mask = REV_F10_DCSM_MASK_BITS;
+		pvt->dcs_mask_notused = REV_F10_DCS_NOTUSED_BITS;
+		pvt->dcs_shift = REV_F10_DCS_SHIFT;
+		pvt->dcsm_shift_bit = REV_F10_DCSM_SHIFT;
+		pvt->num_dcsm = REV_F10_DCSM_COUNT;
+}
+
+/*
+ * map_to_dcs_mask
+ *
+ *	Map from a CSROW entry to the mask entry that operates on it
+ */
+static inline u32 map_to_dcs_mask(struct amd64_pvt *pvt, int csrow)
+{
+	return csrow >> pvt->dcsm_shift_bit;
+}
+
+/*
+ * get_dctBase()
+ *
+ *	getter function to return the 'base' address the i'th CS entry
+ *	of the 'dct' DRAM controller
+ */
+static u32 get_dctBase(struct amd64_pvt *pvt, int dct, int csrow)
+{
+	if (dct == 0)
+		return pvt->dcsb0[csrow];
+	else
+		return pvt->dcsb1[csrow];
+}
+
+/*
+ * get_dctMask()
+ *
+ *	getter function to return the 'mask' address the i'th CS entry.
+ *	This getter function is needed because there different number
+ *	of DCSM registers on Rev E and prior vs Rev F and later
+ */
+static u32 get_dctMask(struct amd64_pvt *pvt, int dct, int csrow)
+{
+	if (dct == 0)
+		return pvt->dcsm0[map_to_dcs_mask(pvt, csrow)];
+	else
+		return pvt->dcsm1[map_to_dcs_mask(pvt, csrow)];
+}
+
+/*
+ * base_from_dctBase
+ *
+ *	Extract the DRAM CS base address from selected csrow register
+ */
+static u64 base_from_dctBase(struct amd64_pvt *pvt, int csrow)
+{
+	return ((u64) (get_dctBase(pvt, 0, csrow) & pvt->dcsb_mask)) <<
+				pvt->dcs_shift;
+}
+
+/*
+ * mask_from_dctMask
+ *
+ *	Extract the Mask from the dcsb0[csrow] entry
+ *	Depends on CPU Revision on how to extract this information
+ */
+static u64 mask_from_dctMask(struct amd64_pvt *pvt, int csrow)
+{
+	u64 dcsm_bits, other_bits;
+	u64 mask;
+
+	/* Extract bits bits 29-21 and 15-9 from DCSM (section 3.5.5). */
+	dcsm_bits = get_dctMask(pvt, 0, csrow) & pvt->dcsm_mask;
+
+	/* Set all bits except bits 33-25 and 19-13. */
+	other_bits = pvt->dcsm_mask;
+	other_bits = ~(other_bits << pvt->dcs_shift);
+
+	/* The extracted bits from DCSM belong in the spaces represented by
+	 * the cleared bits in other_bits.
+	 */
+	mask = (dcsm_bits << pvt->dcs_shift) | other_bits;
+
+	return mask;
+}
+
+/*
+ * k8_read_dctBase_dctMask()
+ *
+ *	Function 2 Offset K8_DCSB0
+ *	Setup the DCSB and DCSM arrays from hardware
+ */
+static void k8_read_dctBase_dctMask(struct amd64_pvt *pvt)
+{
+	struct pci_dev *ctl = pvt->dram_f2_ctl;
+	int cs;
+	int err;
+
+	/* Set the dcsb and dcsm mask bits and their shift value */
+	k8_set_dctBase_dctMask_rev_specific(pvt);
+
+	/* Retrieve the DRAM CS Base Address Registers from hardware */
+	for (cs = 0; cs < CHIPSELECT_COUNT; cs++) {
+		err = pci_read_config_dword(ctl, K8_DCSB0 + (cs * 4),
+					&pvt->dcsb0[cs]);
+		if (err != 0)
+			debugf0("%s() Reading K8_DCSB0 failed\n", __func__);
+
+	}
+
+	/* The number of DCSMs differents at the Rev E/Rev F boundary
+	 * so we retrieve the number of registers defined for this processor
+	 */
+	for (cs = 0; cs < pvt->num_dcsm; cs++) {
+		err = pci_read_config_dword(ctl, K8_DCSM0 + (cs * 4),
+					&pvt->dcsm0[cs]);
+		if (err != 0)
+			debugf0("%s() Reading K8_DCSM0 failed\n", __func__);
+	}
+
+	/* Debug dump the DCSB and DCSM registers */
+	for (cs = 0; cs < CHIPSELECT_COUNT; cs++) {
+		debugf1("  dcsb0[%d]: 0x%8.8x  dcsm0[%d]: 0x%x\n",
+			cs, get_dctBase(pvt, 0, cs),
+			map_to_dcs_mask(pvt, cs), get_dctMask(pvt, 0, cs));
+	}
+}
+
+/*
+ * f10_read_dctBase_dctMask
+ *
+ *	Function 2 Offset F10_DCSB0
+ *	Read in the DCS Base and DCS Mask hw registers
+ */
+static void f10_read_dctBase_dctMask(struct amd64_pvt *pvt)
+{
+	struct pci_dev *dram_f2_ctl = pvt->dram_f2_ctl;
+	int DctGangedEnabled = pvt->dram_dct_ganged_enable;
+	int cs;
+	int err;
+	int reg;
+
+	debugf0("%s()\n", __func__);
+
+	/* Set the dcsb and dcsm mask bits and their shift value */
+	f10_set_dctBase_dctMask_rev_specific(pvt);
+
+	/* Retrieve the DRAM CS Base Address Registers from hardware */
+	for (cs = 0; cs < CHIPSELECT_COUNT; cs++) {
+		reg = K8_DCSB0 + (cs * 4);
+		err = pci_read_config_dword(dram_f2_ctl, reg, &pvt->dcsb0[cs]);
+		if (err != 0)
+			debugf0("%s() Reading K8_DCSB0[%d] failed\n",
+				__func__, cs);
+
+		debugf0("  DCSB0[%d]=0x%08x reg=0x%x\n",
+			cs, pvt->dcsb0[cs], reg);
+
+		/* If DCT are NOT ganged, then read in DCT1's base */
+		if (!DctGangedEnabled) {
+			reg = F10_DCSB1 + (cs * 4);
+			err = pci_read_config_dword(dram_f2_ctl,
+					reg, &pvt->dcsb1[cs]);
+			if (err != 0)
+				debugf0("%s() Reading F10_DCSB1[%d] failed\n",
+					__func__, cs);
+			debugf0("  DCSB1[%d]=0x%08x reg=0x%x\n",
+				cs, pvt->dcsb1[cs], reg);
+		} else {
+			pvt->dcsb1[cs] = 0;
+		}
+	}
+
+	/*
+	 * Read the Mask registers
+	 */
+	for (cs = 0; cs < pvt->num_dcsm; cs++) {
+		err = pci_read_config_dword(dram_f2_ctl, K8_DCSM0 + (cs * 4),
+					&pvt->dcsm0[cs]);
+		if (err != 0)
+			debugf0("%s() Reading K8_DCSM0 failed\n", __func__);
+		else
+			debugf0("    DCSM0[%d]=0x%08x\n", cs, pvt->dcsm0[cs]);
+
+		/* If DCT are NOT ganged, then read in DCT1's mask */
+		if (!DctGangedEnabled) {
+			err = pci_read_config_dword(dram_f2_ctl,
+					F10_DCSM1 + (cs * 4),
+					&pvt->dcsm1[cs]);
+			if (err != 0)
+				debugf0("%s() Reading F10_DCSM1 failed\n",
+					__func__);
+			else
+				debugf0("    DCSM1[%d]=0x%08x\n",
+					cs, pvt->dcsm1[cs]);
+		} else
+			pvt->dcsm1[cs] = 0;
+	}
+}
+
+/*
+ * get_base_and_limit()
+ *
+ * In *base and *limit, pass back the full 40-bit base and limit physical
+ * addresses for the node given by node_id.  This information is obtained from
+ * DRAM Base (section 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers. The
+ * base and limit addresses are of type SysAddr, as defined at the start of
+ * section 3.4.4 (p. 70).  They are the lowest and highest physical addresses
+ * in the address range they represent.
+ */
+static void get_base_and_limit(struct amd64_pvt *pvt, int node_id,
+			       u64 *base, u64 *limit)
+{
+	*base = pvt->dram_base[node_id];
+	*limit = pvt->dram_limit[node_id];
+}
+
+/* Return 1 if the SysAddr given by sys_addr matches the base/limit associated
+ * with node_id
+ */
+static int base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, int node_id)
+{
+	u64 base, limit, addr;
+
+	get_base_and_limit(pvt, node_id, &base, &limit);
+
+	/* The K8 treats this as a 40-bit value.  However, bits 63-40 will be
+	 * all ones if the most significant implemented address bit is 1.
+	 * Here we discard bits 63-40.  See section 3.4.2 of AMD publication
+	 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
+	 * Application Programming.
+	 */
+	addr = sys_addr & 0x000000ffffffffffull;
+
+	return (addr >= base) && (addr <= limit);
+}
+
+/* find_mc_by_sys_addr
+ *
+ *	Attempt to map a SysAddr to a node.
+ *
+ *	On success, return a pointer to the mem_ctl_info structure for
+ *	the node that the SysAddr maps to.
+ *
+ *	On failure, return NULL
+ */
+static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
+						u64 sys_addr)
+{
+	struct amd64_pvt *pvt;
+	int node_id;
+	u32 intlv_en, bits;
+
+	/* Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
+	 * 3.4.4.2) registers to map the SysAddr to a node ID.
+	 */
+
+	pvt = mci->pvt_info;
+
+	/* The value of this field should be the same for all DRAM Base
+	 * registers.  Therefore we arbitrarily choose to read it from the
+	 * register for node 0.
+	 */
+	intlv_en = pvt->dram_IntlvEn[0];
+
+	if (intlv_en == 0) {	/* node interleaving is disabled */
+		debugf2("%s(): node interleaving disabled\n", __func__);
+		for (node_id = 0; ; ) {
+			if (base_limit_match(pvt, sys_addr, node_id))
+				break;
+
+			if (++node_id >= DRAM_REG_COUNT) {
+				debugf2("%s(): sys_addr 0x%lx "
+					"does not match any node\n", __func__,
+					(unsigned long)sys_addr);
+				return NULL;
+			}
+		}
+
+		goto found;
+	}
+
+	if (unlikely((intlv_en != (0x01 << 8)) &&
+		     (intlv_en != (0x03 << 8)) && (intlv_en != (0x07 << 8)))) {
+		amd64_printk(KERN_WARNING,
+			  "%s(): junk value of 0x%x extracted from IntlvEn "
+			  "field of DRAM Base Register for node 0: This "
+			  "probably indicates a BIOS bug.\n", __func__,
+			  intlv_en);
+		return NULL;
+	}
+
+	/* If we get this far, node interleaving is enabled. */
+	debugf2("%s(): node interleaving enabled\n", __func__);
+	bits = (((u32) sys_addr) >> 12) & intlv_en;
+
+	for (node_id = 0; ; ) {
+		if ((pvt->dram_limit[node_id] & intlv_en) == bits)
+			break;	/* intlv_sel field matches */
+
+		if (++node_id >= DRAM_REG_COUNT) {
+			debugf2("%s(): sys_addr 0x%lx does not match any "
+				"node\n", __func__, (unsigned long)sys_addr);
+			return NULL;
+		}
+	}
+
+	/* sanity test for sys_addr */
+	if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
+		amd64_printk(KERN_WARNING,
+			  "%s(): sys_addr 0x%lx falls outside base/limit "
+			  "address range for node %d with node interleaving "
+			  "enabled.\n", __func__, (unsigned long)sys_addr,
+			  node_id);
+		return NULL;
+	}
+
+found:
+	debugf2("%s(): sys_addr 0x%lx matches node %d\n", __func__,
+		(unsigned long)sys_addr, node_id);
+
+	return edac_mc_find(node_id);
+}
+
+/* Return the base value defined by the DRAM Base register for the node
+ * represented by mci.  This function returns the full 40-bit value despite
+ * the fact that the register only stores bits 39-24 of the value.  See
+ * section 3.4.4.1.
+ */
+static inline u64 get_dram_base(struct mem_ctl_info *mci)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	return pvt->dram_base[pvt->mc_node_id];
+}
+
+/* Obtain info from the DRAM Hole Address Register (section 3.4.8) for the
+ * node represented by mci.  Info is passed back in *hole_base, *hole_offset,
+ * and *hole_size.  Function returns 0 if info is valid or 1 if info is
+ * invalid.  Info may be invalid for either of the following reasons:
+ *
+ *     - The revision of the node is not E or greater.  In this case, the DRAM
+ *       Hole Address Register does not exist.
+ *     - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
+ *       indicating that its contents are not valid.
+ *
+ * The values passed back in *hole_base, *hole_offset, and *hole_size are
+ * complete 32-bit values despite the fact that the bitfields in the DHAR
+ * only represent bits 31-24 of the base and offset values.
+ */
+static int k8_get_dram_hole_info(struct mem_ctl_info *mci,
+			u64 *hole_base, u64 *hole_offset, u64 *hole_size)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	u64 base;
+
+	/* Only Rev E and later have the DRAM Hole Address Register */
+	if (pvt->ext_model < OPTERON_CPU_REV_E) {
+		debugf1("  revision %d for node %d does not support DHAR\n",
+			pvt->ext_model, pvt->mc_node_id);
+		return 1;
+	}
+
+	/* If the DRAM Hole Address Register is OFF, return that status */
+	if ((pvt->dhar & K8_DHAR_VALID) == 0) {
+		debugf1("  DRAM Memory Hoisting is DISABLED on this node %d\n",
+			pvt->mc_node_id);
+		return 1;       /* DramHoleValid bit is cleared */
+	}
+
+	/* +------------------+--------------------+--------------------+-----
+	 * | memory           | DRAM hole          | relocated          |
+	 * | [0, (x - 1)]     | [x, 0xffffffff]    | addresses from     |
+	 * |                  |                    | DRAM hole          |
+	 * |                  |                    | [0x100000000,      |
+	 * |                  |                    |  (0x100000000+     |
+	 * |                  |                    |   (0xffffffff-x))] |
+	 * +------------------+--------------------+--------------------+-----
+	 *
+	 * Above is a diagram of physical memory showing the DRAM hole and the
+	 * relocated addresses from the DRAM hole.  As shown, the DRAM hole
+	 * starts at address x (the base address) and extends through address
+	 * 0xffffffff.  The DRAM Hole Address Register (DHAR) relocates the
+	 * addresses in the hole so that they start at 0x100000000.
+	 */
+
+	base = EXTRACT_K8_DHAR_BASE(pvt->dhar);
+
+	*hole_base = base;
+	*hole_offset = EXTRACT_K8_DHAR_OFFSET(pvt->dhar);
+	*hole_size = (0x1ull << 32) - base;
+
+	debugf1("  DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
+		pvt->mc_node_id, (unsigned long)*hole_base,
+		(unsigned long)*hole_offset, (unsigned long)*hole_size);
+
+	return 0;
+}
+
+/*
+ *	f10_get_dram_hole_info
+ */
+static int f10_get_dram_hole_info(struct mem_ctl_info *mci,
+			u64 *hole_base, u64 *hole_offset, u64 *hole_size)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	u64 base;
+
+	/*
+	 * Check if DRAM Memory Hoisting if OFF on this system
+	 */
+	if ((pvt->dhar & F10_DRAM_MEM_HOIST_VALID) == 0) {
+		debugf1("  Dram Memory Hoisting is DISABLED on this system\n");
+		return 1;
+	}
+
+	/* Check if this NODE has Hoisting OFF */
+	if ((pvt->dhar & K8_DHAR_VALID) == 0) {
+		debugf1("  Dram Memory Hoisting is DISABLED on this node %d\n",
+			pvt->mc_node_id);
+		return 1;	/* DramHoleValid bit is cleared */
+	}
+
+	/* This node has Memory Hoisting */
+
+	/* +------------------+--------------------+--------------------+-----
+	 * | memory           | DRAM hole          | relocated          |
+	 * | [0, (x - 1)]     | [x, 0xffffffff]    | addresses from     |
+	 * |                  |                    | DRAM hole          |
+	 * |                  |                    | [0x100000000,      |
+	 * |                  |                    |  (0x100000000+     |
+	 * |                  |                    |   (0xffffffff-x))] |
+	 * +------------------+--------------------+--------------------+-----
+	 *
+	 * Above is a diagram of physical memory showing the DRAM hole and the
+	 * relocated addresses from the DRAM hole.  As shown, the DRAM hole
+	 * starts at address x (the base address) and extends through address
+	 * 0xffffffff.  The DRAM Hole Address Register (DHAR) relocates the
+	 * addresses in the hole so that they start at 0x100000000.
+	 */
+
+	base = EXTRACT_F10_DHAR_BASE(pvt->dhar);
+
+	*hole_base = base;
+	*hole_offset = EXTRACT_F10_DHAR_OFFSET(pvt->dhar);
+	*hole_size = (0x1ull << 32) - base;
+
+	debugf1("  DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
+		pvt->mc_node_id, (unsigned long)*hole_base,
+		(unsigned long)*hole_offset, (unsigned long)*hole_size);
+
+	return 0;
+}
+
+/*
+ * k8_decode_misc_regs
+ *
+ *	for debug purposes, display and decode various NB registers that
+ *	are for the k8 family.
+ *
+ *	This function become a no-op when DEBUG is disabled
+ */
+static void k8_decode_misc_regs(struct amd64_pvt *pvt)
+{
+	debugf1("  nbcap:0x%8.08x DctDualCap=%s DualNode=%s 8-Node=%s\n",
+		pvt->nbcap,
+		(pvt->nbcap & K8_NBCAP_DCT_DUAL) ? "True" : "False",
+		(pvt->nbcap & K8_NBCAP_DUAL_NODE) ? "True" : "False",
+		(pvt->nbcap & K8_NBCAP_8_NODE) ? "True" : "False");
+	debugf1("    ECC Capable=%s   ChipKill Capable=%s\n",
+		(pvt->nbcap & K8_NBCAP_SECDED) ? "True" : "False",
+		(pvt->nbcap & K8_NBCAP_CHIPKILL) ? "True" : "False");
+	debugf1("  DramCfg0-low=0x%08x DIMM-ECC=%s Parity=%s Width=%s\n",
+		pvt->dcl0,
+		(pvt->dcl0 & BIT(19)) ?  "Enabled" : "Disabled",
+		(pvt->dcl0 & BIT(8)) ?  "Enabled" : "Disabled",
+		(pvt->dcl0 & BIT(11)) ?  "128b" : "64b");
+	debugf1("    DIMM x4 Present: L0=%s L1=%s L2=%s L3=%s  DIMM Type=%s\n",
+		(pvt->dcl0 & BIT(12)) ?  "Y" : "N",
+		(pvt->dcl0 & BIT(13)) ?  "Y" : "N",
+		(pvt->dcl0 & BIT(14)) ?  "Y" : "N",
+		(pvt->dcl0 & BIT(15)) ?  "Y" : "N",
+		(pvt->dcl0 & BIT(16)) ?  "UN-Buffered" : "Buffered");
+
+	/* K8 DHAR regiseter */
+	debugf1("  dhar: 0x%8.08x Base=0x%08x Offset=0x%08x\n",
+		pvt->dhar, EXTRACT_K8_DHAR_BASE(pvt->dhar),
+		EXTRACT_K8_DHAR_OFFSET(pvt->dhar));
+	debugf1("      DramHoleValid=%s\n",
+		(pvt->dhar & K8_DHAR_VALID) ?  "True" : "False");
+
+	debugf1("  dbam-dkt: 0x%8.08x\n", pvt->dbam0);
+}
+
+/*
+ * debug_display_dimm_sizes
+ *
+ *	debug routine to display the memory sizes of a DIMM
+ *	(ganged or not) and it CSROWs as well
+ */
+static void debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt, int ganged)
+{
+	int dimm;
+	int size0;
+	int size1;
+	u32 dbam;
+	u32 *dcsb;
+
+	debugf1("  dbam%d: 0x%8.08x  CSROW is %s\n", ctrl,
+			ctrl ? pvt->dbam1 : pvt->dbam0,
+			ganged ? "GANGED - dbam1 not used" : "NON-GANGED");
+
+	/* Extract which controller reg, based on 'ctrl' selected */
+	dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
+	dcsb = ctrl ? pvt->dcsb1 : pvt->dcsb0;
+
+	/* Dump memory sizes for DIMM and its CSROWs */
+	for (dimm = 0; dimm < 4; dimm++) {
+
+		size0 = 0;
+		if (dcsb[dimm*2] & K8_DCSB_CS_ENABLE)
+			size0 = map_dbam_to_csrow_size(DBAM_DIMM(dimm, dbam));
+
+		size1 = 0;
+		if (dcsb[(dimm*2)+1] & K8_DCSB_CS_ENABLE)
+			size1 = map_dbam_to_csrow_size(DBAM_DIMM(dimm, dbam));
+
+		debugf1("     CTRL-%d DIMM-%d=%5dMB   CSROW-%d=%5dMB "
+				"CSROW-%d=%5dMB\n",
+				ctrl,
+				dimm,
+				size0 + size1,
+				dimm * 2,
+				size0,
+				(dimm * 2) + 1,
+				size1);
+	}
+}
+
+/*
+ * f10_decode_misc_regs
+ *
+ *	for debug purposes, display and decode various NB registers that
+ *	are for the f10 family.
+ *
+ *	This function become a no-op when DEBUG is disabled
+ */
+static void f10_decode_misc_regs(struct amd64_pvt *pvt)
+{
+	int ganged;
+
+	debugf1("  nbcap:0x%8.08x DctDualCap=%s DualNode=%s 8-Node=%s\n",
+		pvt->nbcap,
+		(pvt->nbcap & K8_NBCAP_DCT_DUAL) ? "True" : "False",
+		(pvt->nbcap & K8_NBCAP_DUAL_NODE) ? "True" : "False",
+		(pvt->nbcap & K8_NBCAP_8_NODE) ? "True" : "False");
+	debugf1("    ECC Capable=%s   ChipKill Capable=%s\n",
+		(pvt->nbcap & K8_NBCAP_SECDED) ? "True" : "False",
+		(pvt->nbcap & K8_NBCAP_CHIPKILL) ? "True" : "False");
+	debugf1("  DramCfg0-low=0x%08x DIMM-ECC=%s Parity=%s Width=%s\n",
+		pvt->dcl0,
+		(pvt->dcl0 & BIT(19)) ?  "Enabled" : "Disabled",
+		(pvt->dcl0 & BIT(8)) ?  "Enabled" : "Disabled",
+		(pvt->dcl0 & BIT(11)) ?  "128b" : "64b");
+	debugf1("    DIMM x4 Present: L0=%s L1=%s L2=%s L3=%s  DIMM Type=%s\n",
+		(pvt->dcl0 & BIT(12)) ?  "Y" : "N",
+		(pvt->dcl0 & BIT(13)) ?  "Y" : "N",
+		(pvt->dcl0 & BIT(14)) ?  "Y" : "N",
+		(pvt->dcl0 & BIT(15)) ?  "Y" : "N",
+		(pvt->dcl0 & BIT(16)) ?  "UN-Buffered" : "Buffered");
+
+	/* Only if NOT ganged does dcl1 have valid info */
+	if (!pvt->dram_dct_ganged_enable) {
+		debugf1("  DramCfg1-low=0x%08x DIMM-ECC=%s Parity=%s "
+			"Width=%s\n", pvt->dcl1,
+			(pvt->dcl1 & BIT(19)) ?  "Enabled" : "Disabled",
+			(pvt->dcl1 & BIT(8)) ?  "Enabled" : "Disabled",
+			(pvt->dcl1 & BIT(11)) ?  "128b" : "64b");
+		debugf1("    DIMM x4 Present: L0=%s L1=%s L2=%s L3=%s  "
+			"DIMM Type=%s\n",
+			(pvt->dcl1 & BIT(12)) ?  "Y" : "N",
+			(pvt->dcl1 & BIT(13)) ?  "Y" : "N",
+			(pvt->dcl1 & BIT(14)) ?  "Y" : "N",
+			(pvt->dcl1 & BIT(15)) ?  "Y" : "N",
+			(pvt->dcl1 & BIT(16)) ?  "UN-Buffered" : "Buffered");
+	}
+
+	/* F10 DHAR register */
+	debugf1("  dhar: 0x%8.08x Base=0x%08x Offset=0x%08x\n",
+		pvt->dhar, EXTRACT_F10_DHAR_BASE(pvt->dhar),
+		EXTRACT_F10_DHAR_OFFSET(pvt->dhar));
+	debugf1("    DramMemHoistValid=%s DramHoleValid=%s\n",
+		(pvt->dhar & F10_DHAR_MEM_HOIST_VALID) ?
+			"True" : "False",
+		(pvt->dhar & K8_DHAR_VALID) ?
+			"True" : "False");
+
+	/* Determine if ganged and then dump memory sizes for
+	 * first controller, and if NOT ganged dump info for 2nd controller
+	 */
+	ganged = pvt->dram_dct_ganged_enable;
+	debug_display_dimm_sizes(0, pvt, ganged);
+
+	if (!ganged)
+		debug_display_dimm_sizes(1, pvt, ganged);
+
+	debugf1("  online-spare: 0x%8.08x\n", pvt->online_spare);
+}
+
+/* Return the DramAddr that the SysAddr given by sys_addr maps to.  It is
+ * assumed that sys_addr maps to the node given by mci.
+ */
+static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
+	int rc;
+
+	/* The first part of section 3.4.4 (p. 70) shows how the DRAM Base
+	 * (section 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are
+	 * used to translate a SysAddr to a DramAddr.  If the DRAM Hole
+	 * Address Register (DHAR) is enabled, then it is also involved in
+	 * translating a SysAddr to a DramAddr.  Sections 3.4.8 and 3.5.8.2
+	 * describe the DHAR and how it is used for memory hoisting.  These
+	 * parts of the documentation are unclear.  I interpret them as
+	 * follows:
+	 *
+	 *     When node n receives a SysAddr, it processes the SysAddr as
+	 *     follows:
+	 *
+	 *         1.  It extracts the DRAMBase and DRAMLimit values from the
+	 *             DRAM Base and DRAM Limit registers for node n.  If the
+	 *             SysAddr is not within the range specified by the base
+	 *             and limit values, then node n ignores the Sysaddr
+	 *             (since it does not map to node n).  Otherwise continue
+	 *             to step 2 below.
+	 *
+	 *         2.  If the DramHoleValid bit of the DHAR for node n is
+	 *             clear, the DHAR is disabled so skip to step 3 below.
+	 *             Otherwise see if the SysAddr is within the range of
+	 *             relocated addresses (starting at 0x100000000) from the
+	 *             DRAM hole.  If not, skip to step 3 below.  Else get the
+	 *             value of the DramHoleOffset field from the DHAR.  To
+	 *             obtain the DramAddr, subtract the offset defined by
+	 *             this value from the SysAddr.
+	 *
+	 *         3.  Obtain the base address for node n from the DRAMBase
+	 *             field of the DRAM Base register for node n.  To obtain
+	 *             the DramAddr, subtract the base address from the
+	 *             SysAddr, as shown near the start of section 3.4.4
+	 *             (p. 70).
+	 */
+
+	dram_base = get_dram_base(mci);
+
+	rc = pvt->ops->get_dram_hole_info(mci,
+				&hole_base, &hole_offset, &hole_size);
+	if (!rc) {
+		if ((sys_addr >= (1ull << 32)) &&
+		    (sys_addr < ((1ull << 32) + hole_size))) {
+			/* use DHAR to translate SysAddr to DramAddr */
+			dram_addr = sys_addr - hole_offset;
+			debugf2("using DHAR to translate SysAddr 0x%lx to "
+				"DramAddr 0x%lx\n",
+				(unsigned long)sys_addr,
+				(unsigned long)dram_addr);
+			return dram_addr;
+		}
+	}
+
+	/* Translate the SysAddr to a DramAddr as shown near the start of
+	 * section 3.4.4 (p. 70).  Although sys_addr is a 64-bit value, the k8
+	 * only deals with 40-bit values.  Therefore we discard bits 63-40 of
+	 * sys_addr below.  If bit 39 of sys_addr is 1 then the bits we
+	 * discard are all 1s.  Otherwise the bits we discard are all 0s.  See
+	 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
+	 * Programmer's Manual Volume 1 Application Programming.
+	 */
+	dram_addr = (sys_addr & 0xffffffffffull) - dram_base;
+
+	debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
+		"DramAddr 0x%lx\n", (unsigned long)sys_addr,
+		(unsigned long)dram_addr);
+	return dram_addr;
+}
+
+/* Parameter intlv_en is the value of the IntlvEn field from a DRAM Base
+ * register (section 3.4.4.1).  Return the number of bits from a SysAddr that
+ * are used for node interleaving.
+ */
+static int num_node_interleave_bits(unsigned intlv_en)
+{
+	static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
+	int n;
+
+	BUG_ON(intlv_en > 7);
+	n = intlv_shift_table[intlv_en];
+	return n;
+}
+
+/* Translate the DramAddr given by dram_addr to an InputAddr and return the
+ * result.
+ */
+static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
+{
+	struct amd64_pvt *pvt;
+	int intlv_shift;
+	u64 input_addr;
+
+	pvt = mci->pvt_info;
+
+	/* Near the start of section 3.4.4 (p. 70), the k8 documentation gives
+	 * instructions for translating a DramAddr to an InputAddr.  Here we
+	 * are following these instructions.
+	 */
+	intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]);
+	input_addr = ((dram_addr >> intlv_shift) & 0xffffff000ull) +
+	    (dram_addr & 0xfff);
+
+	debugf2("  Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
+		intlv_shift,
+		(unsigned long)dram_addr, (unsigned long)input_addr);
+	return input_addr;
+}
+
+/* Translate the SysAddr represented by sys_addr to an InputAddr and return
+ * the result.  It is assumed that sys_addr maps to the node given by mci.
+ */
+static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
+{
+	u64 input_addr;
+
+	input_addr =
+	    dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
+	debugf2("%s(): SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
+		__func__, (unsigned long)sys_addr, (unsigned long)input_addr);
+	return input_addr;
+}
+
+/* input_addr is an InputAddr associated with the node given by mci.  Return
+ * the csrow that input_addr maps to, or -1 on failure (no csrow claims
+ * input_addr).
+ */
+static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
+{
+	struct amd64_pvt *pvt;
+	int csrow;
+	u64 base, mask;
+
+	pvt = mci->pvt_info;
+
+	/* Here we use the DRAM CS Base (section 3.5.4) and DRAM CS Mask
+	 * (section 3.5.5) registers.  For each CS base/mask register pair,
+	 * test the condition shown near the start of section 3.5.4 (p. 84).
+	 */
+
+	for (csrow = 0; csrow < CHIPSELECT_COUNT; csrow++) {
+
+		if ((pvt->dcsb0[csrow] & K8_DCSB_CS_ENABLE) == 0) {
+			debugf2("input_addr_to_csrow: CSBE bit is cleared "
+				"for csrow %d (node %d)\n",
+				csrow, pvt->mc_node_id);
+			continue;	/* CSBE bit is cleared, no csrow */
+		}
+
+		/* Get the base addr and the mask values for this csrow */
+		base = base_from_dctBase(pvt, csrow);
+		mask = ~mask_from_dctMask(pvt, csrow);
+
+		if ((input_addr & mask) == (base & mask)) {
+			debugf2("InputAddr 0x%lx matches csrow %d "
+				"(MC node %d)\n",
+				(unsigned long)input_addr,
+				csrow, pvt->mc_node_id);
+			return csrow;	/* success: csrow matches */
+		}
+	}
+
+	debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
+		(unsigned long)input_addr, pvt->mc_node_id);
+	return -1;		/* failed to find matching csrow */
+}
+
+/* input_addr is an InputAddr associated with the node represented by mci.
+ * Translate input_addr to a DramAddr and return the result.
+ */
+static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
+{
+	struct amd64_pvt *pvt;
+	int node_id, intlv_shift;
+	u64 bits, dram_addr;
+	u32 intlv_sel;
+
+	/* Near the start of section 3.4.4 (p. 70), the k8 documentation shows
+	 * how to translate a DramAddr to an InputAddr.  Here we reverse this
+	 * procedure.  When translating from a DramAddr to an InputAddr, the
+	 * bits used for node interleaving are discarded.  Here we recover
+	 * these bits from the IntlvSel field of the DRAM Limit register
+	 * (section 3.4.4.2) for the node that input_addr is associated with.
+	 */
+
+	pvt = mci->pvt_info;
+	node_id = pvt->mc_node_id;
+	BUG_ON((node_id < 0) || (node_id > 7));
+
+	intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]);
+
+	if (intlv_shift == 0) {
+		debugf1("  node interleaving disabled:\n");
+		debugf1("    InputAddr 0x%lx translates "
+			"to DramAddr of same value\n",
+			(unsigned long)input_addr);
+		return input_addr;
+	}
+
+	bits = ((input_addr & 0xffffff000ull) << intlv_shift) +
+	    (input_addr & 0xfff);
+
+	intlv_sel = pvt->dram_IntlvSel[node_id] & ((1 << intlv_shift) - 1);
+	dram_addr = bits + (intlv_sel << 12);
+
+	debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
+		"(%d node interleave bits)\n", (unsigned long)input_addr,
+		(unsigned long)dram_addr, intlv_shift);
+	return dram_addr;
+}
+
+/* dram_addr is a DramAddr that maps to the node represented by mci.  Convert
+ * dram_addr to a SysAddr and return the result.
+ */
+static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	u64 hole_base, hole_offset, hole_size, base, limit, sys_addr;
+	int rc;
+
+	rc = pvt->ops->get_dram_hole_info(mci,
+				&hole_base, &hole_offset, &hole_size);
+	if (!rc) {
+		if ((dram_addr >= hole_base) &&
+		    (dram_addr < (hole_base + hole_size))) {
+			/* use DHAR to translate DramAddr to SysAddr */
+			sys_addr = dram_addr + hole_offset;
+			debugf1("using DHAR to translate DramAddr 0x%lx to "
+				"SysAddr 0x%lx\n", (unsigned long)dram_addr,
+				(unsigned long)sys_addr);
+			return sys_addr;
+		}
+	}
+
+	get_base_and_limit(pvt, pvt->mc_node_id, &base, &limit);
+	sys_addr = dram_addr + base;
+
+	/* The sys_addr we have computed up to this point is a 40-bit value
+	 * because the k8 deals with 40-bit values.  However, the value we are
+	 * supposed to return is a full 64-bit physical address.  The AMD
+	 * x86-64 architecture specifies that the most significant implemented
+	 * address bit through bit 63 of a physical address must be either all
+	 * 0s or all 1s.  Therefore we sign-extend the 40-bit sys_addr to a
+	 * 64-bit value below.  See section 3.4.2 of AMD publication 24592:
+	 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
+	 * Programming.
+	 */
+	sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
+
+	debugf1("  Using DRAM Base reg on node %d to translate\n",
+		pvt->mc_node_id);
+	debugf1("    DramAddr 0x%lx to SysAddr 0x%lx\n",
+		(unsigned long)dram_addr, (unsigned long)sys_addr);
+	return sys_addr;
+}
+
+/* input_addr is an InputAddr associated with the node given by mci.
+ * Translate input_addr to a SysAddr and return the result.
+ */
+static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
+					 u64 input_addr)
+{
+	return dram_addr_to_sys_addr(mci,
+				     input_addr_to_dram_addr(mci, input_addr));
+}
+
+/* Find the minimum and maximum InputAddr values that map to the given csrow.
+ * Pass back these values in *input_addr_min and *input_addr_max.
+ */
+static void find_csrow_limits(struct mem_ctl_info *mci,
+			      int csrow,
+			      u64 *input_addr_min, u64 *input_addr_max)
+{
+	struct amd64_pvt *pvt;
+	u64 base, mask;
+
+	pvt = mci->pvt_info;
+	BUG_ON((csrow < 0) || (csrow >= CHIPSELECT_COUNT));
+
+	base = base_from_dctBase(pvt, csrow);
+	mask = mask_from_dctMask(pvt, csrow);
+
+	*input_addr_min = base & ~mask;
+	*input_addr_max = base | mask | pvt->dcs_mask_notused;
+}
+
+/*
+ * k8_get_error_address
+ *	extract from the hardware copies of the error register
+ *	the ERROR ADDRESS for the K8 and Family 0Fh CPUs
+ */
+static u64 k8_get_error_address(struct mem_ctl_info *mci,
+			struct amd64_error_info_regs *info)
+{
+	return (((u64) (info->nbeah & 0xff)) << 32) +
+			(info->nbeal & ~0x03);
+}
+
+/*
+ * f10_get_error_address
+ *	extract from the hardware copies of the error register
+ *	the ERROR ADDRESS for Family 10h CPUs
+ */
+static u64 f10_get_error_address(struct mem_ctl_info *mci,
+			struct amd64_error_info_regs *info)
+{
+	return (((u64) (info->nbeah & 0xffff)) << 32) +
+			(info->nbeal & ~0x01);
+}
+
+/*
+ * static u64 extract_error_address
+ *	Extract error address from MCA NB Address Low (section 3.6.4.5) and
+ *	MCA NB Address High (section 3.6.4.6) register values and return the
+ *	result. Address is located in the info structure (nbeah and nbeal)
+ *	the encoding is device specific.
+ */
+static u64 extract_error_address(struct mem_ctl_info *mci,
+			struct amd64_error_info_regs *info)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	/* call the proper CPU type function to extract the address */
+	return pvt->ops->get_error_address(mci, info);
+}
+
+
+/*
+ * error_address_to_page_and_offset
+ *
+ *	Map the Error address to a PAGE and PAGE OFFSET
+ */
+static inline void error_address_to_page_and_offset(u64 error_address,
+						    u32 *page, u32 *offset)
+{
+	*page = (u32) (error_address >> PAGE_SHIFT);
+	*offset = ((u32) error_address) & ~PAGE_MASK;
+}
+
+/*
+ * amd64_error_info_valid
+ *
+ * Return 1 if hardware contains valid error information.
+ * Else return 0, no valid error information.
+ */
+static inline int amd64_error_info_valid(struct amd64_error_info_regs *regs)
+{
+	return (regs->nbsh & K8_NBSH_VALID_BIT) != 0;
+}
+
+/*
+ * amd64_get_error_info_regs
+ *
+ *	read the North Bridge Status register HIGH and test the
+ *	VALID ERROR status bit in that register.
+ *
+ *	If set, proceed to read:
+ *		NBS Low
+ *		NBEA LOW
+ *		NBEA HIGH
+ *
+ *	and store data into error structure
+ *
+ * return 1: if hardware regs contains valid error info
+ * return 0: if no valid error is indicated
+ */
+static int amd64_get_error_info_regs(struct mem_ctl_info *mci,
+				  struct amd64_error_info_regs *regs)
+{
+	struct amd64_pvt *pvt;
+	struct pci_dev *misc_f3_ctl;
+	int err;
+
+	pvt = mci->pvt_info;
+	misc_f3_ctl = pvt->misc_f3_ctl;
+
+	/* Reg the NB Status Register High to get ERROR VALID bit */
+	err = pci_read_config_dword(misc_f3_ctl, K8_NBSH, &regs->nbsh);
+	if (err != 0)
+		debugf0("%s() Reading K8_NBSH failed\n", __func__);
+
+	if (!amd64_error_info_valid(regs))
+		return 0;
+
+	/* valid error, read remaining error information registers */
+	err = pci_read_config_dword(misc_f3_ctl, K8_NBSL, &regs->nbsl);
+	if (err != 0)
+		debugf0("%s() Reading K8_NBSL failed\n", __func__);
+	err = pci_read_config_dword(misc_f3_ctl, K8_NBEAL, &regs->nbeal);
+	if (err != 0)
+		debugf0("%s() Reading K8_NBEAL failed\n", __func__);
+	err = pci_read_config_dword(misc_f3_ctl, K8_NBEAH, &regs->nbeah);
+	if (err != 0)
+		debugf0("%s() Reading K8_NBEAH failed\n", __func__);
+	err = pci_read_config_dword(misc_f3_ctl, K8_NBCFG, &regs->nbcfg);
+	if (err != 0)
+		debugf0("%s() Reading K8_NBCFG failed\n", __func__);
+
+	return 1;
+}
+
+/*
+ * amd64_get_error_info
+ *
+ *	this function is called to retrieve the error data from hardware
+ *	and store it in the info structure.
+ *
+ *	return:
+ *		1 if a valid error is found
+ *		0 if no error is found
+ */
+static int amd64_get_error_info(struct mem_ctl_info *mci,
+			      struct amd64_error_info_regs *info)
+{
+	struct amd64_pvt *pvt;
+	struct amd64_error_info_regs regs;
+
+	pvt = mci->pvt_info;
+
+	if (!amd64_get_error_info_regs(mci, info))
+		return 0;
+
+	/*
+	 * Here's the problem with the K8's EDAC reporting:
+	 * There are four registers which report pieces of error
+	 * information.  These four registers are shared between
+	 * CEs and UEs.  Furthermore, contrary to what is stated in
+	 * the OBKG, the overflow bit is never used!  Every error
+	 * always updates the reporting registers.
+	 *
+	 * Can you see the race condition?  All four error reporting
+	 * registers must be read before a new error updates them!
+	 * There is no way to read all four registers atomically.  The
+	 * best than can be done is to detect that a race has occured
+	 * and then report the error without any kind of precision.
+	 *
+	 * What is still positive is that errors are
+	 * still reported and thus problems can still be detected -
+	 * just not localized because the syndrome and address are
+	 * spread out across registers.
+	 *
+	 * Grrrrr!!!!!  Here's hoping that AMD fixes this in some
+	 * future K8 rev. UEs and CEs should have separate
+	 * register sets with proper overflow bits that are used!
+	 * At very least the problem can be fixed by honoring the
+	 * ErrValid bit in 'nbsh' and not updating registers - just
+	 * set the overflow bit - unless the current error is CE
+	 * and the new error is UE which would be the only situation
+	 * for overwriting the current values.
+	 */
+
+	regs = *info;
+
+	/* Use info from the second read - most current */
+	if (unlikely(!amd64_get_error_info_regs(mci, info)))
+		return 0;
+
+	/* clear the error bits in hardware */
+	pci_write_bits32(pvt->misc_f3_ctl, K8_NBSH, 0, K8_NBSH_VALID_BIT);
+
+	/* Check for the possible race condition */
+	if ((regs.nbsh != info->nbsh) ||
+	     (regs.nbsl != info->nbsl) ||
+	     (regs.nbeah != info->nbeah) ||
+	     (regs.nbeal != info->nbeal)) {
+		amd64_mc_printk(mci, KERN_WARNING,
+			"hardware STATUS read access race "
+			"condition detected!\n");
+		return 0;
+	}
+
+	return 1;
+}
+
+/*
+ * amd64_decode_gart_tlb_error
+ *
+ *	decode GART TBL Error code
+ */
+static inline void amd64_decode_gart_tlb_error(struct mem_ctl_info *mci,
+					 struct amd64_error_info_regs *info)
+{
+	u32 err_code;
+	u32 ec_tt;		/* error code transaction type (2b) */
+	u32 ec_ll;		/* error code cache level (2b) */
+
+	err_code = EXTRACT_ERROR_CODE(info->nbsl);
+	ec_ll = EXTRACT_LL_CODE(err_code);
+	ec_tt = EXTRACT_TT_CODE(err_code);
+
+	amd64_mc_printk(mci, KERN_ERR,
+		     "GART TLB event: transaction type(%s), "
+		     "cache level(%s)\n", tt_msgs[ec_tt], ll_msgs[ec_ll]);
+}
+
+/*
+ * amd64_decode_mem_cache_error
+ *
+ *	decode Memory/CACHE Error code
+ */
+static inline void amd64_decode_mem_cache_error(struct mem_ctl_info *mci,
+				      struct amd64_error_info_regs *info)
+{
+	u32 err_code;
+	u32 ec_rrrr;		/* error code memory transaction (4b) */
+	u32 ec_tt;		/* error code transaction type (2b) */
+	u32 ec_ll;		/* error code cache level (2b) */
+
+	err_code = EXTRACT_ERROR_CODE(info->nbsl);
+	ec_ll = EXTRACT_LL_CODE(err_code);
+	ec_tt = EXTRACT_TT_CODE(err_code);
+	ec_rrrr = EXTRACT_RRRR_CODE(err_code);
+
+	amd64_mc_printk(mci, KERN_ERR,
+		     "cache hierarchy error: memory transaction type(%s), "
+		     "transaction type(%s), cache level(%s)\n",
+		     rrrr_msgs[ec_rrrr], tt_msgs[ec_tt], ll_msgs[ec_ll]);
+}
+
+/* sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
+ * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
+ * of a node that detected an ECC memory error.  mci represents the node that
+ * the error address maps to (possibly different from the node that detected
+ * the error).  Return the number of the csrow that sys_addr maps to, or -1 on
+ * error.
+ */
+static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
+{
+	int csrow;
+
+	csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
+
+	if (csrow == -1)
+		amd64_mc_printk(mci, KERN_ERR,
+			     "Failed to translate InputAddr to csrow for "
+			     "address 0x%lx\n", (unsigned long)sys_addr);
+	return csrow;
+}
+
+
+/* Hack for the time being - Can we get this from BIOS?? */
+#define	CH0SPARE_RANK	0
+#define	CH1SPARE_RANK	1
+
+/*
+ * f10_process_possible_spare
+ *
+ *	checks if the csrow passed in is marked as SPARED, if so
+ *	returns the new spare row
+ */
+static inline int f10_process_possible_spare(int csrow,
+				u32 ChannelSelect, struct amd64_pvt *pvt)
+{
+	u32 SwapDone;
+	u32 BadDramCs;
+	u32 OnLineSpareCTL;
+
+	/* Read online spare reg */
+	OnLineSpareCTL = pvt->online_spare;
+
+	/* Depending on channel, isolate respective SPARING info */
+	if (ChannelSelect) {
+		SwapDone = F10_ONLINE_SPARE_SWAPDONE1(OnLineSpareCTL);
+		BadDramCs = F10_ONLINE_SPARE_BADDRAM_CS1(OnLineSpareCTL);
+		if (SwapDone && (csrow == BadDramCs))
+			csrow = CH1SPARE_RANK;
+	} else {
+		SwapDone = F10_ONLINE_SPARE_SWAPDONE0(OnLineSpareCTL);
+		BadDramCs = F10_ONLINE_SPARE_BADDRAM_CS0(OnLineSpareCTL);
+		if (SwapDone && (csrow == BadDramCs))
+			csrow = CH0SPARE_RANK;
+	}
+	return csrow;
+}
+
+/*
+ * f10_lookup_addr_in_dct
+ *
+ *	Iterate over the DRAM DCT "base" and "mask" register looking for
+ *	a SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
+ *
+ * Return:
+ *	-1  NOT FOUND
+ *	0..csrow = Chip-Select Row
+ */
+static int f10_lookup_addr_in_dct(u32 InputAddr, u32 NodeID, u32 ChannelSelect)
+{
+	struct mem_ctl_info *mci;
+	struct amd64_pvt *pvt;
+	u32 CSBase, CSMask;
+	int CSFound = -1;	/* Assume NOT FOUND */
+	int csrow;
+
+	/* get pointer to NodeID's structure, if NULL, we are done */
+	mci = mci_lookup[NodeID];
+	if (!mci)
+		return CSFound;
+
+	pvt = mci->pvt_info;
+
+	debugf1("%s() InputAddr=0x%x  channelselect=%d\n",
+			__func__, InputAddr, ChannelSelect);
+
+	/* Iterate over the csrow of DRAM DCTs looking for an addr match */
+	for (csrow = 0; csrow < CHIPSELECT_COUNT; csrow++) {
+
+		/* Retrieve the CS Base.
+		 * If this row is NOT enabled, continue
+		 */
+		CSBase = get_dctBase(pvt, ChannelSelect, csrow);
+		if (!(CSBase & K8_DCSB_CS_ENABLE))
+			continue;
+
+		/* We have an ENABLED CSROW, Isolate just the MASK
+		 * bits of the target: 28:19 and 13:5, which map to
+		 * 36:27 and 21:13 of the actual address
+		 */
+		CSBase &= REV_F10_DCSB_BASE_BITS;
+
+		/* Get the DCT Mask, and ENABLE the reserved bits:
+		 * 18:16 and 4:0 to become ON. Then mask off bits
+		 * 28:0 (36:8)
+		 */
+		CSMask = get_dctMask(pvt, ChannelSelect, csrow);
+
+		debugf1("    CSROW=%d CSBase=0x%x RAW CSMask=0x%x\n",
+				csrow, CSBase, CSMask);
+
+		CSMask = (CSMask | 0x0007C01F) & 0x1FFFFFFF;
+
+		debugf1("              Final CSMask=0x%x\n", CSMask);
+		debugf1("    (InputAddr & ~CSMask)=0x%x "
+				"(CSBase & ~CSMask)=0x%x\n",
+				(InputAddr & ~CSMask), (CSBase & ~CSMask));
+
+		/* Perform the lookup MATCH operation */
+		if ((InputAddr & ~CSMask) == (CSBase & ~CSMask)) {
+			/* Found a csrow, now check for possible SPARIE on it */
+			CSFound = f10_process_possible_spare(csrow,
+							ChannelSelect, pvt);
+
+			debugf1(" MATCH csrow=%d\n", CSFound);
+			break;
+		}
+	} /* for each CS */
+
+	return CSFound;
+}
+
+/*
+ * map_IntlvEn_to_shift
+ *
+ *	map the IntlvEn value to a shift value
+ */
+static inline u32 map_IntlvEn_to_shift(u32 IntlvEn)
+{
+	u32 shift;
+
+	if (IntlvEn == 1)
+		shift = 1;
+	else if (IntlvEn == 3)
+		shift = 2;
+	else if (IntlvEn == 7)
+		shift = 3;
+	else
+		shift = 0;
+
+	return shift;
+}
+
+/* fUnaryXOR
+ *
+ *	return odd parity bit from argument
+ */
+static u32 fUnaryXOR(u32 value, int num_bits)
+{
+	int i;
+	u32 odd = 0;
+
+	for (i = 0; i < num_bits; i++) {
+		odd ^= value & 1;
+		value >>= 1;
+	}
+
+	return odd;
+}
+
+static u32 f10_determine_channel(struct amd64_pvt *pvt, u64 SystemAddr,
+				int HiRangeSelected, u32 IntlvEn)
+{
+	u32 ChannelSelect;
+	u32 Temp = pvt->dram_ctl_select_low;
+	u32 DctSelIntLvAddr, DctGangEn, DctSelIntLvEn;
+	u32 DctSelHiRngEn, DctSelHi;
+
+	DctGangEn = Temp & F10_DCTL_SEL_LOW_DctGangEn;
+	DctSelIntLvEn = Temp & F10_DCTL_SEL_LOW_DctSelIntLvEn;
+	DctSelIntLvAddr = F10_DCTL_SEL_LOW_DctSelIntLvAddr(Temp);
+	DctSelHiRngEn = Temp & F10_DCTL_SEL_LOW_DctSelHiRngEn;
+	DctSelHi = (Temp >> 1) & 1;
+
+	if (DctGangEn)
+		ChannelSelect = 0;
+	else if (HiRangeSelected)
+		ChannelSelect = DctSelHi;
+	else if (DctSelIntLvEn && DctSelIntLvAddr == 0)
+		ChannelSelect = SystemAddr >> 6 & 1;
+	else if (DctSelIntLvEn && (DctSelIntLvAddr >> 1) & 1) {
+		/* function returns odd parity */
+		Temp = fUnaryXOR((u32) (SystemAddr>>16 & 0x1F), 5);
+			/* 1 = number of set bits in argrume is odd
+			 * 0 = number of set bits in argument is even
+			 */
+
+		if (DctSelIntLvAddr & 1)
+			ChannelSelect = (SystemAddr >> 9 & 1) ^ Temp;
+		else
+			ChannelSelect = (SystemAddr >> 6 & 1) ^ Temp;
+	} else if (DctSelIntLvEn && IntlvEn & 4)
+		ChannelSelect = SystemAddr >> 15 & 1;
+	else if (DctSelIntLvEn && IntlvEn & 2)
+		ChannelSelect = SystemAddr >> 14 & 1;
+	else if (DctSelIntLvEn && IntlvEn & 1)
+		ChannelSelect = SystemAddr >> 13 & 1;
+	else if (DctSelIntLvEn)
+		ChannelSelect = SystemAddr >> 12 & 1;
+	else if (DctSelHiRngEn && DctGangEn == 0)
+		ChannelSelect = ~DctSelHi & 1;
+	else
+		ChannelSelect = 0;
+
+	return ChannelSelect;
+}
+
+static inline u64 f10_determine_base_addr_offset(
+						u64 SystemAddr,
+						int HiRangeSelected,
+						u32 DctSelBaseAddr,
+						u64 DctSelBaseOffsetLong,
+						u32 HoleEn,
+						u32 HoleOffset,
+						u64 DramBaseLong)
+{
+	u64 ChannelAddrLong;
+	u64 ChannelOffsetLong;
+
+	if (HiRangeSelected) {
+		if ((!DctSelBaseAddr & 0xFFFF0000) &&
+		   (HoleEn & 1) && (SystemAddr >= 0x100000000ULL))
+			ChannelOffsetLong = HoleOffset << 16;
+		else
+			ChannelOffsetLong = DctSelBaseOffsetLong;
+	} else {
+		/* If the HoleHoist is enabled on this node AND
+		 * the address is greater than 4GB
+		 */
+		if ((HoleEn & 1) && (SystemAddr >= 0x100000000ULL))
+			ChannelOffsetLong = HoleOffset << 16;
+		else
+			ChannelOffsetLong = DramBaseLong & 0xFFFFF8000000ULL;
+	}
+
+	ChannelAddrLong = (SystemAddr & 0x0000FFFFFFFFFFC0ULL) -
+			(ChannelOffsetLong & 0x0000FFFFFF800000ULL);
+
+	return ChannelAddrLong;
+}
+
+/*
+ * f10_match_to_this_node
+ *
+ * For a given 'DramRange' value, check if 'SystemAddr' fall within this value
+ */
+static int f10_match_to_this_node(struct amd64_pvt *pvt, int DramRange,
+				u64 SystemAddr,
+				int *node_id,
+				int *channel_select)
+{
+	int CSFound = -1;
+	int NodeID;
+	int HiRangeSelected;
+	u32 IntlvEn, IntlvSel;
+	u32 DramEn;
+	u32 Ilog;
+	u32 HoleOffset, HoleEn;
+	u32 InputAddr, Temp;
+	u32 DctSelBaseAddr, DctSelIntLvAddr, DctGangEn, DctSelIntLvEn;
+	u32 DctSelHiRngEn, DctSelHi;
+	u32 ChannelSelect;
+	u64 DramBaseLong, DramLimitLong;
+	u64 DctSelBaseOffsetLong, ChannelAddrLong;
+
+	/* Get the DRAM Base value for this DRAM instance:
+	 * To extract DRAM Base Low, DramEnable and Interleave
+	 */
+	DramBaseLong = pvt->dram_base[DramRange];
+	DramEn = pvt->dram_rw_en[DramRange];
+	IntlvEn = pvt->dram_IntlvEn[DramRange];
+
+	/* Get the DRAM Limit value for this DRAM instance */
+	DramLimitLong = pvt->dram_limit[DramRange];
+	NodeID = pvt->dram_DstNode[DramRange];
+	IntlvSel = pvt->dram_IntlvSel[DramRange];
+
+	debugf1("%s(dram=%d) Base=0x%llx SystemAddr= 0x%llx Limit=0x%llx\n",
+		__func__, DramRange, DramBaseLong, SystemAddr, DramLimitLong);
+
+	/* This assumes that one node's DHAR is the same as
+	 * all the other node's DHARs
+	 */
+	HoleEn = pvt->dhar;
+	HoleOffset = (HoleEn & 0x0000FF80);
+	HoleEn = (HoleEn & 0x00000003);
+
+	debugf1("   HoleOffset=0x%x  HoleEn=0x%x IntlvSel=0x%x\n",
+			HoleOffset, HoleEn, IntlvSel);
+
+	if ((IntlvEn == 0) || IntlvSel == ((SystemAddr >> 12) & IntlvEn)) {
+
+		Ilog = map_IntlvEn_to_shift(IntlvEn);
+
+		/* Fetch the F10 DRAM Controller Select Low Reg */
+		Temp = pvt->dram_ctl_select_low;
+		DctSelBaseOffsetLong = pvt->dram_ctl_select_high << 16;
+
+		DctSelHiRngEn = Temp & F10_DCTL_SEL_LOW_DctSelHiRngEn;
+		DctSelHi = (Temp >> 1) & 1;
+		DctSelIntLvEn = Temp & F10_DCTL_SEL_LOW_DctSelIntLvEn;
+		DctGangEn = Temp & F10_DCTL_SEL_LOW_DctGangEn;
+		DctSelIntLvAddr = F10_DCTL_SEL_LOW_DctSelIntLvAddr(Temp);
+		DctSelBaseAddr = F10_DCTL_SEL_LOW_DctSelBaseAddr(Temp);
+
+		/* Determine if High Range is selected */
+		if (DctSelHiRngEn && (DctGangEn == 0) &&
+			((SystemAddr >> 27) >= (DctSelBaseAddr >> 11)))
+			HiRangeSelected = 1;
+		else
+			HiRangeSelected = 0;
+
+		/* Determine Channel */
+		ChannelSelect = f10_determine_channel(pvt, SystemAddr,
+						HiRangeSelected, IntlvEn);
+
+		/* Determine Base Address Offset to use */
+		ChannelAddrLong = f10_determine_base_addr_offset(
+						SystemAddr,
+						HiRangeSelected,
+						DctSelBaseAddr,
+						DctSelBaseOffsetLong,
+						HoleEn,
+						HoleOffset,
+						DramBaseLong);
+
+		/* Remove Node ID (in case of Processor interleaving */
+		Temp = ChannelAddrLong & 0xFC0;
+
+		ChannelAddrLong = ((ChannelAddrLong >> Ilog) &
+					0xFFFFFFFFF000ULL) | Temp;
+
+		/* Remove Channel interleave and hash */
+		if (DctSelIntLvEn && (DctSelHiRngEn == 0) && (DctGangEn == 0)) {
+			if (DctSelIntLvAddr != 1)
+				ChannelAddrLong =
+					(ChannelAddrLong >> 1) &
+					0xFFFFFFFFFFFFFFC0ULL;
+			else {
+				Temp = ChannelAddrLong & 0xFC0;
+				ChannelAddrLong =
+					((ChannelAddrLong &
+					0xFFFFFFFFFFFFC000ULL)
+					>> 1) | Temp;
+			}
+		}
+
+		/* Form a normalize InputAddr (Move bits 36:8 down to 28:0
+		 * which will set it up to match the DCT Base register
+		 */
+		InputAddr = ChannelAddrLong >> 8;
+
+		debugf1("   (ChannelAddrLong=0x%llx) >> 8 becomes "
+			"InputAddr=0x%x\n", ChannelAddrLong, InputAddr);
+
+		/* Iterate over the DRAM DCTs looking for a
+		 * match for InputAddr on the selected NodeID
+		 */
+		CSFound = f10_lookup_addr_in_dct(InputAddr,
+						NodeID, ChannelSelect);
+
+		/* IF 'FOUND', return data */
+		if (CSFound >= 0) {
+			*node_id = NodeID;
+			*channel_select = ChannelSelect;
+		}
+	} /* if Interleave */
+
+	return CSFound;
+}
+
+/*
+ * f10_translate_sysaddr_to_CS
+ *
+ *	Code straight from the AMD's BKDG, section 2.8.5
+ *	with interface changes
+ */
+static int f10_translate_sysaddr_to_CS(struct amd64_pvt *pvt,
+			u64 SysAddr,
+			int *node,
+			int *chanSel)
+{
+	int DramRange;
+	int CSFound = -1;
+	u64 DramBaseLong, DramLimitLong;
+
+	/* Iterate over each of the possible DRAM reg entries on this node */
+	for (DramRange = 0; DramRange < DRAM_REG_COUNT; DramRange++) {
+
+		/* If this DRAM range is NOT enabled, continue on */
+		if (!pvt->dram_rw_en[DramRange])
+			continue;
+
+		DramBaseLong = pvt->dram_base[DramRange];
+		DramLimitLong = pvt->dram_limit[DramRange];
+
+		/* If the SysAddr falls within START and END of
+		 * this DRAM segment, then do further processing
+		 */
+		if ((DramBaseLong <= SysAddr) && (SysAddr <= DramLimitLong)) {
+
+			/* See if 'SysAddr' falls within this 'DramRange' */
+			CSFound = f10_match_to_this_node(pvt,
+							DramRange, SysAddr,
+							node, chanSel);
+			if (CSFound >= 0)
+				break; /* 'FOUND' Exit logic: return data */
+		}
+
+	} /* for each DramRange */
+
+	return CSFound;
+}
+
+/*
+ * f10_map_sysaddr_to_csrow
+ *
+ *	map a SystemAddress to NodeID, CSROW, Channel
+ */
+static void f10_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
+					struct amd64_error_info_regs *info,
+					u64 SystemAddress)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	u32 page, offset;
+	unsigned short syndrome;
+	int chan = 0;
+	int node_id;
+	int csrow;
+
+	/* Map Error Address to node_id and csrow */
+	csrow = f10_translate_sysaddr_to_CS(pvt,
+					SystemAddress,
+					&node_id,
+					&chan);
+
+	/* If address is found, log and provide information
+	 * else if not found, log the event w/o info
+	 */
+	if (csrow >= 0) {
+		/* Make the CSROW to a PAGE and OFFSET */
+		error_address_to_page_and_offset(SystemAddress, &page, &offset);
+
+		/* Extract the syndrome */
+		syndrome = EXTRACT_HIGH_SYNDROME(info->nbsl) << 8;
+		syndrome |= EXTRACT_LOW_SYNDROME(info->nbsh);
+
+		/* Is CHIPKILL ON?
+		 * If so, then we can attempt to use the 'syndrome' to isolate
+		 * which channel the error was on
+		 */
+		if (pvt->nbcfg & K8_NBCFG_CHIPKILL) {
+			if (pvt->ext_nbcfg & F10_EXT_NBCFG_ECC_SYM_SIZE) {
+				/* X8 symbol size */
+				chan = get_channel_from_x8_syndrome(syndrome);
+			} else {
+				/* X4 symbol size */
+				chan = get_channel_from_x4_syndrome(syndrome);
+			}
+		}
+
+		/* If channel found, report the error */
+		if (chan >= 0) {
+			/* Report the error on this CSROW:CHANNEL tuple */
+			edac_mc_handle_ce(mci, page, offset, syndrome,
+					csrow, chan, EDAC_MOD_STR);
+		} else {
+			/* Channel is not known,
+			 * report all channels on this CSROW as failed.
+			 */
+			for (chan = 0; chan < mci->csrows[csrow].nr_channels;
+								chan++) {
+					edac_mc_handle_ce(mci, page, offset,
+							syndrome,
+							csrow, chan,
+							EDAC_MOD_STR);
+			}
+		}
+
+	} else {
+		/* Report the error, but without any info */
+		edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
+	}
+}
+
+/*
+ * k8_map_sysaddr_to_csrow
+ *
+ *	map a SystemAddress to NodeID, CSROW, Channel
+ */
+static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
+					struct amd64_error_info_regs *info,
+					u64 SystemAddress)
+{
+	struct mem_ctl_info *src_mci;
+	unsigned short syndrome;
+	int channel, csrow;
+	u32 page, offset;
+
+	/* Extract the syndrome parts and form a 16-bit syndrome */
+	syndrome = EXTRACT_HIGH_SYNDROME(info->nbsl) << 8;
+	syndrome |= EXTRACT_LOW_SYNDROME(info->nbsh);
+
+	/* CHIPKILL enabled */
+	if (info->nbcfg & K8_NBCFG_CHIPKILL) {
+		/* x4 chipkill ecc mode - determine channel */
+		channel = get_channel_from_x4_syndrome(syndrome);
+		if (channel < 0) {
+			/* Syndrome didn't map, so we don't know which of
+			 * the 2 DIMMs is in error. So we need to ID 'both'
+			 * of them as suspect.
+			 */
+			amd64_mc_printk(mci, KERN_WARNING,
+				     "unknown syndrome 0x%x - possible error "
+				     "reporting race\n", syndrome);
+			edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
+			return;
+		}
+	} else {
+		/* non-chipkill ecc mode
+		 *
+		 * The k8 documentation is unclear about how to determine the
+		 * channel number when using non-chipkill memory.  This method
+		 * was obtained from email communication with someone at AMD.
+		 * (Wish the email was placed in this comment - norsk)
+		 */
+		channel = ((SystemAddress & BIT(3)) != 0);
+	}
+
+	/* Find out which node the error address belongs to.  This may be
+	 * different from the node that detected the error.
+	 */
+	src_mci = find_mc_by_sys_addr(mci, SystemAddress);
+	if (src_mci == NULL) {
+		amd64_mc_printk(mci, KERN_ERR,
+			     "failed to map error address 0x%lx to a node\n",
+			     (unsigned long)SystemAddress);
+		edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
+		return;
+	}
+
+	/* Now map the SystemAddress to a CSROW */
+	csrow = sys_addr_to_csrow(src_mci, SystemAddress);
+	if (csrow < 0) {
+		/* Report the error, but without any info */
+		edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
+	} else {
+		/* Make the CSROW to a PAGE and OFFSET */
+		error_address_to_page_and_offset(SystemAddress, &page, &offset);
+
+		/* Report the error */
+		edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
+				  channel, EDAC_MOD_STR);
+	}
+}
+
+/*
+ * amd64_handle_ce
+ *
+ *	this routine is called to handle any Correctable Errors (CEs)
+ *	that have occurred. Check for valid ERROR ADDRESS and process
+ */
+static void amd64_handle_ce(struct mem_ctl_info *mci,
+				struct amd64_error_info_regs *info)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	u64 SystemAddress;
+
+	/* Ensure that the Error Address is VALID */
+	if ((info->nbsh & K8_NBSH_VALID_ERROR_ADDR) == 0) {
+		amd64_mc_printk(mci, KERN_ERR,
+			"HW has no ERROR_ADDRESS available\n");
+		edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
+		return;
+	}
+
+	/* Retrieve the error address and syndrome */
+	SystemAddress = extract_error_address(mci, info);
+
+	amd64_mc_printk(mci, KERN_ERR,
+		"CE ERROR_ADDRESS= 0x%llx\n", SystemAddress);
+
+	/* call the specific mapper */
+	pvt->ops->map_sysaddr_to_csrow(mci, info, SystemAddress);
+}
+
+/*
+ * amd64_handle_ue
+ *
+ *	this routine is called to handle any Un-correctable Errors (UEs)
+ */
+static void amd64_handle_ue(struct mem_ctl_info *mci,
+				struct amd64_error_info_regs *info)
+{
+	int csrow;
+	u64 SystemAddress;
+	u32 page, offset;
+	struct mem_ctl_info *log_mci, *src_mci;
+
+	log_mci = mci;
+
+	/* Ensure that the Error Address is VALID */
+	if ((info->nbsh & K8_NBSH_VALID_ERROR_ADDR) == 0) {
+		amd64_mc_printk(mci, KERN_CRIT,
+			"HW has no ERROR_ADDRESS available\n");
+		edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
+		return;
+	}
+
+	/* Retrive the error address */
+	SystemAddress = extract_error_address(mci, info);
+
+	/* Find out which node the error address belongs to.  This may be
+	 * different from the node that detected the error.
+	 */
+	src_mci = find_mc_by_sys_addr(mci, SystemAddress);
+	if (src_mci == NULL) {
+		amd64_mc_printk(mci, KERN_CRIT,
+			"ERROR ADDRESS (0x%lx) value NOT mapped to a MC\n",
+			(unsigned long)SystemAddress);
+		edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
+		return;
+	}
+
+	log_mci = src_mci;
+
+	/* Map the SystemAddress to a 'csrow' */
+	csrow = sys_addr_to_csrow(log_mci, SystemAddress);
+	if (csrow < 0) {
+		amd64_mc_printk(mci, KERN_CRIT,
+			"ERROR_ADDRESS (0x%lx) value NOT mapped to 'csrow'\n",
+			(unsigned long)SystemAddress);
+		edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
+	} else {
+		error_address_to_page_and_offset(SystemAddress, &page, &offset);
+		edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
+	}
+}
+
+/*
+ * amd64_decode_bus_error
+ *
+ *	Perform decode of the AMD64 "BUS" error code
+ */
+static void amd64_decode_bus_error(struct mem_ctl_info *mci,
+			     struct amd64_error_info_regs *info)
+{
+	u32 err_code, ext_ec;
+	u32 ec_pp;		/* error code participating processor (2p) */
+	u32 ec_to;		/* error code timed out (1b) */
+	u32 ec_rrrr;		/* error code memory transaction (4b) */
+	u32 ec_ii;		/* error code memory or I/O (2b) */
+	u32 ec_ll;		/* error code cache level (2b) */
+
+	debugf0("MC%d: %s()\n", mci->mc_idx, __func__);
+
+	ext_ec = EXTRACT_EXT_ERROR_CODE(info->nbsl);
+	err_code = EXTRACT_ERROR_CODE(info->nbsl);
+
+	ec_ll = EXTRACT_LL_CODE(err_code);
+	ec_ii = EXTRACT_II_CODE(err_code);
+	ec_rrrr = EXTRACT_RRRR_CODE(err_code);
+	ec_to = EXTRACT_TO_CODE(err_code);
+	ec_pp = EXTRACT_PP_CODE(err_code);
+
+	/* Log AMD Specific information */
+	amd64_mc_printk(mci, KERN_ERR,
+		"BUS ERROR:\n"
+		"  time-out(%s) mem or i/o(%s)\n"
+		"  participating processor(%s)\n"
+		"  memory transaction type(%s)\n"
+		"  cache level(%s) Error Found by: %s\n",
+		to_msgs[ec_to],
+		ii_msgs[ec_ii],
+		pp_msgs[ec_pp],
+		rrrr_msgs[ec_rrrr],
+		ll_msgs[ec_ll],
+		(info->nbsh & K8_NBSH_ERR_SCRUBER) ?
+			"Scrubber" : "Normal Operation");
+
+	/* If this was an 'observed' error, early out */
+	if (ec_pp == K8_NBSL_PP_OBS)
+		return;		/* We aren't the node involved */
+
+	/* Parse out the extended error code for ECC events */
+	switch (ext_ec) {
+	/* F10 changed to one Extended ECC error code */
+	case F10_NBSL_EXT_ERR_RES:		/* Reserved field */
+	case F10_NBSL_EXT_ERR_ECC:		/* F10 ECC ext err code */
+
+	/* K8 ext error codes, ECC and CHIPKILL ECC - uses same value as F10
+	case K8_NBSL_EXT_ERR_ECC:
+	case K8_NBSL_EXT_ERR_CHIPKILL_ECC:
+	*/
+		break;
+
+	default:
+		amd64_mc_printk(mci, KERN_ERR,
+			"NOT ECC: no special error handling for this error\n");
+		return;
+	}
+
+	/* CE or UE event? */
+	if (info->nbsh & K8_NBSH_CECC)
+		amd64_handle_ce(mci, info);
+	else if (info->nbsh & K8_NBSH_UECC)
+		amd64_handle_ue(mci, info);
+
+	/* If main error is CE then overflow must be CE.  If main error is UE
+	 * then overflow is unknown.  We'll call the overflow a CE - if
+	 * panic_on_ue is set then we're already panic'ed and won't arrive
+	 * here.  If panic_on_ue is not set then apparently someone doesn't
+	 * think that UE's are catastrophic.
+	 */
+	if (info->nbsh & K8_NBSH_OVERFLOW)
+		edac_mc_handle_ce_no_info(mci,
+			EDAC_MOD_STR " Error Overflow set");
+}
+
+/*
+ * amd64_process_error_info
+ *
+ *	process the error information of the AMD64 chipset
+ *
+ *	return:
+ *		1 if error found or
+ *		0 if error not found
+ */
+static int amd64_process_error_info(struct mem_ctl_info *mci,
+					struct amd64_error_info_regs *info,
+					int handle_errors)
+{
+	struct amd64_pvt *pvt;
+	struct amd64_error_info_regs *regs;
+	u32 err_code, ext_ec;
+	int gart_tlb_error = 0;
+
+	pvt = mci->pvt_info;
+
+	/* If caller doesn't want us to process the error, return */
+	if (!handle_errors)
+		return 1;
+
+	regs = info;
+
+	debugf1("NorthBridge ERROR: mci(0x%p)\n", mci);
+	debugf1("  MC node(%d) Error-Address(0x%.8x-%.8x)\n",
+		pvt->mc_node_id, regs->nbeah, regs->nbeal);
+	debugf1("  nbsh(0x%.8x) nbsl(0x%.8x)\n",
+		regs->nbsh, regs->nbsl);
+	debugf1("  Valid Error=%s Overflow=%s\n",
+		(regs->nbsh & K8_NBSH_VALID_BIT) ? "True" : "False",
+		(regs->nbsh & K8_NBSH_OVERFLOW) ? "True" : "False");
+	debugf1("  Err Uncorrected=%s MCA Error Reporting=%s\n",
+		(regs->nbsh & K8_NBSH_UNCORRECTED_ERR) ?
+			"True" : "False",
+		(regs->nbsh & K8_NBSH_ERR_ENABLE) ?
+			"True" : "False");
+	debugf1("  MiscErr Valid=%s ErrAddr Valid=%s PCC=%s\n",
+		(regs->nbsh & K8_NBSH_MISC_ERR_VALID) ?
+			"True" : "False",
+		(regs->nbsh & K8_NBSH_VALID_ERROR_ADDR) ?
+			"True" : "False",
+		(regs->nbsh & K8_NBSH_PCC) ?
+			"True" : "False");
+	debugf1("  CECC=%s UECC=%s Found by Scruber=%s\n",
+		(regs->nbsh & K8_NBSH_CECC) ?
+			"True" : "False",
+		(regs->nbsh & K8_NBSH_UECC) ?
+			"True" : "False",
+		(regs->nbsh & K8_NBSH_ERR_SCRUBER) ?
+			"True" : "False");
+	debugf1("  CORE0=%s CORE1=%s CORE2=%s CORE3=%s\n",
+		(regs->nbsh & K8_NBSH_CORE0) ? "True" : "False",
+		(regs->nbsh & K8_NBSH_CORE1) ? "True" : "False",
+		(regs->nbsh & K8_NBSH_CORE2) ? "True" : "False",
+		(regs->nbsh & K8_NBSH_CORE3) ? "True" : "False");
+
+
+	/* Extract the error code from the hw register value */
+	err_code = EXTRACT_ERROR_CODE(regs->nbsl);
+
+	/* Determine which error type:
+	 *	1) GART errors - non-fatal, developmental events
+	 *	2) MEMORY errors
+	 *	3) BUS errors
+	 *	4) Unknown error
+	 */
+	if (TEST_TLB_ERROR(err_code)) {
+		/*
+		 * GART errors are intended to help graphics driver
+		 * developers to detect bad GART PTEs. It is recommended by
+		 * AMD to disable GART table walk error reporting by default[1]
+		 * (currently being disabled in mce_cpu_quirks()) and according
+		 * to the comment in mce_cpu_quirks(), such GART errors can be
+		 * incorrectly triggered. We may see these errors anyway and
+		 * unless requested by the user, they won't be reported.
+		 *
+		 * [1] section 13.10.1 on BIOS and Kernel Developers Guide for
+		 *     AMD NPT family 0Fh processors
+		 */
+		if (report_gart_errors == 0)
+			return 1;
+
+		/* Only if GART error reporting are requested should
+		 * we generate any logs. If not requested, we left above.
+		 */
+		gart_tlb_error = 1;
+
+		debugf1("GART TLB error\n");
+		amd64_decode_gart_tlb_error(mci, info);
+	} else if (TEST_MEM_ERROR(err_code)) {
+		debugf1("Memory/Cache error\n");
+		amd64_decode_mem_cache_error(mci, info);
+	} else if (TEST_BUS_ERROR(err_code)) {
+		debugf1("Bus (Link/DRAM) error\n");
+		amd64_decode_bus_error(mci, info);
+	} else {
+		/* shouldn't reach here! */
+		amd64_mc_printk(mci, KERN_WARNING,
+			     "%s(): unknown MCE error 0x%x\n", __func__,
+			     err_code);
+	}
+
+	/* fetch and log the extended error code */
+	ext_ec = EXTRACT_EXT_ERROR_CODE(regs->nbsl);
+	amd64_mc_printk(mci, KERN_ERR,
+		"ExtErr=(0x%x) %s\n", ext_ec, ext_msgs[ext_ec]);
+
+	/* Range Check the extended error code for HyperTransport error */
+	if (((ext_ec >= F10_NBSL_EXT_ERR_CRC &&
+			ext_ec <= F10_NBSL_EXT_ERR_TGT) ||
+			(ext_ec == F10_NBSL_EXT_ERR_RMW)) &&
+			EXTRACT_LDT_LINK(info->nbsh)) {
+
+		/* Log a message on HT error */
+		amd64_mc_printk(mci, KERN_ERR,
+			"Error on hypertransport link: %s\n",
+			htlink_msgs[
+			EXTRACT_LDT_LINK(info->nbsh)]);
+	}
+
+	/* Check the UE bit of the NB status high register, if set
+	 * Generate some logs.
+	 * If NOT a GART error, then process the event as a NO-INFO event.
+	 * If it was a GART error, skip that process.
+	 */
+	if (regs->nbsh & K8_NBSH_UNCORRECTED_ERR) {
+		amd64_mc_printk(mci, KERN_CRIT, "uncorrected error\n");
+		if (!gart_tlb_error)
+			edac_mc_handle_ue_no_info(mci, "UE bit is set\n");
+	}
+
+	/* Check for Processor Context Corrupt: Just note it */
+	if (regs->nbsh & K8_NBSH_PCC)
+		amd64_mc_printk(mci, KERN_CRIT,
+			"PCC (processor context corrupt) set\n");
+
+	return 1;
+}
+
+/*
+ * amd64_check
+ *
+ * The main polling 'check' function, called to perform the error checking
+ * and if an error is there, error processing:
+ */
+static void amd64_check(struct mem_ctl_info *mci)
+{
+	struct amd64_error_info_regs info;
+
+	debugf3("%s()\n", __func__);
+	if (amd64_get_error_info(mci, &info))
+		amd64_process_error_info(mci, &info, 1);
+}
+
+/*
+ * reserve_mc_sibling_devices
+ *
+ * Input:
+ *	1) struct amd64_pvt which contains pvt->dram_f2_ctl pointer
+ *	2) AMD Family index value
+ *
+ * Ouput:
+ *	Upon return of 0, the following filled in:
+ *
+ *		struct pvt->addr_f1_ctl
+ *		struct pvt->misc_f3_ctl
+ *
+ *	Filled in with related device funcitions of 'dram_f2_ctl'
+ *	These devices are "reserved" via the pci_get_device()
+ *
+ *	Upon return of 1 (error status):
+ *
+ *		Nothing reserved
+ */
+static int reserve_mc_sibling_devices(struct amd64_pvt *pvt,
+					 int mc_type_index)
+{
+	const struct amd64_family_type
+		*amd64_dev = &amd64_family_types[mc_type_index];
+
+	/* Reserve the ADDRESS MAP Device */
+	pvt->addr_f1_ctl = pci_get_related_function(pvt->dram_f2_ctl->vendor,
+						amd64_dev->addr_f1_ctl,
+						pvt->dram_f2_ctl);
+	if (pvt->addr_f1_ctl == NULL) {
+		amd64_printk(KERN_ERR, "error address map device not found: "
+			  "vendor %x device 0x%x (broken BIOS?)\n",
+			  PCI_VENDOR_ID_AMD, amd64_dev->addr_f1_ctl);
+		return 1;
+	}
+
+	/* Reserve the MISC Device */
+	pvt->misc_f3_ctl = pci_get_related_function(pvt->dram_f2_ctl->vendor,
+						amd64_dev->misc_f3_ctl,
+						pvt->dram_f2_ctl);
+	if (pvt->misc_f3_ctl == NULL) {
+		/* Release the priviously reserved address map */
+		pci_dev_put(pvt->addr_f1_ctl);
+		pvt->addr_f1_ctl = NULL;
+
+		amd64_printk(KERN_ERR, "error miscellaneous device not found: "
+			  "vendor %x device 0x%x (broken BIOS?)\n",
+			  PCI_VENDOR_ID_AMD, amd64_dev->misc_f3_ctl);
+		return 1;
+	}
+
+	debugf1("    Addr Map device PCI Bus ID:\t%s\n",
+		pci_name(pvt->addr_f1_ctl));
+	debugf1("    DRAM MEM-CTL PCI Bus ID:\t%s\n",
+		pci_name(pvt->dram_f2_ctl));
+	debugf1("    Misc device PCI Bus ID:\t\t%s\n",
+		pci_name(pvt->misc_f3_ctl));
+
+	return 0;
+}
+
+
+/*
+ * k8_read_dram_base_limit
+ *
+ *	Read the Base and Limit registers for K8 based Memory controllers
+ *	Extract fields from the 'raw' reg into separate data fields
+ *
+ *		Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN
+ */
+static void k8_read_dram_base_limit(struct amd64_pvt *pvt, int dram)
+{
+	u32 low;
+	u32 off = dram << 3;	/* 8 bytes between DRAM entries */
+	int err;
+
+	/* read the BASE register */
+	err = pci_read_config_dword(pvt->addr_f1_ctl,
+					K8_DRAM_BASE_LOW + off, &low);
+	if (err != 0)
+		debugf0("%s() Reading K8_DRAM_BASE_LOW failed\n", __func__);
+
+	/* Extract parts into separate data entries */
+	pvt->dram_base[dram] = ((u64) low & 0xFFFF0000) << 8;
+	pvt->dram_IntlvEn[dram] = (low >> 8) & 0x7;
+	pvt->dram_rw_en[dram] = (low & 0x3);
+
+	/* read the LIMIT register */
+	err = pci_read_config_dword(pvt->addr_f1_ctl, K8_DRAM_LIMIT_LOW + off,
+		&low);
+	if (err != 0)
+		debugf0("%s() Reading K8_DRAM_LIMIT_LOW failed\n", __func__);
+
+	/* Extract parts into separate data entries
+	 * Limit is the HIGHEST memory location of the region, so lower
+	 * 24-bit needs to be all ones
+	 */
+	pvt->dram_limit[dram] = (((u64) low & 0xFFFF0000) << 8) | 0x00FFFFFF;
+	pvt->dram_IntlvSel[dram] = (low >> 8) & 0x7;
+	pvt->dram_DstNode[dram] = (low & 0x7);
+}
+
+/*
+ * f10_read_dram_base_limit
+ *
+ *	Read the Base and Limit registers for F10 based Memory controllers
+ *	Extract fields from the 'raw' reg into separate data fields
+ *
+ *		Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN
+ */
+static void f10_read_dram_base_limit(struct amd64_pvt *pvt, int dram)
+{
+	u32 high_offset;
+	u32 low_offset;
+	u32 high_base;
+	u32 low_base;
+	u32 high_limit;
+	u32 low_limit;
+	int err;
+
+	/* 8 bytes between DRAM entries */
+	low_offset = K8_DRAM_BASE_LOW + (dram << 3);
+	high_offset = F10_DRAM_BASE_HIGH + (dram << 3);
+
+	/* read the 'raw' DRAM BASE Address register */
+	err = pci_read_config_dword(pvt->addr_f1_ctl, low_offset, &low_base);
+	if (err != 0)
+		debugf0("%s() Reading low_offset= 0x%x failed\n", __func__,
+			low_offset);
+
+	/* Read from the ECS data register */
+	err = pci_read_config_dword(pvt->addr_f1_ctl,
+					high_offset, &high_base);
+	if (err != 0)
+		debugf0("%s() Reading high_offset= 0x%x failed. Default=0\n",
+			__func__, high_offset);
+
+	/* Extract parts into separate data entries */
+	pvt->dram_rw_en[dram] = (low_base & 0x3);
+
+	/* if this DRAM is NOT enabled for R or W, then leave */
+	if (pvt->dram_rw_en[dram] == 0)
+		return;
+
+	pvt->dram_IntlvEn[dram] = (low_base >> 8) & 0x7;
+
+	/* Extract the base address from hardware value & form a long value */
+	pvt->dram_base[dram] = (((((u64) high_base & 0x000000FF) << 32) |
+				((u64) low_base & 0xFFFF0000))) << 8;
+
+	low_offset = K8_DRAM_LIMIT_LOW + (dram << 3);
+	high_offset = F10_DRAM_LIMIT_HIGH + (dram << 3);
+
+	/* read the 'raw' LIMIT registers */
+	err = pci_read_config_dword(pvt->addr_f1_ctl, low_offset, &low_limit);
+	if (err != 0)
+		debugf0("%s() Reading low_offset= 0x%x failed\n", __func__,
+			low_offset);
+
+	/* Read from the ECS data register for the HIGH portion */
+	err = pci_read_config_dword(pvt->addr_f1_ctl,
+			high_offset, &high_limit);
+	if (err != 0)
+		debugf0("%s() Reading high_ofset= 0x%x failed\n", __func__,
+			high_offset);
+
+	debugf0("%s() on dram=%d\n", __func__, dram);
+	debugf0("  HW Regs: BASE=0x%08x-%08x      LIMIT=  0x%08x-%08x\n",
+		high_base, low_base, high_limit, low_limit);
+
+	/* Extract Dest Node and Interleave Select */
+	pvt->dram_DstNode[dram] = (low_limit & 0x7);
+	pvt->dram_IntlvSel[dram] = (low_limit >> 8) & 0x7;
+
+	/* Extract address values and form a LIMIT address
+	 * Limit is the HIGHEST memory location of the region, so lower
+	 * 24-bit needs to be all ones
+	 */
+	low_limit |= 0x0000FFFF;
+	pvt->dram_limit[dram] =
+		((((u64) high_limit << 32) + (u64) low_limit) << 8) | (0xFF);
+}
+
+/*
+ * f10_read_dram_ctl_register
+ *	Read DRAM Controller Select registers for the F10 that are NOT
+ *	in the K8 series
+ */
+static void f10_read_dram_ctl_register(struct amd64_pvt *pvt)
+{
+	int err;
+
+	err = pci_read_config_dword(pvt->dram_f2_ctl,
+				F10_DCTL_SEL_LOW,
+				&pvt->dram_ctl_select_low);
+	if (err != 0) {
+		debugf0("%s() Reading F10_DCTL_SEL_LOW failed\n", __func__);
+	} else {
+		/* Get BOOLEAN 1 or 0 for GANGED mode */
+		pvt->dram_dct_ganged_enable =
+			(pvt->dram_ctl_select_low &
+				F10_DCTL_SEL_LOW_DctGangEn) ? 1 : 0;
+
+		debugf0("%s() DRAM_DCTL_SEL_LOW=0x%x  DctSelBaseAddr=0x%x\n",
+				__func__,
+				pvt->dram_ctl_select_low,
+		F10_DCTL_SEL_LOW_DctSelBaseAddr(pvt->dram_ctl_select_low));
+		debugf0("  DRAM DCTs are=%s DRAM Is=%s DRAM-Ctl-"
+				"sel-hi-range=%s\n",
+				(pvt->dram_dct_ganged_enable) ?
+				"GANGED " : "NOT GANGED ",
+				(pvt->dram_ctl_select_low &
+				F10_DCTL_SEL_LOW_DramEnable) ?
+				"Enabled " : "Disabled ",
+				(pvt->dram_ctl_select_low &
+				F10_DCTL_SEL_LOW_DctSelHiRngEn) ?
+				"Enabled " : "Disabled ");
+		debugf0("  DctDatIntLv=%s  MemCleared=%s "
+				"DctSelIntLvAddr=0x%x\n",
+				(pvt->dram_ctl_select_low &
+				F10_DCTL_SEL_LOW_DctDatIntLv) ?
+				"Enabled " : "Disabled ",
+				(pvt->dram_ctl_select_low &
+				F10_DCTL_SEL_LOW_MemCleared) ?
+				"True " : "False ",
+		F10_DCTL_SEL_LOW_DctSelIntLvAddr(pvt->dram_ctl_select_low));
+	}
+
+	err = pci_read_config_dword(pvt->dram_f2_ctl,
+				F10_DCTL_SEL_HIGH,
+				&pvt->dram_ctl_select_high);
+	if (err != 0)
+		debugf0("%s() Reading F10_DCTL_SEL_HIGH failed\n",
+			__func__);
+	debugf0("%s() DRAM_CTL_SELECT_HIGH=0x%x\n", __func__,
+			pvt->dram_ctl_select_high);
+
+	/* read in the extended NB CFG */
+	err = pci_read_config_dword(pvt->misc_f3_ctl, F10_EXT_NBCFG,
+				&pvt->ext_nbcfg);
+	if (err != 0)
+		debugf0("%s() Reading F10_EXT_NBCFG failed\n", __func__);
+
+	debugf0("%s() ECC %s Symbol size and code used\n",
+			__func__,
+			(pvt->ext_nbcfg & F10_EXT_NBCFG_ECC_SYM_SIZE) ?
+			"x8" : "x4");
+
+}
+
+/*
+ * k8_read_dbam_reg
+ *
+ * Read in the single DBAM register for K8 cpus
+ */
+static void k8_read_dbam_reg(struct amd64_pvt *pvt)
+{
+	int err;
+
+	err = pci_read_config_dword(pvt->dram_f2_ctl, DBAM0, &pvt->dbam0);
+	if (err != 0)
+		debugf0("%s() Reading DBAM0 failed\n", __func__);
+}
+
+/*
+ * f10_read_dbam_reg
+ *
+ * Read in both of DBAM registers for F10h cpus
+ */
+static void f10_read_dbam_reg(struct amd64_pvt *pvt)
+{
+	int err;
+
+	err = pci_read_config_dword(pvt->dram_f2_ctl, DBAM0, &pvt->dbam0);
+	if (err != 0)
+		debugf0("%s() Reading DBAM0 failed\n", __func__);
+
+	err = pci_read_config_dword(pvt->dram_f2_ctl, DBAM1, &pvt->dbam1);
+	if (err != 0)
+		debugf0("%s() Reading DBAM1 failed\n", __func__);
+}
+
+/*
+ * f10_setup
+ *
+ *	Perform needed operations on this NB bridge thare are needed
+ *	for the duration of time that the module is operating
+ *
+ *	1) Enable "extended configuration access via 0xCF8" feature
+ */
+static void f10_setup(struct amd64_pvt *pvt)
+{
+	u32 reg;
+	int err;
+
+	err = pci_read_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, &reg);
+	if (err != 0)
+		debugf0("%s() Reading F10_NB_CFG_HIGH failed\n",
+			__func__);
+
+	/* save old value */
+	pvt->old_enable_cf8_extcfg = reg & F10_NB_CFG_LOW_ENABLE_EXT_CFG;
+	reg |= F10_NB_CFG_LOW_ENABLE_EXT_CFG;
+	err = pci_write_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, reg);
+}
+
+/*
+ * f10_teardown
+ *
+ *	teardown any items that were 'setup' in the setup func
+ *
+ *	1) Restore the "extended configuration access via 0xCF8" feature
+ */
+static void f10_teardown(struct amd64_pvt *pvt)
+{
+	u32 reg;
+	int err;
+
+	err = pci_read_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, &reg);
+	if (err != 0)
+		debugf0("%s() Reading F10_NB_CFG_HIGH failed\n",
+			__func__);
+
+	reg &= ~F10_NB_CFG_LOW_ENABLE_EXT_CFG;
+	reg |= pvt->old_enable_cf8_extcfg;
+	err = pci_write_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, reg);
+}
+
+/*
+ * amd64_read_mc_registers
+ *
+ *	Retrieve the hardware registers of the memory controller
+ *	(this includes the 'Address Map' and 'Misc' device regs)
+ *	and cache that data in the private data area for this instance
+ */
+static void amd64_read_mc_registers(struct amd64_pvt *pvt)
+{
+	int dram;
+	int err;
+
+	debugf0("%s(MC node-id=%d): (ExtModel=%d)\n",
+		__func__, pvt->mc_node_id, pvt->ext_model);
+
+	/* Display info on type of processor we have */
+	pvt->ops->display_info(pvt);
+
+	/* Read in the North Bridge Capabilities registers */
+	err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCAP, &pvt->nbcap);
+	if (err != 0)
+		debugf0("%s() Reading K8_NBCAP failed\n", __func__);
+
+	/* Read DRAM Controller Select registers (if available) */
+	pvt->ops->read_dram_ctl_register(pvt);
+
+	/* Retrieve the DRAM Base i'th and Limit i'th Registers */
+	for (dram = 0; dram < DRAM_REG_COUNT; dram++) {
+
+		/* call CPU specific READ function to get the DRAM Base and
+		 * Limit values from the Memory Controller
+		 */
+		pvt->ops->read_dram_base_limit(pvt, dram);
+
+		/* Only print out debug info on rows with both R and W
+		 * Enabled.  Normal processing, compiler should optimize
+		 * this whole 'if' debug output block away
+		 */
+		if (pvt->dram_rw_en[dram] != 0) {
+			debugf1("  DRAM_BASE[%d]: 0x%8.08x-%8.08x "
+				"DRAM_LIMIT:  0x%8.08x-%8.08x\n",
+				dram,
+				(u32)(pvt->dram_base[dram] >> 32),
+				(u32)(pvt->dram_base[dram] & 0xFFFFFFFF),
+				(u32)(pvt->dram_limit[dram] >> 32),
+				(u32)(pvt->dram_limit[dram] & 0xFFFFFFFF));
+			debugf1("        IntlvEn=%s %s %s "
+				"IntlvSel=%d DstNode=%d\n",
+				pvt->dram_IntlvEn[dram] ?
+					"Enabled" : "Disabled",
+				(pvt->dram_rw_en[dram] & 0x2) ? "W" : "!W",
+				(pvt->dram_rw_en[dram] & 0x1) ? "R" : "!R",
+				pvt->dram_IntlvSel[dram],
+				pvt->dram_DstNode[dram]);
+		}
+	}
+
+	/* Setup the DCSB and DCSM arrays from hardware */
+	pvt->ops->read_dctBase_dctMask(pvt);
+
+	/* Read the DRAM Hole Address Register */
+	err = pci_read_config_dword(pvt->addr_f1_ctl, K8_DHAR, &pvt->dhar);
+	if (err != 0)
+		debugf0("%s() Reading K8_DHAR failed\n", __func__);
+
+	/* Read in the DRAM Base Address Mapping */
+	pvt->ops->read_dbam_reg(pvt);
+
+	/* Read the ONLINE Spare register */
+	err = pci_read_config_dword(pvt->misc_f3_ctl,
+			F10_ONLINE_SPARE, &pvt->online_spare);
+	if (err != 0)
+		debugf0("%s() Reading ONLINE_SPARE failed\n", __func__);
+
+	/* for Debugging: decode and display version specific registers */
+	pvt->ops->decode_misc_regs(pvt);
+}
+
+/*
+ * k8_early_channel_count
+ *
+ *	NOTE: CPU Revision Dependent code
+ *
+ * 	MUST read the hardware DCL register, and decode it, then return it
+ *
+ *	the DCL - DRAM Configuration Low Register contains various
+ *	configuration bits on memory.
+ *
+ *	BUT it is different between CG, D & E revs and the later
+ *	Rev F memory controllers (DDR vs DDR2)
+ *
+ * Return:
+ *	number of Memory Channels in operation
+ * Pass back:
+ *	contents of the DCL0_LOW register
+ */
+static int k8_early_channel_count(struct amd64_pvt *pvt)
+{
+	int flag;
+	int err;
+
+	err = pci_read_config_dword(pvt->dram_f2_ctl, K8_DCL0_LOW, &pvt->dcl0);
+	if (err != 0)
+		debugf0("%s() Reading K8_DCL0_LOW failed\n", __func__);
+
+	if (node_rev(pvt->mc_node_id) >= OPTERON_CPU_REV_F) {
+		/* Rev F (NPT) and later */
+		flag = pvt->dcl0 & F10_WIDTH_128;
+	} else {
+		/* Rev E and earlier */
+		flag = pvt->dcl0 & REVE_WIDTH_128;
+	}
+
+	/* not used */
+	pvt->dcl1 = 0;
+
+	return (flag) ? 2 : 1;
+}
+
+/*
+ * f10_early_channel_count
+ *
+ *	the DCL - DRAM Configuration Low Register contains various
+ *	configuration bits on memory.
+ *
+ * MUST read the hardware and return the value and the count of
+ * channels
+ *
+ * Return:
+ *	number of Memory Channels in operation
+ * Pass back:
+ *	contents of the DCL0_LOW register
+ */
+static int f10_early_channel_count(struct amd64_pvt *pvt)
+{
+	int err;
+	int channels = 0;
+	u32 dbam;
+
+	/* read in the HW reg, and store it at the callers scope */
+	err = pci_read_config_dword(pvt->dram_f2_ctl, K8_DCL0_LOW, &pvt->dcl0);
+	if (err != 0)
+		debugf0("%s() Reading K8_DCL0_LOW failed\n", __func__);
+
+	/* read in the HW reg, and store it at the callers scope */
+	err = pci_read_config_dword(pvt->dram_f2_ctl,
+					F10_DCL1_LOW, &pvt->dcl1);
+	if (err != 0)
+		debugf0("%s() Reading F10_DCL1_LOW failed\n", __func__);
+
+	/* If we are in 128 bit mode, then we are using 2 channels */
+	if (pvt->dcl0 & F10_WIDTH_128) {
+		debugf0("%s() Data WIDTH is 128 bits - 2 channels\n",
+			__func__);
+		channels = 2;
+		return channels;
+	}
+
+	/* Need to check if in UN-ganged mode: In such, there are 2
+	 * channels, but they are NOT in 128 bit mode and thus the above
+	 * 'dcl0' status bit will be OFF, but there still are 2 channels
+	 *
+	 * Need to check DCT0[0] and DCT1[0] to see if only one of them
+	 * has their CSEnable bit on. If so, then SINGLE DIMM case.
+	 */
+	debugf0("%s() Data WIDTH is NOT 128 bits - need more decoding\n",
+		__func__);
+
+	/* Check DRAM Bank Address Mapping values for each DIMM
+`	 * to see if there is more than just one DIMM present in an
+	 * unganged mode. Need to check both controllers since DIMMs
+	 * can be placed in either one
+	 */
+	channels = 0;
+	err = pci_read_config_dword(pvt->dram_f2_ctl, DBAM0, &dbam);
+	if (err != 0)
+		debugf0("%s() Reading DBAM0 failed\n", __func__);
+
+	if (DBAM_DIMM(0, dbam) > 0)
+		channels++;
+	if (DBAM_DIMM(1, dbam) > 0)
+		channels++;
+	if (DBAM_DIMM(2, dbam) > 0)
+		channels++;
+	if (DBAM_DIMM(3, dbam) > 0)
+		channels++;
+
+	/* If more than 2 DIMMs are present, then we have 2 channels */
+	if (channels > 2)
+		channels = 2;
+	else if (channels == 0) {
+		/* No DIMMs on DCT0, so look at DCT1 */
+		err = pci_read_config_dword(pvt->dram_f2_ctl,
+						DBAM0, &dbam);
+		if (err != 0)
+			debugf0("%s() Reading DBAM1 failed\n", __func__);
+
+		if (DBAM_DIMM(0, dbam) > 0)
+			channels++;
+		if (DBAM_DIMM(1, dbam) > 0)
+			channels++;
+		if (DBAM_DIMM(2, dbam) > 0)
+			channels++;
+		if (DBAM_DIMM(3, dbam) > 0)
+			channels++;
+
+		if (channels > 2)
+			channels = 2;
+	}
+
+	/* If we found ALL 0 values, then assume just ONE DIMM-ONE Channel */
+	if (channels == 0)
+		channels = 1;
+
+	debugf0("%s() DIMM count= %d\n", __func__, channels);
+
+	return channels;
+}
+
+/*
+ * k8_dbam_map_to_pages
+ *
+ *	determrine the number of PAGES in for this DIMM's size
+ *	based on its DRAM Address Mapping.
+ */
+static int k8_dbam_map_to_pages(struct amd64_pvt *pvt, int dram_map)
+{
+	int nr_pages;
+
+	/* First step is to calc the number of bits to shift a value of 1
+	 * left to indicate show many pages. Start with the DBAM value
+	 * as the starting bits, then proceed to adjust those shift
+	 * bits, based on CPU REV and the table. See BKDG on the DBAM
+	 */
+	if (pvt->ext_model >= OPTERON_CPU_REV_F) {
+
+		/* REV F and greater section */
+		nr_pages = 1 << (revf_quad_ddr2_shift[dram_map] - PAGE_SHIFT);
+	} else {
+		/* REV E and less section This line is tricky.
+		 * It collapses the table used by revision D and later to one
+		 * that matches revision CG and earlier
+		 */
+		dram_map -= (pvt->ext_model >= OPTERON_CPU_REV_D) ?
+				(dram_map > 8 ? 4 : (dram_map > 5 ?
+				3 : (dram_map > 2 ? 1 : 0))) : 0;
+
+		/* 25 shift, is 32MiB minimum DIMM size in REV E and prior
+		 */
+		nr_pages = 1 << (dram_map + 25 - PAGE_SHIFT);
+	}
+
+	return nr_pages;
+}
+
+/*
+ * f10_dbam_map_to_pages
+ */
+static int f10_dbam_map_to_pages(struct amd64_pvt *pvt, int dram_map)
+{
+	return 1 << (revf_quad_ddr2_shift[dram_map] - PAGE_SHIFT);
+}
+
+/*
+ * csrow_nr_pages
+ *
+ *	NOTE: CPU Revision Dependent code
+ *
+ *	Input:
+ *		csrow_nr  ChipSelect Row Number (0..CHIPSELECT_COUNT-1)
+ *		k8 private pointer to -->
+ *			DRAM Bank Address mapping register
+ *			node_id
+ *			DCL register where dual_channel_active is
+ *
+ *	The DBAM register consists of 4 sets of 4 bits each definitions:
+ *
+ *	Bits:		CSROWs
+ *	0-3		CSROWs 0 and 1
+ *	4-7		CSROWs 2 and 3
+ *	8-11		CSROWs 4 and 5
+ *	12-15		CSROWs 6 and 7
+ *
+ * 	Values range from: 0 to 15
+ *	The meanings of the values depends on CPU REV and dual-channel state
+ *
+ *	Various CPU revisions have different size definitions for this
+ *	4 bit values.
+ *	Therefore, we need to examine which CPU we are running on to
+ *	extract meaning for these bits:
+ *
+ *	REV CG and earlier have a given DIMM size definition
+ *	REV D & REV E share another set of size definitions
+ *
+ *	REV CG, D and E all have a common beginning set of size definitions,
+ *	but change the meanings midway into the table.
+ *
+ *	REV F has yet another set of size definitions, which has a totally
+ *	different starting point.
+ *
+ *	REV ? future revs? We won't know till we get the specs
+ *
+ *	The memory controller provides for total of only 8 CSROWs in its
+ *	current architecture. Each "pair" of CSROWs normally represents
+ *	just one (1) *	DIMM in single channel or just two (2) DIMMs
+ *	in dual channel mode.
+ *
+ *	The following code logic collapses the various tables for CSROW
+ *	based on CPU Rev number.
+ *	See the tables/algorithms in the respective BKDG manuals.
+ *
+ *	return:
+ *		The number of PAGE_SIZE pages on the specified CSROW number
+ *		This number incompasses
+ *
+ */
+static u32 csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt)
+{
+	u32 dram_map;
+	u32 nr_pages;
+
+	/* The math on this doesn't look right on the surface because x/2*4
+	 * can be simplified to x*2 but this expression makes use of the fact
+	 * that it is integral math where 1/2=0. This intermediate value
+	 * becomes the number of bits to shift the DBAM register to extract
+	 * the proper CSROW field.
+	 */
+	dram_map = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF;	/* PG88 */
+
+	nr_pages = pvt->ops->dbam_map_to_pages(pvt, dram_map);
+
+	/* If dual channel then double the memory size of single channel */
+	/* channel count is 1 or 2 */
+	nr_pages <<= (pvt->channel_count - 1);
+
+	debugf0("  %s(csrow=%d) DBAM map index= %d\n", __func__,
+		csrow_nr, dram_map);
+	debugf0("    nr_pages= %u  channel-count = %d\n",
+		nr_pages, pvt->channel_count);
+
+	return nr_pages;
+}
+
+/*
+ * k8_determine_memory_type
+ *
+ *	NOTE: CPU Revision Dependent code
+ *
+ *	determine the memory type in operation on this controller
+ */
+static enum mem_type k8_determine_memory_type(struct amd64_pvt *pvt)
+{
+	enum mem_type type;
+
+	if (pvt->ext_model >= OPTERON_CPU_REV_F) {
+		/* Rev F and later */
+		type = (pvt->dcl0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
+	} else {
+		/* Rev E and earlier */
+		type = (pvt->dcl0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
+	}
+
+	debugf1("  Memory type is: %s\n",
+		(type == MEM_DDR2) ? "MEM_DDR2" :
+		(type == MEM_RDDR2) ? "MEM_RDDR2" :
+		(type == MEM_DDR) ? "MEM_DDR" : "MEM_RDDR");
+
+	return type;
+}
+
+/*
+ * f10_determine_memory_type
+ *
+ *      determine the memory type in operation on this F10 controller
+ *      Type is: Unbuffered (Unregistered) or Buffered (Registered)
+ */
+static enum mem_type f10_determine_memory_type(struct amd64_pvt *pvt)
+{
+	enum mem_type type;
+
+	type = (pvt->dcl0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
+
+	debugf1("  Memory type is: %s\n",
+		(type == MEM_DDR2) ? "MEM_DDR2" : "MEM_RDDR2");
+
+	return type;
+}
+
+/*
+ * k8_determine_dram_type
+ *
+ *	NOTE: CPU Revision Dependent code
+ *
+ *	determine the DRAM type in operation
+ *	There are CHIPSELECT_COUNT  (8) and 2 CSROWS per DIMM, therefore
+ *	there are 4 Logical DIMMs possible, thus 4 bits in the
+ *	configuration register indicating whether there are
+ *	X4 or X8 devices, one per logical DIMM
+ */
+static enum dev_type k8_determine_dram_type(struct amd64_pvt *pvt, int row)
+{
+	int bit;
+	enum dev_type type;
+
+	/* the starting bit depends on Revision value */
+	bit = (pvt->ext_model >= OPTERON_CPU_REV_F) ? 12 : 20;
+	type = ((pvt->dcl0 >> (bit + (row / 2))) & 0x01) ? DEV_X4 : DEV_X8;
+
+	debugf1("  DRAM type is: %s\n", (type == DEV_X4) ? "DEV-x4" : "DEV-x8");
+
+	return type;
+}
+
+/*
+ * f10_determine_dram_type
+ *
+ *      determine the DRAM type in operation
+ *      Type is: x8 or x4 type of devices
+ *
+ *      There are CHIPSELECT_COUNT  (8) and 2 CSROWS per DIMM, therefore
+ *      there are 4 Logical DIMMs possible, thus 4 bits in the
+ *      configuration register indicating whether there are
+ *      X4 or X8 devices, one per logical DIMM
+ */
+static enum dev_type f10_determine_dram_type(struct amd64_pvt *pvt, int row)
+{
+	enum dev_type type;
+
+	type = ((pvt->dcl0 >> (12 + (row / 2))) & 0x01) ? DEV_X4 : DEV_X8;
+
+	debugf1("  DRAM type is: %s\n", (type == DEV_X4) ? "DEV-x4" : "DEV-x8");
+
+	return type;
+}
+
+/*
+ * k8_determine_edac_cap
+ *
+ *	NOTE: CPU Revision Dependent code
+ *
+ *	determine if the DIMMs have ECC enabled
+ *	ECC is enabled ONLY if all the DIMMs are ECC capable
+ */
+static enum edac_type k8_determine_edac_cap(struct amd64_pvt *pvt)
+{
+	int bit;
+	enum dev_type edac_cap = EDAC_NONE;
+
+	bit = (pvt->ext_model >= OPTERON_CPU_REV_F) ? 19 : 17;
+
+	if ((pvt->dcl0 >> bit) & 0x1) {
+		debugf1("  edac_type is: EDAC_FLAG_SECDED\n");
+		edac_cap = EDAC_FLAG_SECDED;
+	}
+
+	return edac_cap;
+}
+
+/*
+ * f10_determine_edac_cap
+ *
+ *	determine if the DIMMs have ECC enabled
+ *
+ *	ECC is enabled ONLY if all the DIMMs are ECC capable
+ */
+static enum edac_type f10_determine_edac_cap(struct amd64_pvt *pvt)
+{
+	enum dev_type edac_cap = EDAC_NONE;
+
+	if ((pvt->dcl0 >> 19) & 0x1) {
+		edac_cap = EDAC_FLAG_SECDED;
+
+		debugf1("  edac_type is: EDAC_FLAG_SECDED\n");
+	}
+
+	return edac_cap;
+}
+
+/*
+ * k8_probe_valid_hardware
+ *
+ * Very early hardware probe on pci_probe thread to determine if this
+ * module can support the hardware. If it cannot, it will display
+ * information as to why, if it can, and then error exit
+ *
+ * Return:
+ *	0 for OK
+ *	1 for error
+ */
+static int k8_probe_valid_hardware(struct amd64_pvt *pvt)
+{
+	return 0;
+}
+
+/*
+ * f10_probe_valid_hardware
+ *
+ * Very early hardware probe on pci_probe thread to determine if this
+ * module can support the hardware. If it cannot, it will display
+ * information as to why, if it can, and then error exit
+ *
+ * Return:
+ *	0 for OK
+ *	1 for error
+ */
+static int f10_probe_valid_hardware(struct amd64_pvt *pvt)
+{
+	int rc = 0;
+	int err;
+	u32 dchr;
+
+	err = pci_read_config_dword(pvt->dram_f2_ctl, F10_DCHR_0, &dchr);
+	if (err != 0)
+		debugf0("%s() Reading F10_DCHR_0 failed\n", __func__);
+
+	/* If we are on a DDR3 machine, we don't know yet if
+	 * we support that properly at this time
+	 */
+	if (dchr & F10_DCHR_Ddr3Mode) {
+		rc = 1;
+
+		amd64_printk(KERN_WARNING,
+			"%s() This machine is running with DDR3 memory. "
+			"This is not currently supported. DCHR=0x%x\n",
+			__func__, dchr);
+		amd64_printk(KERN_WARNING,
+			"   Contact '%s' module MAINTAINER to help add"
+			" support.\n",
+			EDAC_MOD_STR);
+		if (dchr & F10_DCHR_MblMode)
+			amd64_printk(KERN_WARNING,
+				"   'Memory Buffer Link (MBL)' Enabled\n");
+		else
+			amd64_printk(KERN_WARNING,
+				"   'Direct Connect Mode' Enabled\n");
+		rc = 1;
+
+	} else {
+		debugf0("%s() DDR2 Memory Installed\n", __func__);
+	}
+
+	return rc;
+}
+
+/*
+ * amd64_init_csrows
+ *
+ *	perform initialization on the array of csrow attribute instances,
+ *	based on the values from Hardware registers
+ */
+static int amd64_init_csrows(struct mem_ctl_info *mci)
+{
+	struct csrow_info *csrow;
+	struct amd64_pvt *pvt;
+	int i;
+	int empty = 1;		/* start out assume the rows are empty */
+	u64 input_addr_min, input_addr_max, sys_addr;
+	int err;
+
+	pvt = mci->pvt_info;
+
+	/* read the North Bridge Configuration reg */
+	err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCFG, &pvt->nbcfg);
+	if (err != 0)
+		debugf0("%s() Reading K8_NBCFG failed\n", __func__);
+
+	debugf0("%s() NBCFG= 0x%x  CHIPKILL= %s ECC_ENABLE= %s\n",
+		__func__, pvt->nbcfg,
+		(pvt->nbcfg & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled",
+		(pvt->nbcfg & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled"
+		);
+
+	/* Iterate over the rows, looking for a valid set */
+	for (i = 0; i < CHIPSELECT_COUNT; i++) {
+		csrow = &mci->csrows[i];
+
+		/* Check the 'CS Enable' bit of this instance */
+		if ((pvt->dcsb0[i] & K8_DCSB_CS_ENABLE) == 0) {
+			debugf1("----CSROW %d EMPTY for node %d\n", i,
+				pvt->mc_node_id);
+			continue;	/* empty */
+		}
+
+		debugf1("----CSROW %d VALID for MC node %d\n",
+			i, pvt->mc_node_id);
+
+		empty = 0;
+		csrow->nr_pages = csrow_nr_pages(i, pvt);
+		find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
+		sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
+		csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
+		sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
+		csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
+		csrow->page_mask = ~mask_from_dctMask(pvt, i);
+		csrow->grain = 8;	/* 8 bytes of resolution */
+
+		csrow->mtype = pvt->ops->determine_memory_type(pvt);
+		csrow->dtype = pvt->ops->determine_dram_type(pvt, i);
+
+		debugf1("  for MC node %d csrow %d:\n", pvt->mc_node_id, i);
+		debugf1("    input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
+			(unsigned long)input_addr_min,
+			(unsigned long)input_addr_max);
+		debugf1("    sys_addr: 0x%lx  page_mask: 0x%lx\n",
+			(unsigned long)sys_addr, csrow->page_mask);
+		debugf1("    nr_pages: %u  first_page: 0x%lx "
+			"last_page: 0x%lx\n",
+			(unsigned)csrow->nr_pages,
+			csrow->first_page, csrow->last_page);
+
+		/* determine wheather:
+		 * 	CHIPKILL or JUST ECC or NO ECC is operating
+		 */
+		if (pvt->nbcfg & K8_NBCFG_ECC_ENABLE)
+			csrow->edac_mode =
+			    (pvt->nbcfg & K8_NBCFG_CHIPKILL) ?
+			    EDAC_S4ECD4ED : EDAC_SECDED;
+		else
+			csrow->edac_mode = EDAC_NONE;
+	}
+
+	return empty;
+}
+
+/*
+ * amd64_enable_ecc_error_reporting
+ *
+ *	Only if 'ecc_enable_override' is set AND BIOS had ECC disabled,
+ *	do "we" enable it.
+ *
+ *	On each NB we need to enable the hardware to
+ *	generate and detect error events
+ *
+ * 	1) NB Control Register
+ *	2) Global MCE Reporting Control Reg (MCGCTL)
+ */
+static void amd64_enable_ecc_error_reporting(struct mem_ctl_info *mci)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	u32 mcgctl_l, mcgctl_h;
+	u32 value;
+	u32 mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn;
+	int err;
+
+	/* if NOT enabled overriden, then leave */
+	if (!ecc_enable_override)
+		return;
+
+	/* Inform admin of this fact */
+	amd64_printk(KERN_WARNING,
+		"'ecc_enable_override' parameter is active, "
+		"Enabling AMD ECC hardware now: CAUTION\n");
+
+	/* 1) read the NB Control register, and save old Enable bits */
+	err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCTL, &value);
+	if (err != 0)
+		debugf0("%s() Reading K8_NBCTL failed\n", __func__);
+
+	/* save old value and then turn on UECCn and CECCEn bits
+	 * and write it back out, thus turning ON ECC for sure
+	 */
+	pvt->old_nbctl = value & mask;
+	pvt->nbctl_mcgctl_saved = 1;	/* Mark 'old' ECC values valid */
+
+	value |= mask;
+	pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCTL, value);
+
+	debugf0("%s() Old NBCTL 0x%x New NBCTL= 0x%x\n",
+		__func__, pvt->old_nbctl, value);
+
+	/* 2) Read and save the NB Enable bit at entry. Enable the bit
+	 * then write the enabled value back to hardware
+	 */
+	do_rdmsr(pvt->mc_node_id, K8_MSR_MCGCTL, &mcgctl_l, &mcgctl_h);
+	pvt->old_mcgctl = mcgctl_l & K8_MSR_MCGCTL_NBE;
+	mcgctl_l |= K8_MSR_MCGCTL_NBE;
+
+	debugf0("%s() Old MCGCTL= 0x%x New MCGCTL=0x%x\n",
+		__func__, (unsigned int) mcgctl_l,
+		(unsigned int) pvt->old_mcgctl);
+
+	/* Ensure NBE is on and write it back out */
+	do_wrmsr(pvt->mc_node_id, K8_MSR_MCGCTL, mcgctl_l, mcgctl_h);
+	do_rdmsr(pvt->mc_node_id, K8_MSR_MCGCTL, &mcgctl_l, &mcgctl_h);
+
+	/* 3) Read the NB CFG to ensure DRAM ECC is on and then
+	 * keep a copy of the hw register in the control structure
+	 */
+	err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCFG, &value);
+	if (err != 0)
+		debugf0("%s() Reading K8_NBCFG failed\n", __func__);
+
+	debugf0("%s() NBCFG(1)= 0x%x  CHIPKILL= %s ECC_ENABLE= %s\n",
+		__func__, value,
+		value & (K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled",
+		value & (K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled"
+		);
+
+	if (!(value & K8_NBCFG_ECC_ENABLE)) {
+		amd64_printk(KERN_WARNING,
+			"This node reports that DRAM ECC is "
+			"currently Disabled; ENABLING now\n");
+
+		/* Attempt to turn on DRAM ECC Enable */
+		value |= K8_NBCFG_ECC_ENABLE;
+		pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCFG, value);
+
+		/* See if the Hardware rejected it */
+		err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCFG, &value);
+		if (err != 0)
+			debugf0("%s() Reading K8_NBCFG failed\n", __func__);
+
+		if (!(value & K8_NBCFG_ECC_ENABLE)) {
+			amd64_printk(KERN_WARNING,
+				"Hardware rejects Enabling DRAM ECC checking\n"
+				"Check memory DIMM configuration\n");
+		} else {
+			amd64_printk(KERN_DEBUG,
+				"Hardware accepted DRAM ECC Enable\n");
+		}
+	}
+
+	debugf0("%s() NBCFG(2)= 0x%x  CHIPKILL= %s ECC_ENABLE= %s\n",
+		__func__, value,
+		(value & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled",
+		(value & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled"
+		);
+
+	pvt->ctl_error_info.nbcfg = value;
+}
+
+
+/*
+ * amd64_restore_ecc_error_reporting
+ *
+ *	restore the hardware registers to their initial condition
+ *	prior to when amd64_enable_ecc_error_reporting was called
+ */
+static void amd64_restore_ecc_error_reporting(struct amd64_pvt *pvt)
+{
+	u32 mcgctl_l, mcgctl_h;
+	u32 value;
+	u32 mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn;
+	int err;
+
+	/* Only if 'nbctl_mcgctl_saved' is TRUE are the "old" values
+	 * valid and need to be restored. Otherwise, we are done
+	 */
+	if (!pvt->nbctl_mcgctl_saved)
+		return;
+
+	/* 1) read the NB Control register, and mask off bits of interest*/
+	err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCTL, &value);
+	if (err != 0)
+		debugf0("%s() Reading K8_NBCTL failed\n", __func__);
+	value &= ~mask;
+	value |= pvt->old_nbctl;
+
+	/* restore the NB Enable MCGCTL bit */
+	pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCTL, value);
+
+	/* 2) Read, mask off then OR in the previous value of the NBE */
+	do_rdmsr(pvt->mc_node_id, K8_MSR_MCGCTL, &mcgctl_l, &mcgctl_h);
+	mcgctl_l &= ~K8_MSR_MCGCTL_NBE;
+	mcgctl_l |= pvt->old_mcgctl;
+	do_wrmsr(pvt->mc_node_id, K8_MSR_MCGCTL, mcgctl_l, mcgctl_h);
+	do_rdmsr(pvt->mc_node_id, K8_MSR_MCGCTL, &mcgctl_l, &mcgctl_h);
+}
+
+/*
+ * amd64_check_ecc_enabled
+ *
+ *	EDAC requires that the BIOS have ECC enabled before taking over the
+ *	processing of ECC errors. This is because the BIOS can properly
+ *	initialize the memory system completely.
+ *
+ *	For development and other purposes, there is a command line option
+ *	which allows for overriding this contraint. If that option is TRUE
+ *	then this module driver will enable the hardware ECC "enable"
+ */
+static int amd64_check_ecc_enabled(struct amd64_pvt *pvt)
+{
+	u32 mcgctl_l, mcgctl_h;
+	u32 value;
+	int tmp;
+	int rc = 0;
+
+	/* 1) read the NB Control register to get ECC enabled bits */
+	tmp = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCTL, &value);
+	if (tmp != 0)
+		debugf0("%s() Reading K8_NBCTL failed\n", __func__);
+
+	/* 2) Read and save the NB Enable bit at entry. Enable the bit
+	 * then write the enabled value back to hardware
+	 */
+	do_rdmsr(pvt->mc_node_id, K8_MSR_MCGCTL, &mcgctl_l, &mcgctl_h);
+
+	/* Both K8_MSR_MCGCTL_NBE and K8_NBCFG_ECC_ENABLE must be enabled */
+	tmp = ((mcgctl_l & K8_MSR_MCGCTL_NBE) &&
+					(value & K8_NBCFG_ECC_ENABLE));
+
+	debugf0("%s() mcgctl_l=0x%x  K8_NBCTL=0x%x,  ECC is Enabled=%s\n",
+		__func__, mcgctl_l, value, tmp ? "True" : "False");
+
+	/* If ECC is NOT enabled by BIOS, check override parameter.
+	 *   If override is NOT set then error return.
+	 *   If override IS set, then go ahead and enable ECC
+	 */
+	if (!tmp) {
+		amd64_printk(KERN_WARNING,
+			"This node reports that Memory ECC is "
+			"currently 'disabled'.\n");
+		amd64_printk(KERN_WARNING,
+			"    bit 0x%lx in register 0x%x of the "
+			"MISC_CONTROL device (%s) should be enabled\n",
+			K8_NBCFG_ECC_ENABLE, K8_NBCTL,
+			pci_name(pvt->dram_f2_ctl));
+		amd64_printk(KERN_WARNING,
+			"    bit 0x%lx in MSR register 0x%x of "
+			"Memory node %d should be enabled\n",
+			K8_MSR_MCGCTL_NBE, K8_MSR_MCGCTL,
+			pvt->mc_node_id);
+
+		/* ECC is not enabled! and no override */
+		if (!ecc_enable_override) {
+			amd64_printk(KERN_WARNING, "WARNING: ECC is NOT "
+				"currently enabled by the BIOS. Module "
+				"will NOT be loaded.\n"
+				"    Either Enable ECC in the BIOS, "
+				"or use the 'ecc_enable_override' "
+				"parameter.\n"
+				"    Might be a BIOS bug, if BIOS says "
+				"ECC is enabled\n"
+				"    Use of the override can cause "
+				"unknown side effects.\n");
+			rc = -ENODEV;
+		}
+	} else {
+		amd64_printk(KERN_INFO,
+			"ECC is enabled by BIOS, Proceeding "
+			"with EDAC module initialization\n");
+
+		/* CLEAR the override, since BIOS controlled it */
+		ecc_enable_override = 0;
+	}
+
+	return rc;
+}
+
+/*
+ * get_top_mem_regs
+ *
+ *	Retrive the 2 TOP of MEMORY values from the northbridge
+ *	1) Below 4 GB boundary
+ *	2) top of physical memory beyond 4 GB
+ */
+static void get_top_mem_regs(struct amd64_pvt *pvt)
+{
+	u32 low, high;
+
+	debugf0("%s()\n", __func__);
+	do_rdmsr(pvt->mc_node_id, K8_MSR_TOP_MEM, &low, &high);
+	pvt->top_mem = ((u64) high << 32) | low;
+	debugf0("  TOP_MEM=  0x%08x-%08x\n", high, low);
+
+	do_rdmsr(pvt->mc_node_id, K8_MSR_TOP_MEM2, &low, &high);
+	pvt->top_mem2 = ((u64) high << 32) | low;
+	debugf0("  TOP_MEM2= 0x%08x-%08x\n", high, low);
+}
+
+/* Forward reference */
+static int amd64_process_error_info(struct mem_ctl_info *mci,
+					struct amd64_error_info_regs *info,
+					int handle_errors);
+
+/*
+ * amd64_nbea_store
+ *
+ * 	Accept a hex value and store it into the virutal error
+ *	register file, field:	nbeal and nbeah
+ *
+ *	Assume virtual error values have already been set for:
+ *		NBSL, NBSH and NBCFG
+ *
+ *	Then proceed to map the error values to a:
+ *		MC, CSROW and CHANNEL
+ */
+static ssize_t amd64_nbea_store(struct mem_ctl_info *mci,
+					const char *data, size_t count)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	unsigned long long value;
+	int rc;
+
+	rc = strict_strtoull(data, 16, &value);
+	if (rc != -EINVAL) {
+		debugf0("%s() received NBEA= 0x%llx\n", __func__, value);
+
+		/* place the value into the virtual error packet */
+		pvt->ctl_error_info.nbeal = (u32) value;
+		value >>= 32;
+		pvt->ctl_error_info.nbeah = (u32) value;
+
+		/* Process the Mapping request */
+		/* TODO: Add race preventation */
+		amd64_process_error_info(mci, &pvt->ctl_error_info, 1);
+
+		return count;
+	}
+	return 0;
+}
+
+/*
+ * amd64_nbea_show
+ *
+ *	display back what the last NBEA address was written
+ */
+static ssize_t amd64_nbea_show(struct mem_ctl_info *mci, char *data)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	u64 value;
+
+	value = pvt->ctl_error_info.nbeah;
+	value <<= 32;
+	value |= pvt->ctl_error_info.nbeal;
+
+	return sprintf(data, "%lx\n", (unsigned long) value);
+}
+
+/*
+ * amd64_nbsl_store
+ *
+ *	accept and store the NBSL value user desires
+ */
+static ssize_t amd64_nbsl_store(struct mem_ctl_info *mci,
+					const char *data, size_t count)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	unsigned long value;
+	int rc;
+
+	rc = strict_strtoul(data, 16, &value);
+	if (rc != -EINVAL) {
+		debugf0("%s() received NBSL= 0x%lx\n", __func__, value);
+
+		/* place the NBSL value into the virtual error packet */
+		pvt->ctl_error_info.nbsl = (u32) value;
+
+		return count;
+	}
+	return 0;
+}
+
+/*
+ * amd64_nbsl_show
+ *
+ *	display back what the last NBSL value written
+ */
+static ssize_t amd64_nbsl_show(struct mem_ctl_info *mci, char *data)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	u32 value;
+
+	value = pvt->ctl_error_info.nbsl;
+
+	return sprintf(data, "%x\n", value);
+}
+
+/*
+ * amd64_nbsh_store
+ *
+ *	accept and store the NBSH value user desires
+ */
+static ssize_t amd64_nbsh_store(struct mem_ctl_info *mci,
+					const char *data, size_t count)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	unsigned long value;
+	int rc;
+
+	rc = strict_strtoul(data, 16, &value);
+	if (rc != -EINVAL) {
+		debugf0("%s() received NBSL= 0x%lx\n", __func__, value);
+
+		/* place the NBSL value into the virtual error packet */
+		pvt->ctl_error_info.nbsh = (u32) value;
+
+		return count;
+	}
+	return 0;
+}
+
+/*
+ * amd64_nbsh_show
+ *
+ *	display back what the last NBSL value written
+ */
+static ssize_t amd64_nbsh_show(struct mem_ctl_info *mci, char *data)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	u32 value;
+
+	value = pvt->ctl_error_info.nbsh;
+
+	return sprintf(data, "%x\n", value);
+}
+
+/*
+ * amd64_nbcfg_store
+ *
+ *	accept and store the NBSL value user desires
+ */
+static ssize_t amd64_nbcfg_store(struct mem_ctl_info *mci,
+					const char *data, size_t count)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	unsigned long value;
+	int rc;
+
+	rc = strict_strtoul(data, 16, &value);
+	if (rc != -EINVAL) {
+		debugf0("%s() received NBCFG= 0x%lx\n", __func__, value);
+
+		/* place the NBSL value into the virtual error packet */
+		pvt->ctl_error_info.nbcfg = (u32) value;
+
+		return count;
+	}
+	return 0;
+}
+
+/*
+ *	Various show routines for the controls of a MCI
+ */
+static ssize_t amd64_nbcfg_show(struct mem_ctl_info *mci, char *data)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	return sprintf(data, "%x\n",
+		pvt->ctl_error_info.nbcfg);
+}
+
+
+static ssize_t amd64_dhar_show(struct mem_ctl_info *mci, char *data)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	return sprintf(data, "%x\n", pvt->dhar);
+}
+
+
+static ssize_t amd64_dbam_show(struct mem_ctl_info *mci, char *data)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	return sprintf(data, "%x\n", pvt->dbam0);
+}
+
+
+static ssize_t amd64_topmem_show(struct mem_ctl_info *mci, char *data)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	return sprintf(data, "%lx\n", (long unsigned int) pvt->top_mem);
+}
+
+
+static ssize_t amd64_topmem2_show(struct mem_ctl_info *mci, char *data)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	return sprintf(data, "%lx\n", (long unsigned int) pvt->top_mem2);
+}
+
+static ssize_t amd64_hole_show(struct mem_ctl_info *mci, char *data)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	u64 hole_base = 0;
+	u64 hole_offset = 0;
+	u64 hole_size = 0;
+
+	pvt->ops->get_dram_hole_info(mci,
+				&hole_base, &hole_offset, &hole_size);
+
+	return sprintf(data, "%lx %lx %lx\n",
+		(long unsigned int) hole_base,
+		(long unsigned int) hole_offset,
+		(long unsigned int) hole_size);
+}
+
+
+/*
+ * amd64_inject_section_store
+ *
+ *	accept and store error injection section value
+ */
+static ssize_t amd64_inject_section_store(struct mem_ctl_info *mci,
+					const char *data, size_t count)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	unsigned long value;
+	int rc;
+
+	rc = strict_strtoul(data, 10, &value);
+	if (rc != -EINVAL) {
+
+		/* save the 16-byte cache section */
+		pvt->injection.section = (u32) value;
+
+		return count;
+	}
+	return 0;
+}
+
+/*
+ * amd64_inject_word_store
+ *
+ *	accept and store error injection word value
+ */
+static ssize_t amd64_inject_word_store(struct mem_ctl_info *mci,
+					const char *data, size_t count)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	unsigned long value;
+	int rc;
+
+	rc = strict_strtoul(data, 10, &value);
+	if (rc != -EINVAL) {
+
+		/* save the 16-bit word */
+		pvt->injection.word = (u32) value;
+
+		return count;
+	}
+	return 0;
+}
+
+/*
+ * amd64_inject_bit_store
+ *
+ *	accept and store error injection bit value
+ */
+static ssize_t amd64_inject_bit_store(struct mem_ctl_info *mci,
+					const char *data, size_t count)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	unsigned long value;
+	int rc;
+
+	rc = strict_strtoul(data, 10, &value);
+	if (rc != -EINVAL) {
+
+		/* save the bit within the 16-bit word */
+		pvt->injection.bit_map = (u32) value;
+
+		return count;
+	}
+	return 0;
+}
+
+/*
+ * amd64_inject_read_store
+ *
+ *	READ action. When called, assemble staged values in the pvt
+ *	area and format into fields needed by the Injection hardware
+ *	Output to hardware and issue a READ operation
+ */
+static ssize_t amd64_inject_read_store(struct mem_ctl_info *mci,
+					const char *data, size_t count)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	unsigned long value;
+	u32 section, word_bits;
+	int rc;
+	int err;
+
+	rc = strict_strtoul(data, 10, &value);
+	if (rc != -EINVAL) {
+
+		/* Form value to choose 16-byte section of cacheline */
+		section = F10_NB_ARRAY_DRAM_ECC |
+				SET_NB_ARRAY_ADDRESS(pvt->injection.section);
+		err = pci_write_config_dword(pvt->misc_f3_ctl,
+					F10_NB_ARRAY_ADDR, section);
+
+		word_bits = SET_NB_DRAM_INJECTION_READ(pvt->injection.word,
+						pvt->injection.bit_map);
+
+		/* Issue 'word' and 'bit' along with the READ request now */
+		err = pci_write_config_dword(pvt->misc_f3_ctl,
+					F10_NB_ARRAY_DATA, word_bits);
+
+		debugf0("%s() section=0x%x word_bits=0x%x\n", __func__,
+			section, word_bits);
+
+		return count;
+	}
+	return 0;
+}
+
+/*
+ * amd64_inject_write_store
+ *
+ *	WRITE action. When called, assemble staged values in the pvt
+ *	area and format into fields needed by the Injection hardware
+ *	Output to hardware and issue a WRITE operation
+ */
+static ssize_t amd64_inject_write_store(struct mem_ctl_info *mci,
+					const char *data, size_t count)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+	unsigned long value;
+	u32 section, word_bits;
+	int rc;
+	int err;
+
+	rc = strict_strtoul(data, 10, &value);
+	if (rc != -EINVAL) {
+
+		/* Form value to choose 16-byte section of cacheline */
+		section = F10_NB_ARRAY_DRAM_ECC |
+				SET_NB_ARRAY_ADDRESS(pvt->injection.section);
+		err = pci_write_config_dword(pvt->misc_f3_ctl,
+					F10_NB_ARRAY_ADDR, section);
+
+		word_bits = SET_NB_DRAM_INJECTION_WRITE(pvt->injection.word,
+						pvt->injection.bit_map);
+
+		/* Issue 'word' and 'bit' along with the READ request now */
+		err = pci_write_config_dword(pvt->misc_f3_ctl,
+					F10_NB_ARRAY_DATA, word_bits);
+
+		debugf0("%s() section=0x%x word_bits=0x%x\n", __func__,
+			section, word_bits);
+
+		return count;
+	}
+	return 0;
+}
+
+/*
+ * Per MC instance Attribute/Control data control structure
+ */
+static struct mcidev_sysfs_attribute amd64_mc_sysfs_ctls_attrs[] = {
+	/* Error injection methods */
+	{
+		.attr = {
+			.name = "z_inject_section",
+			.mode = (S_IRUGO | S_IWUSR)
+		},
+		.show = NULL,
+		.store = amd64_inject_section_store,
+	},
+	{
+		.attr = {
+			.name = "z_inject_word",
+			.mode = (S_IRUGO | S_IWUSR)
+		},
+		.show = NULL,
+		.store = amd64_inject_word_store,
+	},
+	{
+		.attr = {
+			.name = "z_inject_bit_map",
+			.mode = (S_IRUGO | S_IWUSR)
+		},
+		.show = NULL,
+		.store = amd64_inject_bit_store,
+	},
+	{
+		.attr = {
+			.name = "z_inject_write",
+			.mode = (S_IRUGO | S_IWUSR)
+		},
+		.show = NULL,
+		.store = amd64_inject_write_store,
+	},
+	{
+		.attr = {
+			.name = "z_inject_read",
+			.mode = (S_IRUGO | S_IWUSR)
+		},
+		.show = NULL,
+		.store = amd64_inject_read_store,
+	},
+
+
+	/* RAW register accessors */
+	{
+		.attr = {
+			.name = "zctl_nbea",
+			.mode = (S_IRUGO | S_IWUSR)
+		},
+		.show = amd64_nbea_show,
+		.store = amd64_nbea_store,
+	},
+	{
+		.attr = {
+			.name = "zctl_nbsl",
+			.mode = (S_IRUGO | S_IWUSR)
+		},
+		.show = amd64_nbsl_show,
+		.store = amd64_nbsl_store,
+	},
+	{
+		.attr = {
+			.name = "zctl_nbsh",
+			.mode = (S_IRUGO | S_IWUSR)
+		},
+		.show = amd64_nbsh_show,
+		.store = amd64_nbsh_store,
+	},
+	{
+		.attr = {
+			.name = "zctl_nbcfg",
+			.mode = (S_IRUGO | S_IWUSR)
+		},
+		.show = amd64_nbcfg_show,
+		.store = amd64_nbcfg_store,
+	},
+	{
+		.attr = {
+			.name = "zhw_dhar",
+			.mode = (S_IRUGO)
+		},
+		.show = amd64_dhar_show,
+		.store = NULL,
+	},
+	{
+		.attr = {
+			.name = "zhw_dbam",
+			.mode = (S_IRUGO)
+		},
+		.show = amd64_dbam_show,
+		.store = NULL,
+	},
+	{
+		.attr = {
+			.name = "zhw_topmem",
+			.mode = (S_IRUGO)
+		},
+		.show = amd64_topmem_show,
+		.store = NULL,
+	},
+	{
+		.attr = {
+			.name = "zhw_topmem2",
+			.mode = (S_IRUGO)
+		},
+		.show = amd64_topmem2_show,
+		.store = NULL,
+	},
+	{
+		.attr = {
+			.name = "zhw_hole",
+			.mode = (S_IRUGO)
+		},
+		.show = amd64_hole_show,
+		.store = NULL,
+	},
+
+	/* End of List */
+	{
+		.attr = { .name = NULL}
+	}
+};
+
+/*
+ * amd64_set_mc_sysfs_attributes
+ */
+static void amd64_set_mc_sysfs_attributes(struct mem_ctl_info *mci)
+{
+	mci->mc_driver_sysfs_attributes = amd64_mc_sysfs_ctls_attrs;
+}
+
+/*
+ * setup_mci_misc_attributes
+ *
+ *	initialize various attributes of the mci structure
+ */
+static void setup_mci_misc_attributes(struct mem_ctl_info *mci)
+{
+	struct amd64_pvt *pvt = mci->pvt_info;
+
+	/* Initialize various states */
+	mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
+	mci->edac_ctl_cap = EDAC_FLAG_NONE;
+	mci->edac_cap = EDAC_FLAG_NONE;
+
+	/* Exam the capabilities of the northbridge in order to reflect them
+	 * in the presentation via sysfs attributes, etc
+	 */
+	if (pvt->nbcap & K8_NBCAP_SECDED)
+		mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
+
+	if (pvt->nbcap & K8_NBCAP_CHIPKILL)
+		mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
+
+	/* What type of SECDED is there? */
+	mci->edac_cap = pvt->ops->determine_edac_cap(pvt);
+
+	/* Misc attributes to set */
+	mci->mod_name = EDAC_MOD_STR;
+	mci->mod_ver = EDAC_AMD64_VERSION;
+	mci->ctl_name = get_amd_family_name(pvt->mc_type_index);
+	mci->dev_name = pci_name(pvt->dram_f2_ctl);
+	mci->ctl_page_to_phys = NULL;
+
+	/* IMPORTANT: Set the polling 'check' function in this module */
+	mci->edac_check = amd64_check;
+
+	/* memory scrubber interface */
+	mci->set_sdram_scrub_rate = set_sdram_scrub_rate;
+	mci->get_sdram_scrub_rate = get_sdram_scrub_rate;
+}
+
+/*
+ * amd64_probe_one_instance
+ *
+ *	probe function to determine if there is a DRAM Controller
+ *	device is present and to construct data tables for it,
+ *
+ *	Due to a hardware feature on Family 10H cpus, the
+ *	Enable Extended Configuration Space feature MUST be
+ *	enabled on ALL Processors prior to actually reading
+ *	from the ECS registers. Since the loading of the module
+ *	can occur on any 'core', and cores don't 'see' all the
+ *	other processors ECS data when the others are NOT enabled
+ *	Our solution is to first enabled ECS access in this routine
+ *	on all processors, gather some data in a amd64_pvt structure
+ *	and later come back in a 'finishup_setup' function to
+ *	perform that final initialization.
+ *
+ *	See amd64_init_2nd_stage() for the 2nd stage
+ */
+static int amd64_probe_one_instance(struct pci_dev *dram_f2_ctl,
+					int mc_type_index)
+{
+	struct amd64_pvt *pvt;
+	int err;
+	int rc;
+
+	/* Get a temporary 'private' structure for initialization */
+	pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
+	if (pvt == NULL) {
+		rc = -ENOMEM;
+		goto exit_now;
+	}
+
+	/* get the index of this MC */
+	pvt->mc_node_id = get_mc_node_id_from_pdev(dram_f2_ctl);
+
+	debugf0("=========== %s(Instance= %d) ===========\n",
+		__func__, pvt->mc_node_id);
+
+	/* fill in other fields about this instance that we can */
+	pvt->dram_f2_ctl = dram_f2_ctl;
+	pvt->ext_model = node_rev(pvt->mc_node_id);
+	pvt->mc_type_index = mc_type_index;
+	pvt->ops = get_amd_family_ops(mc_type_index);
+
+	/*
+	 * We have the dram_f2_ctl device as an argument,
+	 * now go reserved its sibling devices from the PCI system
+	 */
+	err = reserve_mc_sibling_devices(pvt, mc_type_index);
+	if (err) {
+		rc = -ENODEV;
+		goto exit_now;
+	}
+
+	/* Check hardware to see if this module can support HW at this time */
+	err = pvt->ops->probe_valid_hardware(pvt);
+	if (err) {
+		rc = -ENODEV;
+		goto exit_release_devices;
+	}
+
+	/* Determine CPU Rev of each CPU. On
+	 * Rev E and earlier we have to do the CPUID instruction.
+	 * Rev F and later we could just do a dword read from
+	 * Function 3 Offset FCh
+	 */
+	build_node_revision_table();
+
+	/* IS ECC enabled? An abort point */
+	rc = amd64_check_ecc_enabled(pvt);
+	if (rc)
+		goto exit_release_devices;
+
+	/* Key operation here: setup of HW prior to performing ops on it.
+	 * Some setup is required to access ECS data.
+	 * After this is performed, then the 'teardown' function must
+	 * be called upon error and normal exit paths.
+	 */
+	pvt->ops->setup(pvt);
+
+	/* Save the pointer to the private data for use in
+	 * 2nd stage initialization
+	 */
+	pvt_lookup[pvt->mc_node_id] = pvt;
+
+	debugf0("%s(): init 1st stage done pvt-%d\n", __func__,
+		pvt->mc_node_id);
+	return 0;
+
+/* Error unwind stack */
+
+exit_release_devices: /* Release the sibling devices */
+	pci_dev_put(pvt->addr_f1_ctl);
+	pci_dev_put(pvt->misc_f3_ctl);
+
+exit_now:
+	return rc;
+}
+
+/*
+ * amd64_init_2nd_stage
+ *
+ *	this is the "finishing" up initialization code
+ *	Needs to be performed after all MCs' Hardware have been
+ *	"prep'ed" for accessing extended config space.
+ */
+static int amd64_init_2nd_stage(struct amd64_pvt *pvt_temp)
+{
+	int node_id = pvt_temp->mc_node_id;
+	struct mem_ctl_info *mci;
+	struct amd64_pvt *pvt;
+	int rc;
+
+	debugf0("%s()\n", __func__);
+
+	/* We need to determine how many memory channels there are.
+	 * Then use that information for calculating the size of the dynamic
+	 * instance tables in the 'mci' structure
+	 */
+	pvt_temp->channel_count = pvt_temp->ops->early_channel_count(pvt_temp);
+
+	/* Allocate our MC control structure */
+	mci = edac_mc_alloc(sizeof(*pvt_temp),
+				CHIPSELECT_COUNT,
+				pvt_temp->channel_count,
+				node_id);
+	if (mci == NULL) {
+		rc = -ENOMEM;
+		goto exit_alloc_failed;
+	}
+
+	/* start filling in the mci data and its private data area */
+	mci->dev = &pvt_temp->dram_f2_ctl->dev;
+
+	/* transfer the info from the interium pvt area to the private
+	 * area of the MC instance structure
+	 */
+	pvt = mci->pvt_info;
+	*pvt = *pvt_temp;
+
+	/* proceed with actual Hardware register retrieval */
+	get_top_mem_regs(pvt);		/* Retrieve TOP_MEM and TOP_MEM2 */
+	amd64_read_mc_registers(pvt);	/* get HW regs and save them */
+
+	setup_mci_misc_attributes(mci);	/* Set fields of MCI */
+
+	/* Now initialize control structures for all CSROWs present */
+	if (amd64_init_csrows(mci)) {
+		debugf1("Setting mci->edac_cap to EDAC_FLAG_NONE because\n");
+		debugf1("   amd64_init_csrows() returned NO csrows found\n");
+		mci->edac_cap = EDAC_FLAG_NONE;	/* no csrows found */
+	}
+
+	/* Enable the ECC reporting hardware if
+	 * ECC was disabled by BIOS and ecc_enable_override is set
+	 */
+	amd64_enable_ecc_error_reporting(mci);
+
+	/* Set table ptr for SYSFS controls and attributes for this MC */
+	amd64_set_mc_sysfs_attributes(mci);
+
+	/* Finalize the registration with the EDAC Helper CORE */
+	if (edac_mc_add_mc(mci)) {
+		debugf1("%s(): failed edac_mc_add_mc()\n", __func__);
+		rc = -ENODEV;
+		goto exit_add_mc_failure;
+	}
+
+	debugf0("%s(): init 2nd stage done mci%d\n", __func__,
+		pvt->mc_node_id);
+
+	/* set MCI entry into global mci lookup list */
+	mci_lookup[node_id] = mci;
+
+	/* release the temporary private data buffer */
+	kfree(pvt_lookup[pvt->mc_node_id]);
+	pvt_lookup[node_id] = NULL;
+	return 0;
+
+	/* Error unwind stack */
+exit_add_mc_failure:
+	edac_mc_free(mci);
+
+exit_alloc_failed:
+	debugf0("%s() failure init 2nd stage: rc=%d\n", __func__, rc);
+
+	/* unwind this instance of operations performed in the 1st stage */
+	amd64_restore_ecc_error_reporting(pvt);
+	pvt->ops->teardown(pvt);
+	pci_dev_put(pvt->addr_f1_ctl);
+	pci_dev_put(pvt->misc_f3_ctl);
+
+	/* release the temporary private data buffer */
+	kfree(pvt_lookup[pvt->mc_node_id]);
+	pvt_lookup[node_id] = NULL;
+
+	return rc;
+}
+
+
+/*
+ * amd64_init_one_instance
+ *
+ *	initialize just one device
+ *
+ *	returns:
+ *		 count (>= 0), or
+ *		negative on error
+ */
+static int __devinit amd64_init_one_instance(struct pci_dev *pdev,
+				 const struct pci_device_id *mc_type)
+{
+	int rc;
+
+	debugf0("%s(MC node=%d,mc_type='%s')\n",
+		__func__,
+		get_mc_node_id_from_pdev(pdev),
+		get_amd_family_name(mc_type->driver_data));
+
+	/* wake up and enable device */
+	rc = pci_enable_device(pdev);
+	if (rc < 0)
+		rc = -EIO;
+	else
+		rc = amd64_probe_one_instance(pdev, mc_type->driver_data);
+
+	if (rc < 0)
+		debugf0("%s() rc=%d\n", __func__, rc);
+
+	return rc;
+}
+
+/*
+ * amd64_remove_one_instance
+ *
+ *	remove just one device instance upon driver unloading
+ */
+static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
+{
+	struct mem_ctl_info *mci;
+	struct amd64_pvt *pvt;
+
+	debugf0("%s()\n", __func__);
+
+	/* Remove from EDAC CORE tracking list */
+	mci = edac_mc_del_mc(&pdev->dev);
+	if (mci == NULL)
+		return;
+
+	pvt = mci->pvt_info;
+
+	/* restore the HW ECC detecting/reporting control bits to
+	 * what they were prior to our module being loaded
+	 */
+	amd64_restore_ecc_error_reporting(pvt);
+
+	/* clean up initial hardware mods */
+	pvt->ops->teardown(pvt);
+
+	/* Release the PCI holds on these PCI devices */
+	pci_dev_put(pvt->addr_f1_ctl);
+	pci_dev_put(pvt->misc_f3_ctl);
+
+	/* clear the lookup table entry */
+	mci_lookup[pvt->mc_node_id] = NULL;
+
+	/* Free the EDAC CORE resources */
+	edac_mc_free(mci);
+}
+
+/*
+ * The 'pci_device_id' table.
+ *
+ *	This table is part of the interface for loading drivers for PCI
+ *	devices. The PCI core identifies what devices are on a system
+ *	during boot, and then inquiry this table to see if this driver
+ *	is for a given device found.
+ *
+ *	The PCI helpper functions walk this table and call the
+ *	'.probe' function of the 'pci_driver' table, for each
+ *	instance in this table
+ */
+static const struct pci_device_id amd64_pci_table[] __devinitdata = {
+	{
+		/* Athlon64/Opteron  Rev F and prior */
+		.vendor = PCI_VENDOR_ID_AMD,
+		.device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
+		.subvendor = PCI_ANY_ID,
+		.subdevice = PCI_ANY_ID,
+		.class = 0,
+		.class_mask = 0,
+		.driver_data = K8_CPUS
+	},
+	{
+		/* Barcelona, Family 10h */
+		.vendor = PCI_VENDOR_ID_AMD,
+		.device = PCI_DEVICE_ID_AMD_F10_NB_MEMCTL,
+		.subvendor = PCI_ANY_ID,
+		.subdevice = PCI_ANY_ID,
+		.class = 0,
+		.class_mask = 0,
+		.driver_data = F10_CPUS
+	},
+	{
+		/* Family 11h */
+		.vendor = PCI_VENDOR_ID_AMD,
+		.device = PCI_DEVICE_ID_AMD_F11_NB_MEMCTL,
+		.subvendor = PCI_ANY_ID,
+		.subdevice = PCI_ANY_ID,
+		.class = 0,
+		.class_mask = 0,
+		.driver_data = F11_CPUS
+	},
+	{0, }			/* 0 terminated list. */
+};
+MODULE_DEVICE_TABLE(pci, amd64_pci_table);
+
+/*
+ * The 'pci_driver' structure to define the name, probe and removal
+ * functions
+ */
+static struct pci_driver amd64_pci_driver = {
+	.name = EDAC_MOD_STR,
+	.probe = amd64_init_one_instance,
+	.remove = __devexit_p(amd64_remove_one_instance),
+	.id_table = amd64_pci_table,
+};
+
+
+/*
+ * amd64_setup_pci_device
+ *
+ *	setup the PCI Device Driver for monitoring PCI errors
+ */
+static void amd64_setup_pci_device(void)
+{
+	struct mem_ctl_info *mci;
+	struct amd64_pvt *pvt;
+
+	/* allocating ONE generic PCI control info */
+	if (!amd64_ctl_pci) {
+
+		mci = mci_lookup[0];
+		if (mci) {
+			debugf1("%s(): Registering ONE PCI control\n",
+				__func__);
+
+			pvt = mci->pvt_info;
+			amd64_ctl_pci = edac_pci_create_generic_ctl(
+						&pvt->dram_f2_ctl->dev,
+						EDAC_MOD_STR);
+			if (!amd64_ctl_pci) {
+				printk(KERN_WARNING
+					"%s(): Unable to create PCI control\n",
+					__func__);
+				printk(KERN_WARNING
+					"%s(): PCI error report via EDAC "
+					"not setup\n",
+					__func__);
+			}
+		} else {
+			debugf1("%s(): ONE PCI control already registered\n",
+				__func__);
+		}
+	}
+}
+
+/*
+ * amd64_edac_init
+ *
+ *	Module ENTRY point after loading
+ */
+static int __init amd64_edac_init(void)
+{
+	int err;
+	int node;
+
+	/* setup the OPSTATE */
+	opstate_init();
+
+	debugf0("%s() ******************  ENTRY  **********************\n",
+		__func__);
+
+	/* Attempt to register drivers for instances
+	 * DUE to the failure of some 'cores' to access Extended Config Space
+	 * prior to all memory controllers having their ECS register enabled,
+	 * the initialization has been created into 2 stages. Here we
+	 * call for the 1st stage. After all have been enabled, then we
+	 * do the 2nd stage to finishup setup.
+	 */
+	err = pci_register_driver(&amd64_pci_driver);
+
+	/* At this point, the array 'pvt_lookup[]' contains pointers to
+	 * allocated struct amd64_pvt control structures. These will be used
+	 * in the 2nd stage init function to finish initialization of
+	 * the MC instances.
+	 */
+
+	/* if no error occurred on first pass init, then do 2nd pass init */
+	if (!err) {
+		for (node = 0; node < MAXNODE; node++) {
+			if (!pvt_lookup[node])
+				continue;
+
+			/* If any failure then need to clean up */
+			err = amd64_init_2nd_stage(pvt_lookup[node]);
+			if (err) {
+				debugf0("%s() 'finish_setup' stage failed\n",
+					__func__);
+
+				/* undo prior instances' registrations
+				 * and leave as failed
+				 */
+				pci_unregister_driver(&amd64_pci_driver);
+				goto error_exit;
+			}
+		}
+
+		/* Now setup PCI monitor if all went well */
+		amd64_setup_pci_device();
+	}
+
+error_exit:
+	return err;
+}
+
+/*
+ * amd64_edac_exit
+ *
+ *	Module EXIT point prior to unloading
+ */
+static void __exit amd64_edac_exit(void)
+{
+	/* release the pci control structure, if there was one */
+	if (amd64_ctl_pci)
+		edac_pci_release_generic_ctl(amd64_ctl_pci);
+
+	pci_unregister_driver(&amd64_pci_driver);
+}
+
+module_init(amd64_edac_init);
+module_exit(amd64_edac_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
+		"Dave Peterson, Thayne Harbaugh");
+MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
+		EDAC_AMD64_VERSION);
+
+module_param(edac_op_state, int, 0444);
+MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
