Mirrors/u-boot
2
0
Fork 0
mirror of https://github.com/AsahiLinux/u-boot synced 2024-11-06 13:14:27 +00:00
Code Issues Projects Releases Packages Wiki Activity
u-boot/drivers/ddr/altera/sequencer.c
Marek Vasut 9a5a90ad9b ddr: altera: Add DDR2 support to Gen5 driver 
Add DDR2 support to Gen5 DRAM driver. As the DDR2 macro names generated
by Quartus are named differently than the DDR3 ones, use anon unions to
store them in the same structures, without growing their size.

Signed-off-by: Marek Vasut <marex@denx.de>
Cc: Ley Foon Tan <ley.foon.tan@intel.com>
Cc: Simon Goldschmidt <simon.k.r.goldschmidt@gmail.com>
2020-02-05 03:01:57 +01:00

3997 lines
112 KiB
C
Raw Blame History

// SPDX-License-Identifier: BSD-3-Clause
/*
* Copyright Altera Corporation (C) 2012-2015
*/
#include <common.h>
#include <asm/io.h>
#include <asm/arch/sdram.h>
#include <errno.h>
#include <hang.h>
#include "sequencer.h"
static const struct socfpga_sdr_rw_load_manager *sdr_rw_load_mgr_regs =
(struct socfpga_sdr_rw_load_manager *)
(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0x800);
static const struct socfpga_sdr_rw_load_jump_manager *sdr_rw_load_jump_mgr_regs
= (struct socfpga_sdr_rw_load_jump_manager *)
(SDR_PHYGRP_RWMGRGRP_ADDRESS | 0xC00);
static const struct socfpga_sdr_reg_file *sdr_reg_file =
(struct socfpga_sdr_reg_file *)SDR_PHYGRP_REGFILEGRP_ADDRESS;
static const struct socfpga_sdr_scc_mgr *sdr_scc_mgr =
(struct socfpga_sdr_scc_mgr *)
(SDR_PHYGRP_SCCGRP_ADDRESS | 0xe00);
static const struct socfpga_phy_mgr_cmd *phy_mgr_cmd =
(struct socfpga_phy_mgr_cmd *)SDR_PHYGRP_PHYMGRGRP_ADDRESS;
static const struct socfpga_phy_mgr_cfg *phy_mgr_cfg =
(struct socfpga_phy_mgr_cfg *)
(SDR_PHYGRP_PHYMGRGRP_ADDRESS | 0x40);
static const struct socfpga_data_mgr *data_mgr =
(struct socfpga_data_mgr *)SDR_PHYGRP_DATAMGRGRP_ADDRESS;
static const struct socfpga_sdr_ctrl *sdr_ctrl =
(struct socfpga_sdr_ctrl *)SDR_CTRLGRP_ADDRESS;
#define DELTA_D 1
/*
* In order to reduce ROM size, most of the selectable calibration steps are
* decided at compile time based on the user's calibration mode selection,
* as captured by the STATIC_CALIB_STEPS selection below.
*
* However, to support simulation-time selection of fast simulation mode, where
* we skip everything except the bare minimum, we need a few of the steps to
* be dynamic. In those cases, we either use the DYNAMIC_CALIB_STEPS for the
* check, which is based on the rtl-supplied value, or we dynamically compute
* the value to use based on the dynamically-chosen calibration mode
*/
#define DLEVEL 0
#define STATIC_IN_RTL_SIM 0
#define STATIC_SKIP_DELAY_LOOPS 0
#define STATIC_CALIB_STEPS (STATIC_IN_RTL_SIM | CALIB_SKIP_FULL_TEST | \
STATIC_SKIP_DELAY_LOOPS)
#define SKIP_DELAY_LOOP_VALUE_OR_ZERO(non_skip_value) \
((non_skip_value) & seq->skip_delay_mask)
bool dram_is_ddr(const u8 ddr)
{
const struct socfpga_sdram_config *cfg = socfpga_get_sdram_config();
const u8 type = (cfg->ctrl_cfg >> SDR_CTRLGRP_CTRLCFG_MEMTYPE_LSB) &
SDR_CTRLGRP_CTRLCFG_MEMTYPE_MASK;
if (ddr == 2 && type == 1) /* DDR2 */
return true;
if (ddr == 3 && type == 2) /* DDR3 */
return true;
return false;
}
static void set_failing_group_stage(struct socfpga_sdrseq *seq,
u32 group, u32 stage, u32 substage)
{
/*
* Only set the global stage if there was not been any other
* failing group
*/
if (seq->gbl.error_stage == CAL_STAGE_NIL) {
seq->gbl.error_substage = substage;
seq->gbl.error_stage = stage;
seq->gbl.error_group = group;
}
}
static void reg_file_set_group(u16 set_group)
{
clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff0000, set_group << 16);
}
static void reg_file_set_stage(u8 set_stage)
{
clrsetbits_le32(&sdr_reg_file->cur_stage, 0xffff, set_stage & 0xff);
}
static void reg_file_set_sub_stage(u8 set_sub_stage)
{
set_sub_stage &= 0xff;
clrsetbits_le32(&sdr_reg_file->cur_stage, 0xff00, set_sub_stage << 8);
}
/**
* phy_mgr_initialize() - Initialize PHY Manager
*
* Initialize PHY Manager.
*/
static void phy_mgr_initialize(struct socfpga_sdrseq *seq)
{
u32 ratio;
debug("%s:%d\n", __func__, __LINE__);
/* Calibration has control over path to memory */
/*
* In Hard PHY this is a 2-bit control:
* 0: AFI Mux Select
* 1: DDIO Mux Select
*/
writel(0x3, &phy_mgr_cfg->mux_sel);
/* USER memory clock is not stable we begin initialization */
writel(0, &phy_mgr_cfg->reset_mem_stbl);
/* USER calibration status all set to zero */
writel(0, &phy_mgr_cfg->cal_status);
writel(0, &phy_mgr_cfg->cal_debug_info);
/* Init params only if we do NOT skip calibration. */
if ((seq->dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL)
return;
ratio = seq->rwcfg->mem_dq_per_read_dqs /
seq->rwcfg->mem_virtual_groups_per_read_dqs;
seq->param.read_correct_mask_vg = (1 << ratio) - 1;
seq->param.write_correct_mask_vg = (1 << ratio) - 1;
seq->param.read_correct_mask = (1 << seq->rwcfg->mem_dq_per_read_dqs)
- 1;
seq->param.write_correct_mask = (1 << seq->rwcfg->mem_dq_per_write_dqs)
- 1;
}
/**
* set_rank_and_odt_mask() - Set Rank and ODT mask
* @rank: Rank mask
* @odt_mode: ODT mode, OFF or READ_WRITE
*
* Set Rank and ODT mask (On-Die Termination).
*/
static void set_rank_and_odt_mask(struct socfpga_sdrseq *seq,
const u32 rank, const u32 odt_mode)
{
u32 odt_mask_0 = 0;
u32 odt_mask_1 = 0;
u32 cs_and_odt_mask;
if (odt_mode == RW_MGR_ODT_MODE_OFF) {
odt_mask_0 = 0x0;
odt_mask_1 = 0x0;
} else { /* RW_MGR_ODT_MODE_READ_WRITE */
switch (seq->rwcfg->mem_number_of_ranks) {
case 1: /* 1 Rank */
/* Read: ODT = 0 ; Write: ODT = 1 */
odt_mask_0 = 0x0;
odt_mask_1 = 0x1;
break;
case 2: /* 2 Ranks */
if (seq->rwcfg->mem_number_of_cs_per_dimm == 1) {
/*
* - Dual-Slot , Single-Rank (1 CS per DIMM)
* OR
* - RDIMM, 4 total CS (2 CS per DIMM, 2 DIMM)
*
* Since MEM_NUMBER_OF_RANKS is 2, they
* are both single rank with 2 CS each
* (special for RDIMM).
*
* Read: Turn on ODT on the opposite rank
* Write: Turn on ODT on all ranks
*/
odt_mask_0 = 0x3 & ~(1 << rank);
odt_mask_1 = 0x3;
if (dram_is_ddr(2))
odt_mask_1 &= ~(1 << rank);
} else {
/*
* - Single-Slot , Dual-Rank (2 CS per DIMM)
*
* Read: Turn on ODT off on all ranks
* Write: Turn on ODT on active rank
*/
odt_mask_0 = 0x0;
odt_mask_1 = 0x3 & (1 << rank);
}
break;
case 4: /* 4 Ranks */
/*
* DDR3 Read, DDR2 Read/Write:
* ----------+-----------------------+
* | ODT |
* +-----------------------+
* Rank | 3 | 2 | 1 | 0 |
* ----------+-----+-----+-----+-----+
* 0 | 0 | 1 | 0 | 0 |
* 1 | 1 | 0 | 0 | 0 |
* 2 | 0 | 0 | 0 | 1 |
* 3 | 0 | 0 | 1 | 0 |
* ----------+-----+-----+-----+-----+
*
* DDR3 Write:
* ----------+-----------------------+
* | ODT |
* Write To +-----------------------+
* Rank | 3 | 2 | 1 | 0 |
* ----------+-----+-----+-----+-----+
* 0 | 0 | 1 | 0 | 1 |
* 1 | 1 | 0 | 1 | 0 |
* 2 | 0 | 1 | 0 | 1 |
* 3 | 1 | 0 | 1 | 0 |
* ----------+-----+-----+-----+-----+
*/
switch (rank) {
case 0:
odt_mask_0 = 0x4;
if (dram_is_ddr(2))
odt_mask_1 = 0x4;
else if (dram_is_ddr(3))
odt_mask_1 = 0x5;
break;
case 1:
odt_mask_0 = 0x8;
if (dram_is_ddr(2))
odt_mask_1 = 0x8;
else if (dram_is_ddr(3))
odt_mask_1 = 0xA;
break;
case 2:
odt_mask_0 = 0x1;
if (dram_is_ddr(2))
odt_mask_1 = 0x1;
else if (dram_is_ddr(3))
odt_mask_1 = 0x5;
break;
case 3:
odt_mask_0 = 0x2;
if (dram_is_ddr(2))
odt_mask_1 = 0x2;
else if (dram_is_ddr(3))
odt_mask_1 = 0xA;
break;
}
break;
}
}
cs_and_odt_mask = (0xFF & ~(1 << rank)) |
((0xFF & odt_mask_0) << 8) |
((0xFF & odt_mask_1) << 16);
writel(cs_and_odt_mask, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
}
/**
* scc_mgr_set() - Set SCC Manager register
* @off: Base offset in SCC Manager space
* @grp: Read/Write group
* @val: Value to be set
*
* This function sets the SCC Manager (Scan Chain Control Manager) register.
*/
static void scc_mgr_set(u32 off, u32 grp, u32 val)
{
writel(val, SDR_PHYGRP_SCCGRP_ADDRESS | off | (grp << 2));
}
/**
* scc_mgr_initialize() - Initialize SCC Manager registers
*
* Initialize SCC Manager registers.
*/
static void scc_mgr_initialize(void)
{
/*
* Clear register file for HPS. 16 (2^4) is the size of the
* full register file in the scc mgr:
* RFILE_DEPTH = 1 + log2(MEM_DQ_PER_DQS + 1 + MEM_DM_PER_DQS +
* MEM_IF_READ_DQS_WIDTH - 1);
*/
int i;
for (i = 0; i < 16; i++) {
debug_cond(DLEVEL >= 1, "%s:%d: Clearing SCC RFILE index %u\n",
__func__, __LINE__, i);
scc_mgr_set(SCC_MGR_HHP_RFILE_OFFSET, i, 0);
}
}
static void scc_mgr_set_dqdqs_output_phase(u32 write_group, u32 phase)
{
scc_mgr_set(SCC_MGR_DQDQS_OUT_PHASE_OFFSET, write_group, phase);
}
static void scc_mgr_set_dqs_bus_in_delay(u32 read_group, u32 delay)
{
scc_mgr_set(SCC_MGR_DQS_IN_DELAY_OFFSET, read_group, delay);
}
static void scc_mgr_set_dqs_en_phase(u32 read_group, u32 phase)
{
scc_mgr_set(SCC_MGR_DQS_EN_PHASE_OFFSET, read_group, phase);
}
static void scc_mgr_set_dqs_en_delay(u32 read_group, u32 delay)
{
scc_mgr_set(SCC_MGR_DQS_EN_DELAY_OFFSET, read_group, delay);
}
static void scc_mgr_set_dq_in_delay(u32 dq_in_group, u32 delay)
{
scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET, dq_in_group, delay);
}
static void scc_mgr_set_dqs_io_in_delay(struct socfpga_sdrseq *seq,
u32 delay)
{
scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET,
seq->rwcfg->mem_dq_per_write_dqs, delay);
}
static void scc_mgr_set_dm_in_delay(struct socfpga_sdrseq *seq, u32 dm,
u32 delay)
{
scc_mgr_set(SCC_MGR_IO_IN_DELAY_OFFSET,
seq->rwcfg->mem_dq_per_write_dqs + 1 + dm,
delay);
}
static void scc_mgr_set_dq_out1_delay(u32 dq_in_group, u32 delay)
{
scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET, dq_in_group, delay);
}
static void scc_mgr_set_dqs_out1_delay(struct socfpga_sdrseq *seq,
u32 delay)
{
scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
seq->rwcfg->mem_dq_per_write_dqs, delay);
}
static void scc_mgr_set_dm_out1_delay(struct socfpga_sdrseq *seq, u32 dm,
u32 delay)
{
scc_mgr_set(SCC_MGR_IO_OUT1_DELAY_OFFSET,
seq->rwcfg->mem_dq_per_write_dqs + 1 + dm,
delay);
}
/* load up dqs config settings */
static void scc_mgr_load_dqs(u32 dqs)
{
writel(dqs, &sdr_scc_mgr->dqs_ena);
}
/* load up dqs io config settings */
static void scc_mgr_load_dqs_io(void)
{
writel(0, &sdr_scc_mgr->dqs_io_ena);
}
/* load up dq config settings */
static void scc_mgr_load_dq(u32 dq_in_group)
{
writel(dq_in_group, &sdr_scc_mgr->dq_ena);
}
/* load up dm config settings */
static void scc_mgr_load_dm(u32 dm)
{
writel(dm, &sdr_scc_mgr->dm_ena);
}
/**
* scc_mgr_set_all_ranks() - Set SCC Manager register for all ranks
* @off: Base offset in SCC Manager space
* @grp: Read/Write group
* @val: Value to be set
* @update: If non-zero, trigger SCC Manager update for all ranks
*
* This function sets the SCC Manager (Scan Chain Control Manager) register
* and optionally triggers the SCC update for all ranks.
*/
static void scc_mgr_set_all_ranks(struct socfpga_sdrseq *seq,
const u32 off, const u32 grp, const u32 val,
const int update)
{
u32 r;
for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
r += NUM_RANKS_PER_SHADOW_REG) {
scc_mgr_set(off, grp, val);
if (update || (r == 0)) {
writel(grp, &sdr_scc_mgr->dqs_ena);
writel(0, &sdr_scc_mgr->update);
}
}
}
static void scc_mgr_set_dqs_en_phase_all_ranks(struct socfpga_sdrseq *seq,
u32 read_group, u32 phase)
{
/*
* USER although the h/w doesn't support different phases per
* shadow register, for simplicity our scc manager modeling
* keeps different phase settings per shadow reg, and it's
* important for us to keep them in sync to match h/w.
* for efficiency, the scan chain update should occur only
* once to sr0.
*/
scc_mgr_set_all_ranks(seq, SCC_MGR_DQS_EN_PHASE_OFFSET,
read_group, phase, 0);
}
static void scc_mgr_set_dqdqs_output_phase_all_ranks(struct socfpga_sdrseq *seq,
u32 write_group, u32 phase)
{
/*
* USER although the h/w doesn't support different phases per
* shadow register, for simplicity our scc manager modeling
* keeps different phase settings per shadow reg, and it's
* important for us to keep them in sync to match h/w.
* for efficiency, the scan chain update should occur only
* once to sr0.
*/
scc_mgr_set_all_ranks(seq, SCC_MGR_DQDQS_OUT_PHASE_OFFSET,
write_group, phase, 0);
}
static void scc_mgr_set_dqs_en_delay_all_ranks(struct socfpga_sdrseq *seq,
u32 read_group, u32 delay)
{
/*
* In shadow register mode, the T11 settings are stored in
* registers in the core, which are updated by the DQS_ENA
* signals. Not issuing the SCC_MGR_UPD command allows us to
* save lots of rank switching overhead, by calling
* select_shadow_regs_for_update with update_scan_chains
* set to 0.
*/
scc_mgr_set_all_ranks(seq, SCC_MGR_DQS_EN_DELAY_OFFSET,
read_group, delay, 1);
}
/**
* scc_mgr_set_oct_out1_delay() - Set OCT output delay
* @write_group: Write group
* @delay: Delay value
*
* This function sets the OCT output delay in SCC manager.
*/
static void scc_mgr_set_oct_out1_delay(struct socfpga_sdrseq *seq,
const u32 write_group, const u32 delay)
{
const int ratio = seq->rwcfg->mem_if_read_dqs_width /
seq->rwcfg->mem_if_write_dqs_width;
const int base = write_group * ratio;
int i;
/*
* Load the setting in the SCC manager
* Although OCT affects only write data, the OCT delay is controlled
* by the DQS logic block which is instantiated once per read group.
* For protocols where a write group consists of multiple read groups,
* the setting must be set multiple times.
*/
for (i = 0; i < ratio; i++)
scc_mgr_set(SCC_MGR_OCT_OUT1_DELAY_OFFSET, base + i, delay);
}
/**
* scc_mgr_set_hhp_extras() - Set HHP extras.
*
* Load the fixed setting in the SCC manager HHP extras.
*/
static void scc_mgr_set_hhp_extras(void)
{
/*
* Load the fixed setting in the SCC manager
* bits: 0:0 = 1'b1 - DQS bypass
* bits: 1:1 = 1'b1 - DQ bypass
* bits: 4:2 = 3'b001 - rfifo_mode
* bits: 6:5 = 2'b01 - rfifo clock_select
* bits: 7:7 = 1'b0 - separate gating from ungating setting
* bits: 8:8 = 1'b0 - separate OE from Output delay setting
*/
const u32 value = (0 << 8) | (0 << 7) | (1 << 5) |
(1 << 2) | (1 << 1) | (1 << 0);
const u32 addr = SDR_PHYGRP_SCCGRP_ADDRESS |
SCC_MGR_HHP_GLOBALS_OFFSET |
SCC_MGR_HHP_EXTRAS_OFFSET;
debug_cond(DLEVEL >= 1, "%s:%d Setting HHP Extras\n",
__func__, __LINE__);
writel(value, addr);
debug_cond(DLEVEL >= 1, "%s:%d Done Setting HHP Extras\n",
__func__, __LINE__);
}
/**
* scc_mgr_zero_all() - Zero all DQS config
*
* Zero all DQS config.
*/
static void scc_mgr_zero_all(struct socfpga_sdrseq *seq)
{
int i, r;
/*
* USER Zero all DQS config settings, across all groups and all
* shadow registers
*/
for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
r += NUM_RANKS_PER_SHADOW_REG) {
for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
/*
* The phases actually don't exist on a per-rank basis,
* but there's no harm updating them several times, so
* let's keep the code simple.
*/
scc_mgr_set_dqs_bus_in_delay(i,
seq->iocfg->dqs_in_reserve
);
scc_mgr_set_dqs_en_phase(i, 0);
scc_mgr_set_dqs_en_delay(i, 0);
}
for (i = 0; i < seq->rwcfg->mem_if_write_dqs_width; i++) {
scc_mgr_set_dqdqs_output_phase(i, 0);
/* Arria V/Cyclone V don't have out2. */
scc_mgr_set_oct_out1_delay(seq, i,
seq->iocfg->dqs_out_reserve);
}
}
/* Multicast to all DQS group enables. */
writel(0xff, &sdr_scc_mgr->dqs_ena);
writel(0, &sdr_scc_mgr->update);
}
/**
* scc_set_bypass_mode() - Set bypass mode and trigger SCC update
* @write_group: Write group
*
* Set bypass mode and trigger SCC update.
*/
static void scc_set_bypass_mode(const u32 write_group)
{
/* Multicast to all DQ enables. */
writel(0xff, &sdr_scc_mgr->dq_ena);
writel(0xff, &sdr_scc_mgr->dm_ena);
/* Update current DQS IO enable. */
writel(0, &sdr_scc_mgr->dqs_io_ena);
/* Update the DQS logic. */
writel(write_group, &sdr_scc_mgr->dqs_ena);
/* Hit update. */
writel(0, &sdr_scc_mgr->update);
}
/**
* scc_mgr_load_dqs_for_write_group() - Load DQS settings for Write Group
* @write_group: Write group
*
* Load DQS settings for Write Group, do not trigger SCC update.
*/
static void scc_mgr_load_dqs_for_write_group(struct socfpga_sdrseq *seq,
const u32 write_group)
{
const int ratio = seq->rwcfg->mem_if_read_dqs_width /
seq->rwcfg->mem_if_write_dqs_width;
const int base = write_group * ratio;
int i;
/*
* Load the setting in the SCC manager
* Although OCT affects only write data, the OCT delay is controlled
* by the DQS logic block which is instantiated once per read group.
* For protocols where a write group consists of multiple read groups,
* the setting must be set multiple times.
*/
for (i = 0; i < ratio; i++)
writel(base + i, &sdr_scc_mgr->dqs_ena);
}
/**
* scc_mgr_zero_group() - Zero all configs for a group
*
* Zero DQ, DM, DQS and OCT configs for a group.
*/
static void scc_mgr_zero_group(struct socfpga_sdrseq *seq,
const u32 write_group, const int out_only)
{
int i, r;
for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
r += NUM_RANKS_PER_SHADOW_REG) {
/* Zero all DQ config settings. */
for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) {
scc_mgr_set_dq_out1_delay(i, 0);
if (!out_only)
scc_mgr_set_dq_in_delay(i, 0);
}
/* Multicast to all DQ enables. */
writel(0xff, &sdr_scc_mgr->dq_ena);
/* Zero all DM config settings. */
for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
if (!out_only)
scc_mgr_set_dm_in_delay(seq, i, 0);
scc_mgr_set_dm_out1_delay(seq, i, 0);
}
/* Multicast to all DM enables. */
writel(0xff, &sdr_scc_mgr->dm_ena);
/* Zero all DQS IO settings. */
if (!out_only)
scc_mgr_set_dqs_io_in_delay(seq, 0);
/* Arria V/Cyclone V don't have out2. */
scc_mgr_set_dqs_out1_delay(seq, seq->iocfg->dqs_out_reserve);
scc_mgr_set_oct_out1_delay(seq, write_group,
seq->iocfg->dqs_out_reserve);
scc_mgr_load_dqs_for_write_group(seq, write_group);
/* Multicast to all DQS IO enables (only 1 in total). */
writel(0, &sdr_scc_mgr->dqs_io_ena);
/* Hit update to zero everything. */
writel(0, &sdr_scc_mgr->update);
}
}
/*
* apply and load a particular input delay for the DQ pins in a group
* group_bgn is the index of the first dq pin (in the write group)
*/
static void scc_mgr_apply_group_dq_in_delay(struct socfpga_sdrseq *seq,
u32 group_bgn, u32 delay)
{
u32 i, p;
for (i = 0, p = group_bgn; i < seq->rwcfg->mem_dq_per_read_dqs;
i++, p++) {
scc_mgr_set_dq_in_delay(p, delay);
scc_mgr_load_dq(p);
}
}
/**
* scc_mgr_apply_group_dq_out1_delay() - Apply and load an output delay for the
* DQ pins in a group
* @delay: Delay value
*
* Apply and load a particular output delay for the DQ pins in a group.
*/
static void scc_mgr_apply_group_dq_out1_delay(struct socfpga_sdrseq *seq,
const u32 delay)
{
int i;
for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) {
scc_mgr_set_dq_out1_delay(i, delay);
scc_mgr_load_dq(i);
}
}
/* apply and load a particular output delay for the DM pins in a group */
static void scc_mgr_apply_group_dm_out1_delay(struct socfpga_sdrseq *seq,
u32 delay1)
{
u32 i;
for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++) {
scc_mgr_set_dm_out1_delay(seq, i, delay1);
scc_mgr_load_dm(i);
}
}
/* apply and load delay on both DQS and OCT out1 */
static void scc_mgr_apply_group_dqs_io_and_oct_out1(struct socfpga_sdrseq *seq,
u32 write_group, u32 delay)
{
scc_mgr_set_dqs_out1_delay(seq, delay);
scc_mgr_load_dqs_io();
scc_mgr_set_oct_out1_delay(seq, write_group, delay);
scc_mgr_load_dqs_for_write_group(seq, write_group);
}
/**
* scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output
* side: DQ, DM, DQS, OCT
* @write_group: Write group
* @delay: Delay value
*
* Apply a delay to the entire output side: DQ, DM, DQS, OCT.
*/
static void scc_mgr_apply_group_all_out_delay_add(struct socfpga_sdrseq *seq,
const u32 write_group,
const u32 delay)
{
u32 i, new_delay;
/* DQ shift */
for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++)
scc_mgr_load_dq(i);
/* DM shift */
for (i = 0; i < RW_MGR_NUM_DM_PER_WRITE_GROUP; i++)
scc_mgr_load_dm(i);
/* DQS shift */
new_delay = READ_SCC_DQS_IO_OUT2_DELAY + delay;
if (new_delay > seq->iocfg->io_out2_delay_max) {
debug_cond(DLEVEL >= 1,
"%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
__func__, __LINE__, write_group, delay, new_delay,
seq->iocfg->io_out2_delay_max,
new_delay - seq->iocfg->io_out2_delay_max);
new_delay -= seq->iocfg->io_out2_delay_max;
scc_mgr_set_dqs_out1_delay(seq, new_delay);
}
scc_mgr_load_dqs_io();
/* OCT shift */
new_delay = READ_SCC_OCT_OUT2_DELAY + delay;
if (new_delay > seq->iocfg->io_out2_delay_max) {
debug_cond(DLEVEL >= 1,
"%s:%d (%u, %u) DQS: %u > %d; adding %u to OUT1\n",
__func__, __LINE__, write_group, delay,
new_delay, seq->iocfg->io_out2_delay_max,
new_delay - seq->iocfg->io_out2_delay_max);
new_delay -= seq->iocfg->io_out2_delay_max;
scc_mgr_set_oct_out1_delay(seq, write_group, new_delay);
}
scc_mgr_load_dqs_for_write_group(seq, write_group);
}
/**
* scc_mgr_apply_group_all_out_delay_add() - Apply a delay to the entire output
* side to all ranks
* @write_group: Write group
* @delay: Delay value
*
* Apply a delay to the entire output side (DQ, DM, DQS, OCT) to all ranks.
*/
static void
scc_mgr_apply_group_all_out_delay_add_all_ranks(struct socfpga_sdrseq *seq,
const u32 write_group,
const u32 delay)
{
int r;
for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
r += NUM_RANKS_PER_SHADOW_REG) {
scc_mgr_apply_group_all_out_delay_add(seq, write_group, delay);
writel(0, &sdr_scc_mgr->update);
}
}
/**
* set_jump_as_return() - Return instruction optimization
*
* Optimization used to recover some slots in ddr3 inst_rom could be
* applied to other protocols if we wanted to
*/
static void set_jump_as_return(struct socfpga_sdrseq *seq)
{
/*
* To save space, we replace return with jump to special shared
* RETURN instruction so we set the counter to large value so that
* we always jump.
*/
writel(0xff, &sdr_rw_load_mgr_regs->load_cntr0);
writel(seq->rwcfg->rreturn, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
}
/**
* delay_for_n_mem_clocks() - Delay for N memory clocks
* @clocks: Length of the delay
*
* Delay for N memory clocks.
*/
static void delay_for_n_mem_clocks(struct socfpga_sdrseq *seq,
const u32 clocks)
{
u32 afi_clocks;
u16 c_loop;
u8 inner;
u8 outer;
debug("%s:%d: clocks=%u ... start\n", __func__, __LINE__, clocks);
/* Scale (rounding up) to get afi clocks. */
afi_clocks = DIV_ROUND_UP(clocks, seq->misccfg->afi_rate_ratio);
if (afi_clocks) /* Temporary underflow protection */
afi_clocks--;
/*
* Note, we don't bother accounting for being off a little
* bit because of a few extra instructions in outer loops.
* Note, the loops have a test at the end, and do the test
* before the decrement, and so always perform the loop
* 1 time more than the counter value
*/
c_loop = afi_clocks >> 16;
outer = c_loop ? 0xff : (afi_clocks >> 8);
inner = outer ? 0xff : afi_clocks;
/*
* rom instructions are structured as follows:
*
* IDLE_LOOP2: jnz cntr0, TARGET_A
* IDLE_LOOP1: jnz cntr1, TARGET_B
* return
*
* so, when doing nested loops, TARGET_A is set to IDLE_LOOP2, and
* TARGET_B is set to IDLE_LOOP2 as well
*
* if we have no outer loop, though, then we can use IDLE_LOOP1 only,
* and set TARGET_B to IDLE_LOOP1 and we skip IDLE_LOOP2 entirely
*
* a little confusing, but it helps save precious space in the inst_rom
* and sequencer rom and keeps the delays more accurate and reduces
* overhead
*/
if (afi_clocks < 0x100) {
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
&sdr_rw_load_mgr_regs->load_cntr1);
writel(seq->rwcfg->idle_loop1,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
writel(seq->rwcfg->idle_loop1, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET);
} else {
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(inner),
&sdr_rw_load_mgr_regs->load_cntr0);
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(outer),
&sdr_rw_load_mgr_regs->load_cntr1);
writel(seq->rwcfg->idle_loop2,
&sdr_rw_load_jump_mgr_regs->load_jump_add0);
writel(seq->rwcfg->idle_loop2,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
do {
writel(seq->rwcfg->idle_loop2,
SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET);
} while (c_loop-- != 0);
}
debug("%s:%d clocks=%u ... end\n", __func__, __LINE__, clocks);
}
static void delay_for_n_ns(struct socfpga_sdrseq *seq, const u32 ns)
{
delay_for_n_mem_clocks(seq, (ns * seq->misccfg->afi_clk_freq *
seq->misccfg->afi_rate_ratio) / 1000);
}
/**
* rw_mgr_mem_init_load_regs() - Load instruction registers
* @cntr0: Counter 0 value
* @cntr1: Counter 1 value
* @cntr2: Counter 2 value
* @jump: Jump instruction value
*
* Load instruction registers.
*/
static void rw_mgr_mem_init_load_regs(struct socfpga_sdrseq *seq,
u32 cntr0, u32 cntr1, u32 cntr2, u32 jump)
{
u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET;
/* Load counters */
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr0),
&sdr_rw_load_mgr_regs->load_cntr0);
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr1),
&sdr_rw_load_mgr_regs->load_cntr1);
writel(SKIP_DELAY_LOOP_VALUE_OR_ZERO(cntr2),
&sdr_rw_load_mgr_regs->load_cntr2);
/* Load jump address */
writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add1);
writel(jump, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
/* Execute count instruction */
writel(jump, grpaddr);
}
/**
* rw_mgr_mem_load_user_ddr2() - Load user calibration values for DDR2
* @handoff: Indicate whether this is initialization or handoff phase
*
* Load user calibration values and optionally precharge the banks.
*/
static void rw_mgr_mem_load_user_ddr2(struct socfpga_sdrseq *seq,
const int handoff)
{
u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET;
u32 r;
for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) {
/* set rank */
set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF);
/* precharge all banks ... */
writel(seq->rwcfg->precharge_all, grpaddr);
writel(seq->rwcfg->emr2, grpaddr);
writel(seq->rwcfg->emr3, grpaddr);
writel(seq->rwcfg->emr, grpaddr);
if (handoff) {
writel(seq->rwcfg->mr_user, grpaddr);
continue;
}
writel(seq->rwcfg->mr_dll_reset, grpaddr);
writel(seq->rwcfg->precharge_all, grpaddr);
writel(seq->rwcfg->refresh, grpaddr);
delay_for_n_ns(seq, 200);
writel(seq->rwcfg->refresh, grpaddr);
delay_for_n_ns(seq, 200);
writel(seq->rwcfg->mr_calib, grpaddr);
writel(/*seq->rwcfg->*/0x0b, grpaddr); // EMR_OCD_ENABLE
writel(seq->rwcfg->emr, grpaddr);
delay_for_n_mem_clocks(seq, 200);
}
}
/**
* rw_mgr_mem_load_user_ddr3() - Load user calibration values
* @fin1: Final instruction 1
* @fin2: Final instruction 2
* @precharge: If 1, precharge the banks at the end
*
* Load user calibration values and optionally precharge the banks.
*/
static void rw_mgr_mem_load_user_ddr3(struct socfpga_sdrseq *seq,
const u32 fin1, const u32 fin2,
const int precharge)
{
u32 grpaddr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET;
u32 r;
for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) {
/* set rank */
set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF);
/* precharge all banks ... */
if (precharge)
writel(seq->rwcfg->precharge_all, grpaddr);
/*
* USER Use Mirror-ed commands for odd ranks if address
* mirrorring is on
*/
if ((seq->rwcfg->mem_address_mirroring >> r) & 0x1) {
set_jump_as_return(seq);
writel(seq->rwcfg->mrs2_mirr, grpaddr);
delay_for_n_mem_clocks(seq, 4);
set_jump_as_return(seq);
writel(seq->rwcfg->mrs3_mirr, grpaddr);
delay_for_n_mem_clocks(seq, 4);
set_jump_as_return(seq);
writel(seq->rwcfg->mrs1_mirr, grpaddr);
delay_for_n_mem_clocks(seq, 4);
set_jump_as_return(seq);
writel(fin1, grpaddr);
} else {
set_jump_as_return(seq);
writel(seq->rwcfg->mrs2, grpaddr);
delay_for_n_mem_clocks(seq, 4);
set_jump_as_return(seq);
writel(seq->rwcfg->mrs3, grpaddr);
delay_for_n_mem_clocks(seq, 4);
set_jump_as_return(seq);
writel(seq->rwcfg->mrs1, grpaddr);
set_jump_as_return(seq);
writel(fin2, grpaddr);
}
if (precharge)
continue;
set_jump_as_return(seq);
writel(seq->rwcfg->zqcl, grpaddr);
/* tZQinit = tDLLK = 512 ck cycles */
delay_for_n_mem_clocks(seq, 512);
}
}
/**
* rw_mgr_mem_load_user() - Load user calibration values
* @fin1: Final instruction 1
* @fin2: Final instruction 2
* @precharge: If 1, precharge the banks at the end
*
* Load user calibration values and optionally precharge the banks.
*/
static void rw_mgr_mem_load_user(struct socfpga_sdrseq *seq,
const u32 fin1, const u32 fin2,
const int precharge)
{
if (dram_is_ddr(2))
rw_mgr_mem_load_user_ddr2(seq, precharge);
else if (dram_is_ddr(3))
rw_mgr_mem_load_user_ddr3(seq, fin1, fin2, precharge);
else
hang();
}
/**
* rw_mgr_mem_initialize() - Initialize RW Manager
*
* Initialize RW Manager.
*/
static void rw_mgr_mem_initialize(struct socfpga_sdrseq *seq)
{
debug("%s:%d\n", __func__, __LINE__);
/* The reset / cke part of initialization is broadcasted to all ranks */
if (dram_is_ddr(3)) {
writel(RW_MGR_RANK_ALL, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_SET_CS_AND_ODT_MASK_OFFSET);
}
/*
* Here's how you load register for a loop
* Counters are located @ 0x800
* Jump address are located @ 0xC00
* For both, registers 0 to 3 are selected using bits 3 and 2, like
* in 0x800, 0x804, 0x808, 0x80C and 0xC00, 0xC04, 0xC08, 0xC0C
* I know this ain't pretty, but Avalon bus throws away the 2 least
* significant bits
*/
/* Start with memory RESET activated */
/* tINIT = 200us */
/*
* 200us @ 266MHz (3.75 ns) ~ 54000 clock cycles
* If a and b are the number of iteration in 2 nested loops
* it takes the following number of cycles to complete the operation:
* number_of_cycles = ((2 + n) * a + 2) * b
* where n is the number of instruction in the inner loop
* One possible solution is n = 0 , a = 256 , b = 106 => a = FF,
* b = 6A
*/
rw_mgr_mem_init_load_regs(seq, seq->misccfg->tinit_cntr0_val,
seq->misccfg->tinit_cntr1_val,
seq->misccfg->tinit_cntr2_val,
seq->rwcfg->init_reset_0_cke_0);
/* Indicate that memory is stable. */
writel(1, &phy_mgr_cfg->reset_mem_stbl);
if (dram_is_ddr(2)) {
writel(seq->rwcfg->nop, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET);
/* Bring up clock enable. */
/* tXRP < 400 ck cycles */
delay_for_n_ns(seq, 400);
} else if (dram_is_ddr(3)) {
/*
* transition the RESET to high
* Wait for 500us
*/
/*
* 500us @ 266MHz (3.75 ns) ~ 134000 clock cycles
* If a and b are the number of iteration in 2 nested loops
* it takes the following number of cycles to complete the
* operation number_of_cycles = ((2 + n) * a + 2) * b
* where n is the number of instruction in the inner loop
* One possible solution is
* n = 2 , a = 131 , b = 256 => a = 83, b = FF
*/
rw_mgr_mem_init_load_regs(seq, seq->misccfg->treset_cntr0_val,
seq->misccfg->treset_cntr1_val,
seq->misccfg->treset_cntr2_val,
seq->rwcfg->init_reset_1_cke_0);
/* Bring up clock enable. */
/* tXRP < 250 ck cycles */
delay_for_n_mem_clocks(seq, 250);
}
rw_mgr_mem_load_user(seq, seq->rwcfg->mrs0_dll_reset_mirr,
seq->rwcfg->mrs0_dll_reset, 0);
}
/**
* rw_mgr_mem_handoff() - Hand off the memory to user
*
* At the end of calibration we have to program the user settings in
* and hand off the memory to the user.
*/
static void rw_mgr_mem_handoff(struct socfpga_sdrseq *seq)
{
rw_mgr_mem_load_user(seq, seq->rwcfg->mrs0_user_mirr,
seq->rwcfg->mrs0_user, 1);
/*
* Need to wait tMOD (12CK or 15ns) time before issuing other
* commands, but we will have plenty of NIOS cycles before actual
* handoff so its okay.
*/
}
/**
* rw_mgr_mem_calibrate_write_test_issue() - Issue write test command
* @group: Write Group
* @use_dm: Use DM
*
* Issue write test command. Two variants are provided, one that just tests
* a write pattern and another that tests datamask functionality.
*/
static void rw_mgr_mem_calibrate_write_test_issue(struct socfpga_sdrseq *seq,
u32 group, u32 test_dm)
{
const u32 quick_write_mode =
(STATIC_CALIB_STEPS & CALIB_SKIP_WRITES) &&
seq->misccfg->enable_super_quick_calibration;
u32 mcc_instruction;
u32 rw_wl_nop_cycles;
/*
* Set counter and jump addresses for the right
* number of NOP cycles.
* The number of supported NOP cycles can range from -1 to infinity
* Three different cases are handled:
*
* 1. For a number of NOP cycles greater than 0, the RW Mgr looping
* mechanism will be used to insert the right number of NOPs
*
* 2. For a number of NOP cycles equals to 0, the micro-instruction
* issuing the write command will jump straight to the
* micro-instruction that turns on DQS (for DDRx), or outputs write
* data (for RLD), skipping
* the NOP micro-instruction all together
*
* 3. A number of NOP cycles equal to -1 indicates that DQS must be
* turned on in the same micro-instruction that issues the write
* command. Then we need
* to directly jump to the micro-instruction that sends out the data
*
* NOTE: Implementing this mechanism uses 2 RW Mgr jump-counters
* (2 and 3). One jump-counter (0) is used to perform multiple
* write-read operations.
* one counter left to issue this command in "multiple-group" mode
*/
rw_wl_nop_cycles = seq->gbl.rw_wl_nop_cycles;
if (rw_wl_nop_cycles == -1) {
/*
* CNTR 2 - We want to execute the special write operation that
* turns on DQS right away and then skip directly to the
* instruction that sends out the data. We set the counter to a
* large number so that the jump is always taken.
*/
writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
/* CNTR 3 - Not used */
if (test_dm) {
mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0_wl_1;
writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_data,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_nop,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
} else {
mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0_wl_1;
writel(seq->rwcfg->lfsr_wr_rd_bank_0_data,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
writel(seq->rwcfg->lfsr_wr_rd_bank_0_nop,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
}
} else if (rw_wl_nop_cycles == 0) {
/*
* CNTR 2 - We want to skip the NOP operation and go straight
* to the DQS enable instruction. We set the counter to a large
* number so that the jump is always taken.
*/
writel(0xFF, &sdr_rw_load_mgr_regs->load_cntr2);
/* CNTR 3 - Not used */
if (test_dm) {
mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0;
writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_dqs,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
} else {
mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0;
writel(seq->rwcfg->lfsr_wr_rd_bank_0_dqs,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
}
} else {
/*
* CNTR 2 - In this case we want to execute the next instruction
* and NOT take the jump. So we set the counter to 0. The jump
* address doesn't count.
*/
writel(0x0, &sdr_rw_load_mgr_regs->load_cntr2);
writel(0x0, &sdr_rw_load_jump_mgr_regs->load_jump_add2);
/*
* CNTR 3 - Set the nop counter to the number of cycles we
* need to loop for, minus 1.
*/
writel(rw_wl_nop_cycles - 1, &sdr_rw_load_mgr_regs->load_cntr3);
if (test_dm) {
mcc_instruction = seq->rwcfg->lfsr_wr_rd_dm_bank_0;
writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_nop,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
} else {
mcc_instruction = seq->rwcfg->lfsr_wr_rd_bank_0;
writel(seq->rwcfg->lfsr_wr_rd_bank_0_nop,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
}
}
writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RESET_READ_DATAPATH_OFFSET);
if (quick_write_mode)
writel(0x08, &sdr_rw_load_mgr_regs->load_cntr0);
else
writel(0x40, &sdr_rw_load_mgr_regs->load_cntr0);
writel(mcc_instruction, &sdr_rw_load_jump_mgr_regs->load_jump_add0);
/*
* CNTR 1 - This is used to ensure enough time elapses
* for read data to come back.
*/
writel(0x30, &sdr_rw_load_mgr_regs->load_cntr1);
if (test_dm) {
writel(seq->rwcfg->lfsr_wr_rd_dm_bank_0_wait,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
} else {
writel(seq->rwcfg->lfsr_wr_rd_bank_0_wait,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
}
writel(mcc_instruction, (SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET) +
(group << 2));
}
/**
* rw_mgr_mem_calibrate_write_test() - Test writes, check for single/multiple
* pass
* @rank_bgn: Rank number
* @write_group: Write Group
* @use_dm: Use DM
* @all_correct: All bits must be correct in the mask
* @bit_chk: Resulting bit mask after the test
* @all_ranks: Test all ranks
*
* Test writes, can check for a single bit pass or multiple bit pass.
*/
static int
rw_mgr_mem_calibrate_write_test(struct socfpga_sdrseq *seq,
const u32 rank_bgn, const u32 write_group,
const u32 use_dm, const u32 all_correct,
u32 *bit_chk, const u32 all_ranks)
{
const u32 rank_end = all_ranks ?
seq->rwcfg->mem_number_of_ranks :
(rank_bgn + NUM_RANKS_PER_SHADOW_REG);
const u32 shift_ratio = seq->rwcfg->mem_dq_per_write_dqs /
seq->rwcfg->mem_virtual_groups_per_write_dqs;
const u32 correct_mask_vg = seq->param.write_correct_mask_vg;
u32 tmp_bit_chk, base_rw_mgr, group;
int vg, r;
*bit_chk = seq->param.write_correct_mask;
for (r = rank_bgn; r < rank_end; r++) {
/* Set rank */
set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
tmp_bit_chk = 0;
for (vg = seq->rwcfg->mem_virtual_groups_per_write_dqs - 1;
vg >= 0; vg--) {
/* Reset the FIFOs to get pointers to known state. */
writel(0, &phy_mgr_cmd->fifo_reset);
group = write_group *
seq->rwcfg->mem_virtual_groups_per_write_dqs
+ vg;
rw_mgr_mem_calibrate_write_test_issue(seq, group,
use_dm);
base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
tmp_bit_chk <<= shift_ratio;
tmp_bit_chk |= (correct_mask_vg & ~(base_rw_mgr));
}
*bit_chk &= tmp_bit_chk;
}
set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
if (all_correct) {
debug_cond(DLEVEL >= 2,
"write_test(%u,%u,ALL) : %u == %u => %i\n",
write_group, use_dm, *bit_chk,
seq->param.write_correct_mask,
*bit_chk == seq->param.write_correct_mask);
return *bit_chk == seq->param.write_correct_mask;
} else {
debug_cond(DLEVEL >= 2,
"write_test(%u,%u,ONE) : %u != %i => %i\n",
write_group, use_dm, *bit_chk, 0, *bit_chk != 0);
return *bit_chk != 0x00;
}
}
/**
* rw_mgr_mem_calibrate_read_test_patterns() - Read back test patterns
* @rank_bgn: Rank number
* @group: Read/Write Group
* @all_ranks: Test all ranks
*
* Performs a guaranteed read on the patterns we are going to use during a
* read test to ensure memory works.
*/
static int
rw_mgr_mem_calibrate_read_test_patterns(struct socfpga_sdrseq *seq,
const u32 rank_bgn, const u32 group,
const u32 all_ranks)
{
const u32 addr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET;
const u32 addr_offset =
(group * seq->rwcfg->mem_virtual_groups_per_read_dqs)
<< 2;
const u32 rank_end = all_ranks ?
seq->rwcfg->mem_number_of_ranks :
(rank_bgn + NUM_RANKS_PER_SHADOW_REG);
const u32 shift_ratio = seq->rwcfg->mem_dq_per_read_dqs /
seq->rwcfg->mem_virtual_groups_per_read_dqs;
const u32 correct_mask_vg = seq->param.read_correct_mask_vg;
u32 tmp_bit_chk, base_rw_mgr, bit_chk;
int vg, r;
int ret = 0;
bit_chk = seq->param.read_correct_mask;
for (r = rank_bgn; r < rank_end; r++) {
/* Set rank */
set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
/* Load up a constant bursts of read commands */
writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
writel(seq->rwcfg->guaranteed_read,
&sdr_rw_load_jump_mgr_regs->load_jump_add0);
writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
writel(seq->rwcfg->guaranteed_read_cont,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
tmp_bit_chk = 0;
for (vg = seq->rwcfg->mem_virtual_groups_per_read_dqs - 1;
vg >= 0; vg--) {
/* Reset the FIFOs to get pointers to known state. */
writel(0, &phy_mgr_cmd->fifo_reset);
writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RESET_READ_DATAPATH_OFFSET);
writel(seq->rwcfg->guaranteed_read,
addr + addr_offset + (vg << 2));
base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
tmp_bit_chk <<= shift_ratio;
tmp_bit_chk |= correct_mask_vg & ~base_rw_mgr;
}
bit_chk &= tmp_bit_chk;
}
writel(seq->rwcfg->clear_dqs_enable, addr + (group << 2));
set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
if (bit_chk != seq->param.read_correct_mask)
ret = -EIO;
debug_cond(DLEVEL >= 1,
"%s:%d test_load_patterns(%u,ALL) => (%u == %u) => %i\n",
__func__, __LINE__, group, bit_chk,
seq->param.read_correct_mask, ret);
return ret;
}
/**
* rw_mgr_mem_calibrate_read_load_patterns() - Load up the patterns for read
* test
* @rank_bgn: Rank number
* @all_ranks: Test all ranks
*
* Load up the patterns we are going to use during a read test.
*/
static void rw_mgr_mem_calibrate_read_load_patterns(struct socfpga_sdrseq *seq,
const u32 rank_bgn,
const int all_ranks)
{
const u32 rank_end = all_ranks ?
seq->rwcfg->mem_number_of_ranks :
(rank_bgn + NUM_RANKS_PER_SHADOW_REG);
u32 r;
debug("%s:%d\n", __func__, __LINE__);
for (r = rank_bgn; r < rank_end; r++) {
/* set rank */
set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
/* Load up a constant bursts */
writel(0x20, &sdr_rw_load_mgr_regs->load_cntr0);
writel(seq->rwcfg->guaranteed_write_wait0,
&sdr_rw_load_jump_mgr_regs->load_jump_add0);
writel(0x20, &sdr_rw_load_mgr_regs->load_cntr1);
writel(seq->rwcfg->guaranteed_write_wait1,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
writel(0x04, &sdr_rw_load_mgr_regs->load_cntr2);
writel(seq->rwcfg->guaranteed_write_wait2,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
writel(0x04, &sdr_rw_load_mgr_regs->load_cntr3);
writel(seq->rwcfg->guaranteed_write_wait3,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
writel(seq->rwcfg->guaranteed_write,
SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET);
}
set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
}
/**
* rw_mgr_mem_calibrate_read_test() - Perform READ test on single rank
* @rank_bgn: Rank number
* @group: Read/Write group
* @num_tries: Number of retries of the test
* @all_correct: All bits must be correct in the mask
* @bit_chk: Resulting bit mask after the test
* @all_groups: Test all R/W groups
* @all_ranks: Test all ranks
*
* Try a read and see if it returns correct data back. Test has dummy reads
* inserted into the mix used to align DQS enable. Test has more thorough
* checks than the regular read test.
*/
static int
rw_mgr_mem_calibrate_read_test(struct socfpga_sdrseq *seq,
const u32 rank_bgn, const u32 group,
const u32 num_tries, const u32 all_correct,
u32 *bit_chk,
const u32 all_groups, const u32 all_ranks)
{
const u32 rank_end = all_ranks ? seq->rwcfg->mem_number_of_ranks :
(rank_bgn + NUM_RANKS_PER_SHADOW_REG);
const u32 quick_read_mode =
((STATIC_CALIB_STEPS & CALIB_SKIP_DELAY_SWEEPS) &&
seq->misccfg->enable_super_quick_calibration);
u32 correct_mask_vg = seq->param.read_correct_mask_vg;
u32 tmp_bit_chk;
u32 base_rw_mgr;
u32 addr;
int r, vg, ret;
*bit_chk = seq->param.read_correct_mask;
for (r = rank_bgn; r < rank_end; r++) {
/* set rank */
set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_READ_WRITE);
writel(0x10, &sdr_rw_load_mgr_regs->load_cntr1);
writel(seq->rwcfg->read_b2b_wait1,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
writel(0x10, &sdr_rw_load_mgr_regs->load_cntr2);
writel(seq->rwcfg->read_b2b_wait2,
&sdr_rw_load_jump_mgr_regs->load_jump_add2);
if (quick_read_mode)
writel(0x1, &sdr_rw_load_mgr_regs->load_cntr0);
/* need at least two (1+1) reads to capture failures */
else if (all_groups)
writel(0x06, &sdr_rw_load_mgr_regs->load_cntr0);
else
writel(0x32, &sdr_rw_load_mgr_regs->load_cntr0);
writel(seq->rwcfg->read_b2b,
&sdr_rw_load_jump_mgr_regs->load_jump_add0);
if (all_groups)
writel(seq->rwcfg->mem_if_read_dqs_width *
seq->rwcfg->mem_virtual_groups_per_read_dqs - 1,
&sdr_rw_load_mgr_regs->load_cntr3);
else
writel(0x0, &sdr_rw_load_mgr_regs->load_cntr3);
writel(seq->rwcfg->read_b2b,
&sdr_rw_load_jump_mgr_regs->load_jump_add3);
tmp_bit_chk = 0;
for (vg = seq->rwcfg->mem_virtual_groups_per_read_dqs - 1;
vg >= 0; vg--) {
/* Reset the FIFOs to get pointers to known state. */
writel(0, &phy_mgr_cmd->fifo_reset);
writel(0, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RESET_READ_DATAPATH_OFFSET);
if (all_groups) {
addr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_ALL_GROUPS_OFFSET;
} else {
addr = SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET;
}
writel(seq->rwcfg->read_b2b, addr +
((group *
seq->rwcfg->mem_virtual_groups_per_read_dqs +
vg) << 2));
base_rw_mgr = readl(SDR_PHYGRP_RWMGRGRP_ADDRESS);
tmp_bit_chk <<=
seq->rwcfg->mem_dq_per_read_dqs /
seq->rwcfg->mem_virtual_groups_per_read_dqs;
tmp_bit_chk |= correct_mask_vg & ~(base_rw_mgr);
}
*bit_chk &= tmp_bit_chk;
}
addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_RUN_SINGLE_GROUP_OFFSET;
writel(seq->rwcfg->clear_dqs_enable, addr + (group << 2));
set_rank_and_odt_mask(seq, 0, RW_MGR_ODT_MODE_OFF);
if (all_correct) {
ret = (*bit_chk == seq->param.read_correct_mask);
debug_cond(DLEVEL >= 2,
"%s:%d read_test(%u,ALL,%u) => (%u == %u) => %i\n",
__func__, __LINE__, group, all_groups, *bit_chk,
seq->param.read_correct_mask, ret);
} else {
ret = (*bit_chk != 0x00);
debug_cond(DLEVEL >= 2,
"%s:%d read_test(%u,ONE,%u) => (%u != %u) => %i\n",
__func__, __LINE__, group, all_groups, *bit_chk,
0, ret);
}
return ret;
}
/**
* rw_mgr_mem_calibrate_read_test_all_ranks() - Perform READ test on all ranks
* @grp: Read/Write group
* @num_tries: Number of retries of the test
* @all_correct: All bits must be correct in the mask
* @all_groups: Test all R/W groups
*
* Perform a READ test across all memory ranks.
*/
static int
rw_mgr_mem_calibrate_read_test_all_ranks(struct socfpga_sdrseq *seq,
const u32 grp, const u32 num_tries,
const u32 all_correct,
const u32 all_groups)
{
u32 bit_chk;
return rw_mgr_mem_calibrate_read_test(seq, 0, grp, num_tries,
all_correct, &bit_chk, all_groups,
1);
}
/**
* rw_mgr_incr_vfifo() - Increase VFIFO value
* @grp: Read/Write group
*
* Increase VFIFO value.
*/
static void rw_mgr_incr_vfifo(const u32 grp)
{
writel(grp, &phy_mgr_cmd->inc_vfifo_hard_phy);
}
/**
* rw_mgr_decr_vfifo() - Decrease VFIFO value
* @grp: Read/Write group
*
* Decrease VFIFO value.
*/
static void rw_mgr_decr_vfifo(struct socfpga_sdrseq *seq, const u32 grp)
{
u32 i;
for (i = 0; i < seq->misccfg->read_valid_fifo_size - 1; i++)
rw_mgr_incr_vfifo(grp);
}
/**
* find_vfifo_failing_read() - Push VFIFO to get a failing read
* @grp: Read/Write group
*
* Push VFIFO until a failing read happens.
*/
static int find_vfifo_failing_read(struct socfpga_sdrseq *seq,
const u32 grp)
{
u32 v, ret, fail_cnt = 0;
for (v = 0; v < seq->misccfg->read_valid_fifo_size; v++) {
debug_cond(DLEVEL >= 2, "%s:%d: vfifo %u\n",
__func__, __LINE__, v);
ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
PASS_ONE_BIT, 0);
if (!ret) {
fail_cnt++;
if (fail_cnt == 2)
return v;
}
/* Fiddle with FIFO. */
rw_mgr_incr_vfifo(grp);
}
/* No failing read found! Something must have gone wrong. */
debug_cond(DLEVEL >= 2, "%s:%d: vfifo failed\n", __func__, __LINE__);
return 0;
}
/**
* sdr_find_phase_delay() - Find DQS enable phase or delay
* @working: If 1, look for working phase/delay, if 0, look for non-working
* @delay: If 1, look for delay, if 0, look for phase
* @grp: Read/Write group
* @work: Working window position
* @work_inc: Working window increment
* @pd: DQS Phase/Delay Iterator
*
* Find working or non-working DQS enable phase setting.
*/
static int sdr_find_phase_delay(struct socfpga_sdrseq *seq, int working,
int delay, const u32 grp, u32 *work,
const u32 work_inc, u32 *pd)
{
const u32 max = delay ? seq->iocfg->dqs_en_delay_max :
seq->iocfg->dqs_en_phase_max;
u32 ret;
for (; *pd <= max; (*pd)++) {
if (delay)
scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, *pd);
else
scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, *pd);
ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
PASS_ONE_BIT, 0);
if (!working)
ret = !ret;
if (ret)
return 0;
if (work)
*work += work_inc;
}
return -EINVAL;
}
/**
* sdr_find_phase() - Find DQS enable phase
* @working: If 1, look for working phase, if 0, look for non-working phase
* @grp: Read/Write group
* @work: Working window position
* @i: Iterator
* @p: DQS Phase Iterator
*
* Find working or non-working DQS enable phase setting.
*/
static int sdr_find_phase(struct socfpga_sdrseq *seq, int working,
const u32 grp, u32 *work, u32 *i, u32 *p)
{
const u32 end = seq->misccfg->read_valid_fifo_size + (working ? 0 : 1);
int ret;
for (; *i < end; (*i)++) {
if (working)
*p = 0;
ret = sdr_find_phase_delay(seq, working, 0, grp, work,
seq->iocfg->delay_per_opa_tap, p);
if (!ret)
return 0;
if (*p > seq->iocfg->dqs_en_phase_max) {
/* Fiddle with FIFO. */
rw_mgr_incr_vfifo(grp);
if (!working)
*p = 0;
}
}
return -EINVAL;
}
/**
* sdr_working_phase() - Find working DQS enable phase
* @grp: Read/Write group
* @work_bgn: Working window start position
* @d: dtaps output value
* @p: DQS Phase Iterator
* @i: Iterator
*
* Find working DQS enable phase setting.
*/
static int sdr_working_phase(struct socfpga_sdrseq *seq, const u32 grp,
u32 *work_bgn, u32 *d, u32 *p, u32 *i)
{
const u32 dtaps_per_ptap = seq->iocfg->delay_per_opa_tap /
seq->iocfg->delay_per_dqs_en_dchain_tap;
int ret;
*work_bgn = 0;
for (*d = 0; *d <= dtaps_per_ptap; (*d)++) {
*i = 0;
scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, *d);
ret = sdr_find_phase(seq, 1, grp, work_bgn, i, p);
if (!ret)
return 0;
*work_bgn += seq->iocfg->delay_per_dqs_en_dchain_tap;
}
/* Cannot find working solution */
debug_cond(DLEVEL >= 2, "%s:%d find_dqs_en_phase: no vfifo/ptap/dtap\n",
__func__, __LINE__);
return -EINVAL;
}
/**
* sdr_backup_phase() - Find DQS enable backup phase
* @grp: Read/Write group
* @work_bgn: Working window start position
* @p: DQS Phase Iterator
*
* Find DQS enable backup phase setting.
*/
static void sdr_backup_phase(struct socfpga_sdrseq *seq, const u32 grp,
u32 *work_bgn, u32 *p)
{
u32 tmp_delay, d;
int ret;
/* Special case code for backing up a phase */
if (*p == 0) {
*p = seq->iocfg->dqs_en_phase_max;
rw_mgr_decr_vfifo(seq, grp);
} else {
(*p)--;
}
tmp_delay = *work_bgn - seq->iocfg->delay_per_opa_tap;
scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, *p);
for (d = 0; d <= seq->iocfg->dqs_en_delay_max && tmp_delay < *work_bgn;
d++) {
scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, d);
ret = rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
PASS_ONE_BIT, 0);
if (ret) {
*work_bgn = tmp_delay;
break;
}
tmp_delay += seq->iocfg->delay_per_dqs_en_dchain_tap;
}
/* Restore VFIFO to old state before we decremented it (if needed). */
(*p)++;
if (*p > seq->iocfg->dqs_en_phase_max) {
*p = 0;
rw_mgr_incr_vfifo(grp);
}
scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, 0);
}
/**
* sdr_nonworking_phase() - Find non-working DQS enable phase
* @grp: Read/Write group
* @work_end: Working window end position
* @p: DQS Phase Iterator
* @i: Iterator
*
* Find non-working DQS enable phase setting.
*/
static int sdr_nonworking_phase(struct socfpga_sdrseq *seq,
const u32 grp, u32 *work_end, u32 *p, u32 *i)
{
int ret;
(*p)++;
*work_end += seq->iocfg->delay_per_opa_tap;
if (*p > seq->iocfg->dqs_en_phase_max) {
/* Fiddle with FIFO. */
*p = 0;
rw_mgr_incr_vfifo(grp);
}
ret = sdr_find_phase(seq, 0, grp, work_end, i, p);
if (ret) {
/* Cannot see edge of failing read. */
debug_cond(DLEVEL >= 2, "%s:%d: end: failed\n",
__func__, __LINE__);
}
return ret;
}
/**
* sdr_find_window_center() - Find center of the working DQS window.
* @grp: Read/Write group
* @work_bgn: First working settings
* @work_end: Last working settings
*
* Find center of the working DQS enable window.
*/
static int sdr_find_window_center(struct socfpga_sdrseq *seq,
const u32 grp, const u32 work_bgn,
const u32 work_end)
{
u32 work_mid;
int tmp_delay = 0;
int i, p, d;
work_mid = (work_bgn + work_end) / 2;
debug_cond(DLEVEL >= 2, "work_bgn=%d work_end=%d work_mid=%d\n",
work_bgn, work_end, work_mid);
/* Get the middle delay to be less than a VFIFO delay */
tmp_delay = (seq->iocfg->dqs_en_phase_max + 1)
* seq->iocfg->delay_per_opa_tap;
debug_cond(DLEVEL >= 2, "vfifo ptap delay %d\n", tmp_delay);
work_mid %= tmp_delay;
debug_cond(DLEVEL >= 2, "new work_mid %d\n", work_mid);
tmp_delay = rounddown(work_mid, seq->iocfg->delay_per_opa_tap);
if (tmp_delay > seq->iocfg->dqs_en_phase_max
* seq->iocfg->delay_per_opa_tap) {
tmp_delay = seq->iocfg->dqs_en_phase_max
* seq->iocfg->delay_per_opa_tap;
}
p = tmp_delay / seq->iocfg->delay_per_opa_tap;
debug_cond(DLEVEL >= 2, "new p %d, tmp_delay=%d\n", p, tmp_delay);
d = DIV_ROUND_UP(work_mid - tmp_delay,
seq->iocfg->delay_per_dqs_en_dchain_tap);
if (d > seq->iocfg->dqs_en_delay_max)
d = seq->iocfg->dqs_en_delay_max;
tmp_delay += d * seq->iocfg->delay_per_dqs_en_dchain_tap;
debug_cond(DLEVEL >= 2, "new d %d, tmp_delay=%d\n", d, tmp_delay);
scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p);
scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, d);
/*
* push vfifo until we can successfully calibrate. We can do this
* because the largest possible margin in 1 VFIFO cycle.
*/
for (i = 0; i < seq->misccfg->read_valid_fifo_size; i++) {
debug_cond(DLEVEL >= 2, "find_dqs_en_phase: center\n");
if (rw_mgr_mem_calibrate_read_test_all_ranks(seq, grp, 1,
PASS_ONE_BIT,
0)) {
debug_cond(DLEVEL >= 2,
"%s:%d center: found: ptap=%u dtap=%u\n",
__func__, __LINE__, p, d);
return 0;
}
/* Fiddle with FIFO. */
rw_mgr_incr_vfifo(grp);
}
debug_cond(DLEVEL >= 2, "%s:%d center: failed.\n",
__func__, __LINE__);
return -EINVAL;
}
/**
* rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase() - Find a good DQS enable to
* use
* @grp: Read/Write Group
*
* Find a good DQS enable to use.
*/
static int
rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(struct socfpga_sdrseq *seq,
const u32 grp)
{
u32 d, p, i;
u32 dtaps_per_ptap;
u32 work_bgn, work_end;
u32 found_passing_read, found_failing_read = 0, initial_failing_dtap;
int ret;
debug("%s:%d %u\n", __func__, __LINE__, grp);
reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
scc_mgr_set_dqs_en_delay_all_ranks(seq, grp, 0);
scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, 0);
/* Step 0: Determine number of delay taps for each phase tap. */
dtaps_per_ptap = seq->iocfg->delay_per_opa_tap /
seq->iocfg->delay_per_dqs_en_dchain_tap;
/* Step 1: First push vfifo until we get a failing read. */
find_vfifo_failing_read(seq, grp);
/* Step 2: Find first working phase, increment in ptaps. */
work_bgn = 0;
ret = sdr_working_phase(seq, grp, &work_bgn, &d, &p, &i);
if (ret)
return ret;
work_end = work_bgn;
/*
* If d is 0 then the working window covers a phase tap and we can
* follow the old procedure. Otherwise, we've found the beginning
* and we need to increment the dtaps until we find the end.
*/
if (d == 0) {
/*
* Step 3a: If we have room, back off by one and
* increment in dtaps.
*/
sdr_backup_phase(seq, grp, &work_bgn, &p);
/*
* Step 4a: go forward from working phase to non working
* phase, increment in ptaps.
*/
ret = sdr_nonworking_phase(seq, grp, &work_end, &p, &i);
if (ret)
return ret;
/* Step 5a: Back off one from last, increment in dtaps. */
/* Special case code for backing up a phase */
if (p == 0) {
p = seq->iocfg->dqs_en_phase_max;
rw_mgr_decr_vfifo(seq, grp);
} else {
p = p - 1;
}
work_end -= seq->iocfg->delay_per_opa_tap;
scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p);
d = 0;
debug_cond(DLEVEL >= 2, "%s:%d p: ptap=%u\n",
__func__, __LINE__, p);
}
/* The dtap increment to find the failing edge is done here. */
sdr_find_phase_delay(seq, 0, 1, grp, &work_end,
seq->iocfg->delay_per_dqs_en_dchain_tap, &d);
/* Go back to working dtap */
if (d != 0)
work_end -= seq->iocfg->delay_per_dqs_en_dchain_tap;
debug_cond(DLEVEL >= 2,
"%s:%d p/d: ptap=%u dtap=%u end=%u\n",
__func__, __LINE__, p, d - 1, work_end);
if (work_end < work_bgn) {
/* nil range */
debug_cond(DLEVEL >= 2, "%s:%d end-2: failed\n",
__func__, __LINE__);
return -EINVAL;
}
debug_cond(DLEVEL >= 2, "%s:%d found range [%u,%u]\n",
__func__, __LINE__, work_bgn, work_end);
/*
* We need to calculate the number of dtaps that equal a ptap.
* To do that we'll back up a ptap and re-find the edge of the
* window using dtaps
*/
debug_cond(DLEVEL >= 2, "%s:%d calculate dtaps_per_ptap for tracking\n",
__func__, __LINE__);
/* Special case code for backing up a phase */
if (p == 0) {
p = seq->iocfg->dqs_en_phase_max;
rw_mgr_decr_vfifo(seq, grp);
debug_cond(DLEVEL >= 2, "%s:%d backedup cycle/phase: p=%u\n",
__func__, __LINE__, p);
} else {
p = p - 1;
debug_cond(DLEVEL >= 2, "%s:%d backedup phase only: p=%u",
__func__, __LINE__, p);
}
scc_mgr_set_dqs_en_phase_all_ranks(seq, grp, p);
/*
* Increase dtap until we first see a passing read (in case the
* window is smaller than a ptap), and then a failing read to
* mark the edge of the window again.
*/
/* Find a passing read. */
debug_cond(DLEVEL >= 2, "%s:%d find passing read\n",
__func__, __LINE__);
initial_failing_dtap = d;
found_passing_read = !sdr_find_phase_delay(seq, 1, 1, grp, NULL, 0, &d);
if (found_passing_read) {
/* Find a failing read. */
debug_cond(DLEVEL >= 2, "%s:%d find failing read\n",
__func__, __LINE__);
d++;
found_failing_read = !sdr_find_phase_delay(seq, 0, 1, grp, NULL,
0, &d);
} else {
debug_cond(DLEVEL >= 1,
"%s:%d failed to calculate dtaps per ptap. Fall back on static value\n",
__func__, __LINE__);
}
/*
* The dynamically calculated dtaps_per_ptap is only valid if we
* found a passing/failing read. If we didn't, it means d hit the max
* (seq->iocfg->dqs_en_delay_max). Otherwise, dtaps_per_ptap retains its
* statically calculated value.
*/
if (found_passing_read && found_failing_read)
dtaps_per_ptap = d - initial_failing_dtap;
writel(dtaps_per_ptap, &sdr_reg_file->dtaps_per_ptap);
debug_cond(DLEVEL >= 2, "%s:%d dtaps_per_ptap=%u - %u = %u",
__func__, __LINE__, d, initial_failing_dtap, dtaps_per_ptap);
/* Step 6: Find the centre of the window. */
ret = sdr_find_window_center(seq, grp, work_bgn, work_end);
return ret;
}
/**
* search_stop_check() - Check if the detected edge is valid
* @write: Perform read (Stage 2) or write (Stage 3) calibration
* @d: DQS delay
* @rank_bgn: Rank number
* @write_group: Write Group
* @read_group: Read Group
* @bit_chk: Resulting bit mask after the test
* @sticky_bit_chk: Resulting sticky bit mask after the test
* @use_read_test: Perform read test
*
* Test if the found edge is valid.
*/
static u32 search_stop_check(struct socfpga_sdrseq *seq, const int write,
const int d, const int rank_bgn,
const u32 write_group, const u32 read_group,
u32 *bit_chk, u32 *sticky_bit_chk,
const u32 use_read_test)
{
const u32 ratio = seq->rwcfg->mem_if_read_dqs_width /
seq->rwcfg->mem_if_write_dqs_width;
const u32 correct_mask = write ? seq->param.write_correct_mask :
seq->param.read_correct_mask;
const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
seq->rwcfg->mem_dq_per_read_dqs;
u32 ret;
/*
* Stop searching when the read test doesn't pass AND when
* we've seen a passing read on every bit.
*/
if (write) { /* WRITE-ONLY */
ret = !rw_mgr_mem_calibrate_write_test(seq, rank_bgn,
write_group, 0,
PASS_ONE_BIT, bit_chk,
0);
} else if (use_read_test) { /* READ-ONLY */
ret = !rw_mgr_mem_calibrate_read_test(seq, rank_bgn, read_group,
NUM_READ_PB_TESTS,
PASS_ONE_BIT, bit_chk,
0, 0);
} else { /* READ-ONLY */
rw_mgr_mem_calibrate_write_test(seq, rank_bgn, write_group, 0,
PASS_ONE_BIT, bit_chk, 0);
*bit_chk = *bit_chk >> (per_dqs *
(read_group - (write_group * ratio)));
ret = (*bit_chk == 0);
}
*sticky_bit_chk = *sticky_bit_chk | *bit_chk;
ret = ret && (*sticky_bit_chk == correct_mask);
debug_cond(DLEVEL >= 2,
"%s:%d center(left): dtap=%u => %u == %u && %u",
__func__, __LINE__, d,
*sticky_bit_chk, correct_mask, ret);
return ret;
}
/**
* search_left_edge() - Find left edge of DQ/DQS working phase
* @write: Perform read (Stage 2) or write (Stage 3) calibration
* @rank_bgn: Rank number
* @write_group: Write Group
* @read_group: Read Group
* @test_bgn: Rank number to begin the test
* @sticky_bit_chk: Resulting sticky bit mask after the test
* @left_edge: Left edge of the DQ/DQS phase
* @right_edge: Right edge of the DQ/DQS phase
* @use_read_test: Perform read test
*
* Find left edge of DQ/DQS working phase.
*/
static void search_left_edge(struct socfpga_sdrseq *seq, const int write,
const int rank_bgn, const u32 write_group,
const u32 read_group, const u32 test_bgn,
u32 *sticky_bit_chk, int *left_edge,
int *right_edge, const u32 use_read_test)
{
const u32 delay_max = write ? seq->iocfg->io_out1_delay_max :
seq->iocfg->io_in_delay_max;
const u32 dqs_max = write ? seq->iocfg->io_out1_delay_max :
seq->iocfg->dqs_in_delay_max;
const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
seq->rwcfg->mem_dq_per_read_dqs;
u32 stop, bit_chk;
int i, d;
for (d = 0; d <= dqs_max; d++) {
if (write)
scc_mgr_apply_group_dq_out1_delay(seq, d);
else
scc_mgr_apply_group_dq_in_delay(seq, test_bgn, d);
writel(0, &sdr_scc_mgr->update);
stop = search_stop_check(seq, write, d, rank_bgn, write_group,
read_group, &bit_chk, sticky_bit_chk,
use_read_test);
if (stop == 1)
break;
/* stop != 1 */
for (i = 0; i < per_dqs; i++) {
if (bit_chk & 1) {
/*
* Remember a passing test as
* the left_edge.
*/
left_edge[i] = d;
} else {
/*
* If a left edge has not been seen
* yet, then a future passing test
* will mark this edge as the right
* edge.
*/
if (left_edge[i] == delay_max + 1)
right_edge[i] = -(d + 1);
}
bit_chk >>= 1;
}
}
/* Reset DQ delay chains to 0 */
if (write)
scc_mgr_apply_group_dq_out1_delay(seq, 0);
else
scc_mgr_apply_group_dq_in_delay(seq, test_bgn, 0);
*sticky_bit_chk = 0;
for (i = per_dqs - 1; i >= 0; i--) {
debug_cond(DLEVEL >= 2,
"%s:%d vfifo_center: left_edge[%u]: %d right_edge[%u]: %d\n",
__func__, __LINE__, i, left_edge[i],
i, right_edge[i]);
/*
* Check for cases where we haven't found the left edge,
* which makes our assignment of the the right edge invalid.
* Reset it to the illegal value.
*/
if ((left_edge[i] == delay_max + 1) &&
(right_edge[i] != delay_max + 1)) {
right_edge[i] = delay_max + 1;
debug_cond(DLEVEL >= 2,
"%s:%d vfifo_center: reset right_edge[%u]: %d\n",
__func__, __LINE__, i, right_edge[i]);
}
/*
* Reset sticky bit
* READ: except for bits where we have seen both
* the left and right edge.
* WRITE: except for bits where we have seen the
* left edge.
*/
*sticky_bit_chk <<= 1;
if (write) {
if (left_edge[i] != delay_max + 1)
*sticky_bit_chk |= 1;
} else {
if ((left_edge[i] != delay_max + 1) &&
(right_edge[i] != delay_max + 1))
*sticky_bit_chk |= 1;
}
}
}
/**
* search_right_edge() - Find right edge of DQ/DQS working phase
* @write: Perform read (Stage 2) or write (Stage 3) calibration
* @rank_bgn: Rank number
* @write_group: Write Group
* @read_group: Read Group
* @start_dqs: DQS start phase
* @start_dqs_en: DQS enable start phase
* @sticky_bit_chk: Resulting sticky bit mask after the test
* @left_edge: Left edge of the DQ/DQS phase
* @right_edge: Right edge of the DQ/DQS phase
* @use_read_test: Perform read test
*
* Find right edge of DQ/DQS working phase.
*/
static int search_right_edge(struct socfpga_sdrseq *seq, const int write,
const int rank_bgn, const u32 write_group,
const u32 read_group, const int start_dqs,
const int start_dqs_en, u32 *sticky_bit_chk,
int *left_edge, int *right_edge,
const u32 use_read_test)
{
const u32 delay_max = write ? seq->iocfg->io_out1_delay_max :
seq->iocfg->io_in_delay_max;
const u32 dqs_max = write ? seq->iocfg->io_out1_delay_max :
seq->iocfg->dqs_in_delay_max;
const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
seq->rwcfg->mem_dq_per_read_dqs;
u32 stop, bit_chk;
int i, d;
for (d = 0; d <= dqs_max - start_dqs; d++) {
if (write) { /* WRITE-ONLY */
scc_mgr_apply_group_dqs_io_and_oct_out1(seq,
write_group,
d + start_dqs);
} else { /* READ-ONLY */
scc_mgr_set_dqs_bus_in_delay(read_group, d + start_dqs);
if (seq->iocfg->shift_dqs_en_when_shift_dqs) {
u32 delay = d + start_dqs_en;
if (delay > seq->iocfg->dqs_en_delay_max)
delay = seq->iocfg->dqs_en_delay_max;
scc_mgr_set_dqs_en_delay(read_group, delay);
}
scc_mgr_load_dqs(read_group);
}
writel(0, &sdr_scc_mgr->update);
stop = search_stop_check(seq, write, d, rank_bgn, write_group,
read_group, &bit_chk, sticky_bit_chk,
use_read_test);
if (stop == 1) {
if (write && (d == 0)) { /* WRITE-ONLY */
for (i = 0;
i < seq->rwcfg->mem_dq_per_write_dqs;
i++) {
/*
* d = 0 failed, but it passed when
* testing the left edge, so it must be
* marginal, set it to -1
*/
if (right_edge[i] == delay_max + 1 &&
left_edge[i] != delay_max + 1)
right_edge[i] = -1;
}
}
break;
}
/* stop != 1 */
for (i = 0; i < per_dqs; i++) {
if (bit_chk & 1) {
/*
* Remember a passing test as
* the right_edge.
*/
right_edge[i] = d;
} else {
if (d != 0) {
/*
* If a right edge has not
* been seen yet, then a future
* passing test will mark this
* edge as the left edge.
*/
if (right_edge[i] == delay_max + 1)
left_edge[i] = -(d + 1);
} else {
/*
* d = 0 failed, but it passed
* when testing the left edge,
* so it must be marginal, set
* it to -1
*/
if (right_edge[i] == delay_max + 1 &&
left_edge[i] != delay_max + 1)
right_edge[i] = -1;
/*
* If a right edge has not been
* seen yet, then a future
* passing test will mark this
* edge as the left edge.
*/
else if (right_edge[i] == delay_max + 1)
left_edge[i] = -(d + 1);
}
}
debug_cond(DLEVEL >= 2, "%s:%d center[r,d=%u]: ",
__func__, __LINE__, d);
debug_cond(DLEVEL >= 2,
"bit_chk_test=%i left_edge[%u]: %d ",
bit_chk & 1, i, left_edge[i]);
debug_cond(DLEVEL >= 2, "right_edge[%u]: %d\n", i,
right_edge[i]);
bit_chk >>= 1;
}
}
/* Check that all bits have a window */
for (i = 0; i < per_dqs; i++) {
debug_cond(DLEVEL >= 2,
"%s:%d write_center: left_edge[%u]: %d right_edge[%u]: %d",
__func__, __LINE__, i, left_edge[i],
i, right_edge[i]);
if ((left_edge[i] == dqs_max + 1) ||
(right_edge[i] == dqs_max + 1))
return i + 1; /* FIXME: If we fail, retval > 0 */
}
return 0;
}
/**
* get_window_mid_index() - Find the best middle setting of DQ/DQS phase
* @write: Perform read (Stage 2) or write (Stage 3) calibration
* @left_edge: Left edge of the DQ/DQS phase
* @right_edge: Right edge of the DQ/DQS phase
* @mid_min: Best DQ/DQS phase middle setting
*
* Find index and value of the middle of the DQ/DQS working phase.
*/
static int get_window_mid_index(struct socfpga_sdrseq *seq,
const int write, int *left_edge,
int *right_edge, int *mid_min)
{
const u32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
seq->rwcfg->mem_dq_per_read_dqs;
int i, mid, min_index;
/* Find middle of window for each DQ bit */
*mid_min = left_edge[0] - right_edge[0];
min_index = 0;
for (i = 1; i < per_dqs; i++) {
mid = left_edge[i] - right_edge[i];
if (mid < *mid_min) {
*mid_min = mid;
min_index = i;
}
}
/*
* -mid_min/2 represents the amount that we need to move DQS.
* If mid_min is odd and positive we'll need to add one to make
* sure the rounding in further calculations is correct (always
* bias to the right), so just add 1 for all positive values.
*/
if (*mid_min > 0)
(*mid_min)++;
*mid_min = *mid_min / 2;
debug_cond(DLEVEL >= 1, "%s:%d vfifo_center: *mid_min=%d (index=%u)\n",
__func__, __LINE__, *mid_min, min_index);
return min_index;
}
/**
* center_dq_windows() - Center the DQ/DQS windows
* @write: Perform read (Stage 2) or write (Stage 3) calibration
* @left_edge: Left edge of the DQ/DQS phase
* @right_edge: Right edge of the DQ/DQS phase
* @mid_min: Adjusted DQ/DQS phase middle setting
* @orig_mid_min: Original DQ/DQS phase middle setting
* @min_index: DQ/DQS phase middle setting index
* @test_bgn: Rank number to begin the test
* @dq_margin: Amount of shift for the DQ
* @dqs_margin: Amount of shift for the DQS
*
* Align the DQ/DQS windows in each group.
*/
static void center_dq_windows(struct socfpga_sdrseq *seq,
const int write, int *left_edge, int *right_edge,
const int mid_min, const int orig_mid_min,
const int min_index, const int test_bgn,
int *dq_margin, int *dqs_margin)
{
const s32 delay_max = write ? seq->iocfg->io_out1_delay_max :
seq->iocfg->io_in_delay_max;
const s32 per_dqs = write ? seq->rwcfg->mem_dq_per_write_dqs :
seq->rwcfg->mem_dq_per_read_dqs;
const s32 delay_off = write ? SCC_MGR_IO_OUT1_DELAY_OFFSET :
SCC_MGR_IO_IN_DELAY_OFFSET;
const s32 addr = SDR_PHYGRP_SCCGRP_ADDRESS | delay_off;
s32 temp_dq_io_delay1;
int shift_dq, i, p;
/* Initialize data for export structures */
*dqs_margin = delay_max + 1;
*dq_margin = delay_max + 1;
/* add delay to bring centre of all DQ windows to the same "level" */
for (i = 0, p = test_bgn; i < per_dqs; i++, p++) {
/* Use values before divide by 2 to reduce round off error */
shift_dq = (left_edge[i] - right_edge[i] -
(left_edge[min_index] - right_edge[min_index]))/2 +
(orig_mid_min - mid_min);
debug_cond(DLEVEL >= 2,
"vfifo_center: before: shift_dq[%u]=%d\n",
i, shift_dq);
temp_dq_io_delay1 = readl(addr + (i << 2));
if (shift_dq + temp_dq_io_delay1 > delay_max)
shift_dq = delay_max - temp_dq_io_delay1;
else if (shift_dq + temp_dq_io_delay1 < 0)
shift_dq = -temp_dq_io_delay1;
debug_cond(DLEVEL >= 2,
"vfifo_center: after: shift_dq[%u]=%d\n",
i, shift_dq);
if (write)
scc_mgr_set_dq_out1_delay(i,
temp_dq_io_delay1 + shift_dq);
else
scc_mgr_set_dq_in_delay(p,
temp_dq_io_delay1 + shift_dq);
scc_mgr_load_dq(p);
debug_cond(DLEVEL >= 2,
"vfifo_center: margin[%u]=[%d,%d]\n", i,
left_edge[i] - shift_dq + (-mid_min),
right_edge[i] + shift_dq - (-mid_min));
/* To determine values for export structures */
if (left_edge[i] - shift_dq + (-mid_min) < *dq_margin)
*dq_margin = left_edge[i] - shift_dq + (-mid_min);
if (right_edge[i] + shift_dq - (-mid_min) < *dqs_margin)
*dqs_margin = right_edge[i] + shift_dq - (-mid_min);
}
}
/**
* rw_mgr_mem_calibrate_vfifo_center() - Per-bit deskew DQ and centering
* @rank_bgn: Rank number
* @rw_group: Read/Write Group
* @test_bgn: Rank at which the test begins
* @use_read_test: Perform a read test
* @update_fom: Update FOM
*
* Per-bit deskew DQ and centering.
*/
static int rw_mgr_mem_calibrate_vfifo_center(struct socfpga_sdrseq *seq,
const u32 rank_bgn,
const u32 rw_group,
const u32 test_bgn,
const int use_read_test,
const int update_fom)
{
const u32 addr =
SDR_PHYGRP_SCCGRP_ADDRESS + SCC_MGR_DQS_IN_DELAY_OFFSET +
(rw_group << 2);
/*
* Store these as signed since there are comparisons with
* signed numbers.
*/
u32 sticky_bit_chk;
s32 left_edge[seq->rwcfg->mem_dq_per_read_dqs];
s32 right_edge[seq->rwcfg->mem_dq_per_read_dqs];
s32 orig_mid_min, mid_min;
s32 new_dqs, start_dqs, start_dqs_en = 0, final_dqs_en;
s32 dq_margin, dqs_margin;
int i, min_index;
int ret;
debug("%s:%d: %u %u", __func__, __LINE__, rw_group, test_bgn);
start_dqs = readl(addr);
if (seq->iocfg->shift_dqs_en_when_shift_dqs)
start_dqs_en = readl(addr - seq->iocfg->dqs_en_delay_offset);
/* set the left and right edge of each bit to an illegal value */
/* use (seq->iocfg->io_in_delay_max + 1) as an illegal value */
sticky_bit_chk = 0;
for (i = 0; i < seq->rwcfg->mem_dq_per_read_dqs; i++) {
left_edge[i] = seq->iocfg->io_in_delay_max + 1;
right_edge[i] = seq->iocfg->io_in_delay_max + 1;
}
/* Search for the left edge of the window for each bit */
search_left_edge(seq, 0, rank_bgn, rw_group, rw_group, test_bgn,
&sticky_bit_chk,
left_edge, right_edge, use_read_test);
/* Search for the right edge of the window for each bit */
ret = search_right_edge(seq, 0, rank_bgn, rw_group, rw_group,
start_dqs, start_dqs_en,
&sticky_bit_chk,
left_edge, right_edge, use_read_test);
if (ret) {
/*
* Restore delay chain settings before letting the loop
* in rw_mgr_mem_calibrate_vfifo to retry different
* dqs/ck relationships.
*/
scc_mgr_set_dqs_bus_in_delay(rw_group, start_dqs);
if (seq->iocfg->shift_dqs_en_when_shift_dqs)
scc_mgr_set_dqs_en_delay(rw_group, start_dqs_en);
scc_mgr_load_dqs(rw_group);
writel(0, &sdr_scc_mgr->update);
debug_cond(DLEVEL >= 1,
"%s:%d vfifo_center: failed to find edge [%u]: %d %d",
__func__, __LINE__, i, left_edge[i], right_edge[i]);
if (use_read_test) {
set_failing_group_stage(seq, rw_group *
seq->rwcfg->mem_dq_per_read_dqs + i,
CAL_STAGE_VFIFO,
CAL_SUBSTAGE_VFIFO_CENTER);
} else {
set_failing_group_stage(seq, rw_group *
seq->rwcfg->mem_dq_per_read_dqs + i,
CAL_STAGE_VFIFO_AFTER_WRITES,
CAL_SUBSTAGE_VFIFO_CENTER);
}
return -EIO;
}
min_index = get_window_mid_index(seq, 0, left_edge, right_edge,
&mid_min);
/* Determine the amount we can change DQS (which is -mid_min) */
orig_mid_min = mid_min;
new_dqs = start_dqs - mid_min;
if (new_dqs > seq->iocfg->dqs_in_delay_max)
new_dqs = seq->iocfg->dqs_in_delay_max;
else if (new_dqs < 0)
new_dqs = 0;
mid_min = start_dqs - new_dqs;
debug_cond(DLEVEL >= 1, "vfifo_center: new mid_min=%d new_dqs=%d\n",
mid_min, new_dqs);
if (seq->iocfg->shift_dqs_en_when_shift_dqs) {
if (start_dqs_en - mid_min > seq->iocfg->dqs_en_delay_max)
mid_min += start_dqs_en - mid_min -
seq->iocfg->dqs_en_delay_max;
else if (start_dqs_en - mid_min < 0)
mid_min += start_dqs_en - mid_min;
}
new_dqs = start_dqs - mid_min;
debug_cond(DLEVEL >= 1,
"vfifo_center: start_dqs=%d start_dqs_en=%d new_dqs=%d mid_min=%d\n",
start_dqs,
seq->iocfg->shift_dqs_en_when_shift_dqs ? start_dqs_en : -1,
new_dqs, mid_min);
/* Add delay to bring centre of all DQ windows to the same "level". */
center_dq_windows(seq, 0, left_edge, right_edge, mid_min, orig_mid_min,
min_index, test_bgn, &dq_margin, &dqs_margin);
/* Move DQS-en */
if (seq->iocfg->shift_dqs_en_when_shift_dqs) {
final_dqs_en = start_dqs_en - mid_min;
scc_mgr_set_dqs_en_delay(rw_group, final_dqs_en);
scc_mgr_load_dqs(rw_group);
}
/* Move DQS */
scc_mgr_set_dqs_bus_in_delay(rw_group, new_dqs);
scc_mgr_load_dqs(rw_group);
debug_cond(DLEVEL >= 2,
"%s:%d vfifo_center: dq_margin=%d dqs_margin=%d",
__func__, __LINE__, dq_margin, dqs_margin);
/*
* Do not remove this line as it makes sure all of our decisions
* have been applied. Apply the update bit.
*/
writel(0, &sdr_scc_mgr->update);
if ((dq_margin < 0) || (dqs_margin < 0))
return -EINVAL;
return 0;
}
/**
* rw_mgr_mem_calibrate_guaranteed_write() - Perform guaranteed write into the
* device
* @rw_group: Read/Write Group
* @phase: DQ/DQS phase
*
* Because initially no communication ca be reliably performed with the memory
* device, the sequencer uses a guaranteed write mechanism to write data into
* the memory device.
*/
static int rw_mgr_mem_calibrate_guaranteed_write(struct socfpga_sdrseq *seq,
const u32 rw_group,
const u32 phase)
{
int ret;
/* Set a particular DQ/DQS phase. */
scc_mgr_set_dqdqs_output_phase_all_ranks(seq, rw_group, phase);
debug_cond(DLEVEL >= 1, "%s:%d guaranteed write: g=%u p=%u\n",
__func__, __LINE__, rw_group, phase);
/*
* Altera EMI_RM 2015.05.04 :: Figure 1-25
* Load up the patterns used by read calibration using the
* current DQDQS phase.
*/
rw_mgr_mem_calibrate_read_load_patterns(seq, 0, 1);
if (seq->gbl.phy_debug_mode_flags & PHY_DEBUG_DISABLE_GUARANTEED_READ)
return 0;
/*
* Altera EMI_RM 2015.05.04 :: Figure 1-26
* Back-to-Back reads of the patterns used for calibration.
*/
ret = rw_mgr_mem_calibrate_read_test_patterns(seq, 0, rw_group, 1);
if (ret)
debug_cond(DLEVEL >= 1,
"%s:%d Guaranteed read test failed: g=%u p=%u\n",
__func__, __LINE__, rw_group, phase);
return ret;
}
/**
* rw_mgr_mem_calibrate_dqs_enable_calibration() - DQS Enable Calibration
* @rw_group: Read/Write Group
* @test_bgn: Rank at which the test begins
*
* DQS enable calibration ensures reliable capture of the DQ signal without
* glitches on the DQS line.
*/
static int
rw_mgr_mem_calibrate_dqs_enable_calibration(struct socfpga_sdrseq *seq,
const u32 rw_group,
const u32 test_bgn)
{
/*
* Altera EMI_RM 2015.05.04 :: Figure 1-27
* DQS and DQS Eanble Signal Relationships.
*/
/* We start at zero, so have one less dq to devide among */
const u32 delay_step = seq->iocfg->io_in_delay_max /
(seq->rwcfg->mem_dq_per_read_dqs - 1);
int ret;
u32 i, p, d, r;
debug("%s:%d (%u,%u)\n", __func__, __LINE__, rw_group, test_bgn);
/* Try different dq_in_delays since the DQ path is shorter than DQS. */
for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
r += NUM_RANKS_PER_SHADOW_REG) {
for (i = 0, p = test_bgn, d = 0;
i < seq->rwcfg->mem_dq_per_read_dqs;
i++, p++, d += delay_step) {
debug_cond(DLEVEL >= 1,
"%s:%d: g=%u r=%u i=%u p=%u d=%u\n",
__func__, __LINE__, rw_group, r, i, p, d);
scc_mgr_set_dq_in_delay(p, d);
scc_mgr_load_dq(p);
}
writel(0, &sdr_scc_mgr->update);
}
/*
* Try rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase across different
* dq_in_delay values
*/
ret = rw_mgr_mem_calibrate_vfifo_find_dqs_en_phase(seq, rw_group);
debug_cond(DLEVEL >= 1,
"%s:%d: g=%u found=%u; Reseting delay chain to zero\n",
__func__, __LINE__, rw_group, !ret);
for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
r += NUM_RANKS_PER_SHADOW_REG) {
scc_mgr_apply_group_dq_in_delay(seq, test_bgn, 0);
writel(0, &sdr_scc_mgr->update);
}
return ret;
}
/**
* rw_mgr_mem_calibrate_dq_dqs_centering() - Centering DQ/DQS
* @rw_group: Read/Write Group
* @test_bgn: Rank at which the test begins
* @use_read_test: Perform a read test
* @update_fom: Update FOM
*
* The centerin DQ/DQS stage attempts to align DQ and DQS signals on reads
* within a group.
*/
static int
rw_mgr_mem_calibrate_dq_dqs_centering(struct socfpga_sdrseq *seq,
const u32 rw_group, const u32 test_bgn,
const int use_read_test,
const int update_fom)
{
int ret, grp_calibrated;
u32 rank_bgn, sr;
/*
* Altera EMI_RM 2015.05.04 :: Figure 1-28
* Read per-bit deskew can be done on a per shadow register basis.
*/
grp_calibrated = 1;
for (rank_bgn = 0, sr = 0;
rank_bgn < seq->rwcfg->mem_number_of_ranks;
rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
ret = rw_mgr_mem_calibrate_vfifo_center(seq, rank_bgn, rw_group,
test_bgn,
use_read_test,
update_fom);
if (!ret)
continue;
grp_calibrated = 0;
}
if (!grp_calibrated)
return -EIO;
return 0;
}
/**
* rw_mgr_mem_calibrate_vfifo() - Calibrate the read valid prediction FIFO
* @rw_group: Read/Write Group
* @test_bgn: Rank at which the test begins
*
* Stage 1: Calibrate the read valid prediction FIFO.
*
* This function implements UniPHY calibration Stage 1, as explained in
* detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
*
* - read valid prediction will consist of finding:
* - DQS enable phase and DQS enable delay (DQS Enable Calibration)
* - DQS input phase and DQS input delay (DQ/DQS Centering)
* - we also do a per-bit deskew on the DQ lines.
*/
static int rw_mgr_mem_calibrate_vfifo(struct socfpga_sdrseq *seq,
const u32 rw_group, const u32 test_bgn)
{
u32 p, d;
u32 dtaps_per_ptap;
u32 failed_substage;
int ret;
debug("%s:%d: %u %u\n", __func__, __LINE__, rw_group, test_bgn);
/* Update info for sims */
reg_file_set_group(rw_group);
reg_file_set_stage(CAL_STAGE_VFIFO);
reg_file_set_sub_stage(CAL_SUBSTAGE_GUARANTEED_READ);
failed_substage = CAL_SUBSTAGE_GUARANTEED_READ;
/* USER Determine number of delay taps for each phase tap. */
dtaps_per_ptap = DIV_ROUND_UP(seq->iocfg->delay_per_opa_tap,
seq->iocfg->delay_per_dqs_en_dchain_tap)
- 1;
for (d = 0; d <= dtaps_per_ptap; d += 2) {
/*
* In RLDRAMX we may be messing the delay of pins in
* the same write rw_group but outside of the current read
* the rw_group, but that's ok because we haven't calibrated
* output side yet.
*/
if (d > 0) {
scc_mgr_apply_group_all_out_delay_add_all_ranks(seq,
rw_group,
d);
}
for (p = 0; p <= seq->iocfg->dqdqs_out_phase_max; p++) {
/* 1) Guaranteed Write */
ret = rw_mgr_mem_calibrate_guaranteed_write(seq,
rw_group,
p);
if (ret)
break;
/* 2) DQS Enable Calibration */
ret = rw_mgr_mem_calibrate_dqs_enable_calibration(seq,
rw_group,
test_bgn);
if (ret) {
failed_substage = CAL_SUBSTAGE_DQS_EN_PHASE;
continue;
}
/* 3) Centering DQ/DQS */
/*
* If doing read after write calibration, do not update
* FOM now. Do it then.
*/
ret = rw_mgr_mem_calibrate_dq_dqs_centering(seq,
rw_group,
test_bgn,
1, 0);
if (ret) {
failed_substage = CAL_SUBSTAGE_VFIFO_CENTER;
continue;
}
/* All done. */
goto cal_done_ok;
}
}
/* Calibration Stage 1 failed. */
set_failing_group_stage(seq, rw_group, CAL_STAGE_VFIFO,
failed_substage);
return 0;
/* Calibration Stage 1 completed OK. */
cal_done_ok:
/*
* Reset the delay chains back to zero if they have moved > 1
* (check for > 1 because loop will increase d even when pass in
* first case).
*/
if (d > 2)
scc_mgr_zero_group(seq, rw_group, 1);
return 1;
}
/**
* rw_mgr_mem_calibrate_vfifo_end() - DQ/DQS Centering.
* @rw_group: Read/Write Group
* @test_bgn: Rank at which the test begins
*
* Stage 3: DQ/DQS Centering.
*
* This function implements UniPHY calibration Stage 3, as explained in
* detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
*/
static int rw_mgr_mem_calibrate_vfifo_end(struct socfpga_sdrseq *seq,
const u32 rw_group,
const u32 test_bgn)
{
int ret;
debug("%s:%d %u %u", __func__, __LINE__, rw_group, test_bgn);
/* Update info for sims. */
reg_file_set_group(rw_group);
reg_file_set_stage(CAL_STAGE_VFIFO_AFTER_WRITES);
reg_file_set_sub_stage(CAL_SUBSTAGE_VFIFO_CENTER);
ret = rw_mgr_mem_calibrate_dq_dqs_centering(seq, rw_group, test_bgn, 0,
1);
if (ret)
set_failing_group_stage(seq, rw_group,
CAL_STAGE_VFIFO_AFTER_WRITES,
CAL_SUBSTAGE_VFIFO_CENTER);
return ret;
}
/**
* rw_mgr_mem_calibrate_lfifo() - Minimize latency
*
* Stage 4: Minimize latency.
*
* This function implements UniPHY calibration Stage 4, as explained in
* detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
* Calibrate LFIFO to find smallest read latency.
*/
static u32 rw_mgr_mem_calibrate_lfifo(struct socfpga_sdrseq *seq)
{
int found_one = 0;
debug("%s:%d\n", __func__, __LINE__);
/* Update info for sims. */
reg_file_set_stage(CAL_STAGE_LFIFO);
reg_file_set_sub_stage(CAL_SUBSTAGE_READ_LATENCY);
/* Load up the patterns used by read calibration for all ranks */
rw_mgr_mem_calibrate_read_load_patterns(seq, 0, 1);
do {
writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
debug_cond(DLEVEL >= 2, "%s:%d lfifo: read_lat=%u",
__func__, __LINE__, seq->gbl.curr_read_lat);
if (!rw_mgr_mem_calibrate_read_test_all_ranks(seq, 0,
NUM_READ_TESTS,
PASS_ALL_BITS, 1))
break;
found_one = 1;
/*
* Reduce read latency and see if things are
* working correctly.
*/
seq->gbl.curr_read_lat--;
} while (seq->gbl.curr_read_lat > 0);
/* Reset the fifos to get pointers to known state. */
writel(0, &phy_mgr_cmd->fifo_reset);
if (found_one) {
/* Add a fudge factor to the read latency that was determined */
seq->gbl.curr_read_lat += 2;
writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
debug_cond(DLEVEL >= 2,
"%s:%d lfifo: success: using read_lat=%u\n",
__func__, __LINE__, seq->gbl.curr_read_lat);
} else {
set_failing_group_stage(seq, 0xff, CAL_STAGE_LFIFO,
CAL_SUBSTAGE_READ_LATENCY);
debug_cond(DLEVEL >= 2,
"%s:%d lfifo: failed at initial read_lat=%u\n",
__func__, __LINE__, seq->gbl.curr_read_lat);
}
return found_one;
}
/**
* search_window() - Search for the/part of the window with DM/DQS shift
* @search_dm: If 1, search for the DM shift, if 0, search for DQS
* shift
* @rank_bgn: Rank number
* @write_group: Write Group
* @bgn_curr: Current window begin
* @end_curr: Current window end
* @bgn_best: Current best window begin
* @end_best: Current best window end
* @win_best: Size of the best window
* @new_dqs: New DQS value (only applicable if search_dm = 0).
*
* Search for the/part of the window with DM/DQS shift.
*/
static void search_window(struct socfpga_sdrseq *seq,
const int search_dm, const u32 rank_bgn,
const u32 write_group, int *bgn_curr, int *end_curr,
int *bgn_best, int *end_best, int *win_best,
int new_dqs)
{
u32 bit_chk;
const int max = seq->iocfg->io_out1_delay_max - new_dqs;
int d, di;
/* Search for the/part of the window with DM/DQS shift. */
for (di = max; di >= 0; di -= DELTA_D) {
if (search_dm) {
d = di;
scc_mgr_apply_group_dm_out1_delay(seq, d);
} else {
/* For DQS, we go from 0...max */
d = max - di;
/*
* Note: This only shifts DQS, so are we limiting
* ourselves to width of DQ unnecessarily.
*/
scc_mgr_apply_group_dqs_io_and_oct_out1(seq,
write_group,
d + new_dqs);
}
writel(0, &sdr_scc_mgr->update);
if (rw_mgr_mem_calibrate_write_test(seq, rank_bgn, write_group,
1, PASS_ALL_BITS, &bit_chk,
0)) {
/* Set current end of the window. */
*end_curr = search_dm ? -d : d;
/*
* If a starting edge of our window has not been seen
* this is our current start of the DM window.
*/
if (*bgn_curr == seq->iocfg->io_out1_delay_max + 1)
*bgn_curr = search_dm ? -d : d;
/*
* If current window is bigger than best seen.
* Set best seen to be current window.
*/
if ((*end_curr - *bgn_curr + 1) > *win_best) {
*win_best = *end_curr - *bgn_curr + 1;
*bgn_best = *bgn_curr;
*end_best = *end_curr;
}
} else {
/* We just saw a failing test. Reset temp edge. */
*bgn_curr = seq->iocfg->io_out1_delay_max + 1;
*end_curr = seq->iocfg->io_out1_delay_max + 1;
/* Early exit is only applicable to DQS. */
if (search_dm)
continue;
/*
* Early exit optimization: if the remaining delay
* chain space is less than already seen largest
* window we can exit.
*/
if (*win_best - 1 > seq->iocfg->io_out1_delay_max
- new_dqs - d)
break;
}
}
}
/*
* rw_mgr_mem_calibrate_writes_center() - Center all windows
* @rank_bgn: Rank number
* @write_group: Write group
* @test_bgn: Rank at which the test begins
*
* Center all windows. Do per-bit-deskew to possibly increase size of
* certain windows.
*/
static int
rw_mgr_mem_calibrate_writes_center(struct socfpga_sdrseq *seq,
const u32 rank_bgn, const u32 write_group,
const u32 test_bgn)
{
int i;
u32 sticky_bit_chk;
u32 min_index;
int left_edge[seq->rwcfg->mem_dq_per_write_dqs];
int right_edge[seq->rwcfg->mem_dq_per_write_dqs];
int mid;
int mid_min, orig_mid_min;
int new_dqs, start_dqs;
int dq_margin, dqs_margin, dm_margin;
int bgn_curr = seq->iocfg->io_out1_delay_max + 1;
int end_curr = seq->iocfg->io_out1_delay_max + 1;
int bgn_best = seq->iocfg->io_out1_delay_max + 1;
int end_best = seq->iocfg->io_out1_delay_max + 1;
int win_best = 0;
int ret;
debug("%s:%d %u %u", __func__, __LINE__, write_group, test_bgn);
dm_margin = 0;
start_dqs = readl((SDR_PHYGRP_SCCGRP_ADDRESS |
SCC_MGR_IO_OUT1_DELAY_OFFSET) +
(seq->rwcfg->mem_dq_per_write_dqs << 2));
/* Per-bit deskew. */
/*
* Set the left and right edge of each bit to an illegal value.
* Use (seq->iocfg->io_out1_delay_max + 1) as an illegal value.
*/
sticky_bit_chk = 0;
for (i = 0; i < seq->rwcfg->mem_dq_per_write_dqs; i++) {
left_edge[i] = seq->iocfg->io_out1_delay_max + 1;
right_edge[i] = seq->iocfg->io_out1_delay_max + 1;
}
/* Search for the left edge of the window for each bit. */
search_left_edge(seq, 1, rank_bgn, write_group, 0, test_bgn,
&sticky_bit_chk,
left_edge, right_edge, 0);
/* Search for the right edge of the window for each bit. */
ret = search_right_edge(seq, 1, rank_bgn, write_group, 0,
start_dqs, 0,
&sticky_bit_chk,
left_edge, right_edge, 0);
if (ret) {
set_failing_group_stage(seq, test_bgn + ret - 1,
CAL_STAGE_WRITES,
CAL_SUBSTAGE_WRITES_CENTER);
return -EINVAL;
}
min_index = get_window_mid_index(seq, 1, left_edge, right_edge,
&mid_min);
/* Determine the amount we can change DQS (which is -mid_min). */
orig_mid_min = mid_min;
new_dqs = start_dqs;
mid_min = 0;
debug_cond(DLEVEL >= 1,
"%s:%d write_center: start_dqs=%d new_dqs=%d mid_min=%d\n",
__func__, __LINE__, start_dqs, new_dqs, mid_min);
/* Add delay to bring centre of all DQ windows to the same "level". */
center_dq_windows(seq, 1, left_edge, right_edge, mid_min, orig_mid_min,
min_index, 0, &dq_margin, &dqs_margin);
/* Move DQS */
scc_mgr_apply_group_dqs_io_and_oct_out1(seq, write_group, new_dqs);
writel(0, &sdr_scc_mgr->update);
/* Centre DM */
debug_cond(DLEVEL >= 2, "%s:%d write_center: DM\n", __func__, __LINE__);
/*
* Set the left and right edge of each bit to an illegal value.
* Use (seq->iocfg->io_out1_delay_max + 1) as an illegal value.
*/
left_edge[0] = seq->iocfg->io_out1_delay_max + 1;
right_edge[0] = seq->iocfg->io_out1_delay_max + 1;
/* Search for the/part of the window with DM shift. */
search_window(seq, 1, rank_bgn, write_group, &bgn_curr, &end_curr,
&bgn_best, &end_best, &win_best, 0);
/* Reset DM delay chains to 0. */
scc_mgr_apply_group_dm_out1_delay(seq, 0);
/*
* Check to see if the current window nudges up aganist 0 delay.
* If so we need to continue the search by shifting DQS otherwise DQS
* search begins as a new search.
*/
if (end_curr != 0) {
bgn_curr = seq->iocfg->io_out1_delay_max + 1;
end_curr = seq->iocfg->io_out1_delay_max + 1;
}
/* Search for the/part of the window with DQS shifts. */
search_window(seq, 0, rank_bgn, write_group, &bgn_curr, &end_curr,
&bgn_best, &end_best, &win_best, new_dqs);
/* Assign left and right edge for cal and reporting. */
left_edge[0] = -1 * bgn_best;
right_edge[0] = end_best;
debug_cond(DLEVEL >= 2, "%s:%d dm_calib: left=%d right=%d\n",
__func__, __LINE__, left_edge[0], right_edge[0]);
/* Move DQS (back to orig). */
scc_mgr_apply_group_dqs_io_and_oct_out1(seq, write_group, new_dqs);
/* Move DM */
/* Find middle of window for the DM bit. */
mid = (left_edge[0] - right_edge[0]) / 2;
/* Only move right, since we are not moving DQS/DQ. */
if (mid < 0)
mid = 0;
/* dm_marign should fail if we never find a window. */
if (win_best == 0)
dm_margin = -1;
else
dm_margin = left_edge[0] - mid;
scc_mgr_apply_group_dm_out1_delay(seq, mid);
writel(0, &sdr_scc_mgr->update);
debug_cond(DLEVEL >= 2,
"%s:%d dm_calib: left=%d right=%d mid=%d dm_margin=%d\n",
__func__, __LINE__, left_edge[0], right_edge[0],
mid, dm_margin);
/* Export values. */
seq->gbl.fom_out += dq_margin + dqs_margin;
debug_cond(DLEVEL >= 2,
"%s:%d write_center: dq_margin=%d dqs_margin=%d dm_margin=%d\n",
__func__, __LINE__, dq_margin, dqs_margin, dm_margin);
/*
* Do not remove this line as it makes sure all of our
* decisions have been applied.
*/
writel(0, &sdr_scc_mgr->update);
if ((dq_margin < 0) || (dqs_margin < 0) || (dm_margin < 0))
return -EINVAL;
return 0;
}
/**
* rw_mgr_mem_calibrate_writes() - Write Calibration Part One
* @rank_bgn: Rank number
* @group: Read/Write Group
* @test_bgn: Rank at which the test begins
*
* Stage 2: Write Calibration Part One.
*
* This function implements UniPHY calibration Stage 2, as explained in
* detail in Altera EMI_RM 2015.05.04 , "UniPHY Calibration Stages".
*/
static int rw_mgr_mem_calibrate_writes(struct socfpga_sdrseq *seq,
const u32 rank_bgn, const u32 group,
const u32 test_bgn)
{
int ret;
/* Update info for sims */
debug("%s:%d %u %u\n", __func__, __LINE__, group, test_bgn);
reg_file_set_group(group);
reg_file_set_stage(CAL_STAGE_WRITES);
reg_file_set_sub_stage(CAL_SUBSTAGE_WRITES_CENTER);
ret = rw_mgr_mem_calibrate_writes_center(seq, rank_bgn, group,
test_bgn);
if (ret)
set_failing_group_stage(seq, group, CAL_STAGE_WRITES,
CAL_SUBSTAGE_WRITES_CENTER);
return ret;
}
/**
* mem_precharge_and_activate() - Precharge all banks and activate
*
* Precharge all banks and activate row 0 in bank "000..." and bank "111...".
*/
static void mem_precharge_and_activate(struct socfpga_sdrseq *seq)
{
int r;
for (r = 0; r < seq->rwcfg->mem_number_of_ranks; r++) {
/* Set rank. */
set_rank_and_odt_mask(seq, r, RW_MGR_ODT_MODE_OFF);
/* Precharge all banks. */
writel(seq->rwcfg->precharge_all, SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET);
writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr0);
writel(seq->rwcfg->activate_0_and_1_wait1,
&sdr_rw_load_jump_mgr_regs->load_jump_add0);
writel(0x0F, &sdr_rw_load_mgr_regs->load_cntr1);
writel(seq->rwcfg->activate_0_and_1_wait2,
&sdr_rw_load_jump_mgr_regs->load_jump_add1);
/* Activate rows. */
writel(seq->rwcfg->activate_0_and_1,
SDR_PHYGRP_RWMGRGRP_ADDRESS |
RW_MGR_RUN_SINGLE_GROUP_OFFSET);
}
}
/**
* mem_init_latency() - Configure memory RLAT and WLAT settings
*
* Configure memory RLAT and WLAT parameters.
*/
static void mem_init_latency(struct socfpga_sdrseq *seq)
{
/*
* For AV/CV, LFIFO is hardened and always runs at full rate
* so max latency in AFI clocks, used here, is correspondingly
* smaller.
*/
const u32 max_latency = (1 << seq->misccfg->max_latency_count_width)
- 1;
u32 rlat, wlat;
debug("%s:%d\n", __func__, __LINE__);
/*
* Read in write latency.
* WL for Hard PHY does not include additive latency.
*/
wlat = readl(&data_mgr->t_wl_add);
wlat += readl(&data_mgr->mem_t_add);
seq->gbl.rw_wl_nop_cycles = wlat - 1;
/* Read in readl latency. */
rlat = readl(&data_mgr->t_rl_add);
/* Set a pretty high read latency initially. */
seq->gbl.curr_read_lat = rlat + 16;
if (seq->gbl.curr_read_lat > max_latency)
seq->gbl.curr_read_lat = max_latency;
writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
/* Advertise write latency. */
writel(wlat, &phy_mgr_cfg->afi_wlat);
}
/**
* @mem_skip_calibrate() - Set VFIFO and LFIFO to instant-on settings
*
* Set VFIFO and LFIFO to instant-on settings in skip calibration mode.
*/
static void mem_skip_calibrate(struct socfpga_sdrseq *seq)
{
u32 vfifo_offset;
u32 i, j, r;
debug("%s:%d\n", __func__, __LINE__);
/* Need to update every shadow register set used by the interface */
for (r = 0; r < seq->rwcfg->mem_number_of_ranks;
r += NUM_RANKS_PER_SHADOW_REG) {
/*
* Set output phase alignment settings appropriate for
* skip calibration.
*/
for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
scc_mgr_set_dqs_en_phase(i, 0);
if (seq->iocfg->dll_chain_length == 6)
scc_mgr_set_dqdqs_output_phase(i, 6);
else
scc_mgr_set_dqdqs_output_phase(i, 7);
/*
* Case:33398
*
* Write data arrives to the I/O two cycles before write
* latency is reached (720 deg).
* -> due to bit-slip in a/c bus
* -> to allow board skew where dqs is longer than ck
* -> how often can this happen!?
* -> can claim back some ptaps for high freq
* support if we can relax this, but i digress...
*
* The write_clk leads mem_ck by 90 deg
* The minimum ptap of the OPA is 180 deg
* Each ptap has (360 / IO_DLL_CHAIN_LENGH) deg of delay
* The write_clk is always delayed by 2 ptaps
*
* Hence, to make DQS aligned to CK, we need to delay
* DQS by:
* (720 - 90 - 180 - 2) *
* (360 / seq->iocfg->dll_chain_length)
*
* Dividing the above by
(360 / seq->iocfg->dll_chain_length)
* gives us the number of ptaps, which simplies to:
*
* (1.25 * seq->iocfg->dll_chain_length - 2)
*/
scc_mgr_set_dqdqs_output_phase(i,
((125 * seq->iocfg->dll_chain_length)
/ 100) - 2);
}
writel(0xff, &sdr_scc_mgr->dqs_ena);
writel(0xff, &sdr_scc_mgr->dqs_io_ena);
for (i = 0; i < seq->rwcfg->mem_if_write_dqs_width; i++) {
writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
SCC_MGR_GROUP_COUNTER_OFFSET);
}
writel(0xff, &sdr_scc_mgr->dq_ena);
writel(0xff, &sdr_scc_mgr->dm_ena);
writel(0, &sdr_scc_mgr->update);
}
/* Compensate for simulation model behaviour */
for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
scc_mgr_set_dqs_bus_in_delay(i, 10);
scc_mgr_load_dqs(i);
}
writel(0, &sdr_scc_mgr->update);
/*
* ArriaV has hard FIFOs that can only be initialized by incrementing
* in sequencer.
*/
vfifo_offset = seq->misccfg->calib_vfifo_offset;
for (j = 0; j < vfifo_offset; j++)
writel(0xff, &phy_mgr_cmd->inc_vfifo_hard_phy);
writel(0, &phy_mgr_cmd->fifo_reset);
/*
* For Arria V and Cyclone V with hard LFIFO, we get the skip-cal
* setting from generation-time constant.
*/
seq->gbl.curr_read_lat = seq->misccfg->calib_lfifo_offset;
writel(seq->gbl.curr_read_lat, &phy_mgr_cfg->phy_rlat);
}
/**
* mem_calibrate() - Memory calibration entry point.
*
* Perform memory calibration.
*/
static u32 mem_calibrate(struct socfpga_sdrseq *seq)
{
u32 i;
u32 rank_bgn, sr;
u32 write_group, write_test_bgn;
u32 read_group, read_test_bgn;
u32 run_groups, current_run;
u32 failing_groups = 0;
u32 group_failed = 0;
const u32 rwdqs_ratio = seq->rwcfg->mem_if_read_dqs_width /
seq->rwcfg->mem_if_write_dqs_width;
debug("%s:%d\n", __func__, __LINE__);
/* Initialize the data settings */
seq->gbl.error_substage = CAL_SUBSTAGE_NIL;
seq->gbl.error_stage = CAL_STAGE_NIL;
seq->gbl.error_group = 0xff;
seq->gbl.fom_in = 0;
seq->gbl.fom_out = 0;
/* Initialize WLAT and RLAT. */
mem_init_latency(seq);
/* Initialize bit slips. */
mem_precharge_and_activate(seq);
for (i = 0; i < seq->rwcfg->mem_if_read_dqs_width; i++) {
writel(i, SDR_PHYGRP_SCCGRP_ADDRESS |
SCC_MGR_GROUP_COUNTER_OFFSET);
/* Only needed once to set all groups, pins, DQ, DQS, DM. */
if (i == 0)
scc_mgr_set_hhp_extras();
scc_set_bypass_mode(i);
}
/* Calibration is skipped. */
if ((seq->dyn_calib_steps & CALIB_SKIP_ALL) == CALIB_SKIP_ALL) {
/*
* Set VFIFO and LFIFO to instant-on settings in skip
* calibration mode.
*/
mem_skip_calibrate(seq);
/*
* Do not remove this line as it makes sure all of our
* decisions have been applied.
*/
writel(0, &sdr_scc_mgr->update);
return 1;
}
/* Calibration is not skipped. */
for (i = 0; i < NUM_CALIB_REPEAT; i++) {
/*
* Zero all delay chain/phase settings for all
* groups and all shadow register sets.
*/
scc_mgr_zero_all(seq);
run_groups = ~0;
for (write_group = 0, write_test_bgn = 0; write_group
< seq->rwcfg->mem_if_write_dqs_width; write_group++,
write_test_bgn += seq->rwcfg->mem_dq_per_write_dqs) {
/* Initialize the group failure */
group_failed = 0;
current_run = run_groups & ((1 <<
RW_MGR_NUM_DQS_PER_WRITE_GROUP) - 1);
run_groups = run_groups >>
RW_MGR_NUM_DQS_PER_WRITE_GROUP;
if (current_run == 0)
continue;
writel(write_group, SDR_PHYGRP_SCCGRP_ADDRESS |
SCC_MGR_GROUP_COUNTER_OFFSET);
scc_mgr_zero_group(seq, write_group, 0);
for (read_group = write_group * rwdqs_ratio,
read_test_bgn = 0;
read_group < (write_group + 1) * rwdqs_ratio;
read_group++,
read_test_bgn += seq->rwcfg->mem_dq_per_read_dqs) {
if (STATIC_CALIB_STEPS & CALIB_SKIP_VFIFO)
continue;
/* Calibrate the VFIFO */
if (rw_mgr_mem_calibrate_vfifo(seq, read_group,
read_test_bgn))
continue;
if (!(seq->gbl.phy_debug_mode_flags &
PHY_DEBUG_SWEEP_ALL_GROUPS))
return 0;
/* The group failed, we're done. */
goto grp_failed;
}
/* Calibrate the output side */
for (rank_bgn = 0, sr = 0;
rank_bgn < seq->rwcfg->mem_number_of_ranks;
rank_bgn += NUM_RANKS_PER_SHADOW_REG, sr++) {
if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
continue;
/* Not needed in quick mode! */
if (STATIC_CALIB_STEPS &
CALIB_SKIP_DELAY_SWEEPS)
continue;
/* Calibrate WRITEs */
if (!rw_mgr_mem_calibrate_writes(seq, rank_bgn,
write_group,
write_test_bgn))
continue;
group_failed = 1;
if (!(seq->gbl.phy_debug_mode_flags &
PHY_DEBUG_SWEEP_ALL_GROUPS))
return 0;
}
/* Some group failed, we're done. */
if (group_failed)
goto grp_failed;
for (read_group = write_group * rwdqs_ratio,
read_test_bgn = 0;
read_group < (write_group + 1) * rwdqs_ratio;
read_group++,
read_test_bgn += seq->rwcfg->mem_dq_per_read_dqs) {
if (STATIC_CALIB_STEPS & CALIB_SKIP_WRITES)
continue;
if (!rw_mgr_mem_calibrate_vfifo_end(seq,
read_group,
read_test_bgn))
continue;
if (!(seq->gbl.phy_debug_mode_flags &
PHY_DEBUG_SWEEP_ALL_GROUPS))
return 0;
/* The group failed, we're done. */
goto grp_failed;
}
/* No group failed, continue as usual. */
continue;
grp_failed: /* A group failed, increment the counter. */
failing_groups++;
}
/*
* USER If there are any failing groups then report
* the failure.
*/
if (failing_groups != 0)
return 0;
if (STATIC_CALIB_STEPS & CALIB_SKIP_LFIFO)
continue;
/* Calibrate the LFIFO */
if (!rw_mgr_mem_calibrate_lfifo(seq))
return 0;
}
/*
* Do not remove this line as it makes sure all of our decisions
* have been applied.
*/
writel(0, &sdr_scc_mgr->update);
return 1;
}
/**
* run_mem_calibrate() - Perform memory calibration
*
* This function triggers the entire memory calibration procedure.
*/
static int run_mem_calibrate(struct socfpga_sdrseq *seq)
{
int pass;
u32 ctrl_cfg;
debug("%s:%d\n", __func__, __LINE__);
/* Reset pass/fail status shown on afi_cal_success/fail */
writel(PHY_MGR_CAL_RESET, &phy_mgr_cfg->cal_status);
/* Stop tracking manager. */
ctrl_cfg = readl(&sdr_ctrl->ctrl_cfg);
writel(ctrl_cfg & ~SDR_CTRLGRP_CTRLCFG_DQSTRKEN_MASK,
&sdr_ctrl->ctrl_cfg);
phy_mgr_initialize(seq);
rw_mgr_mem_initialize(seq);
/* Perform the actual memory calibration. */
pass = mem_calibrate(seq);
mem_precharge_and_activate(seq);
writel(0, &phy_mgr_cmd->fifo_reset);
/* Handoff. */
rw_mgr_mem_handoff(seq);
/*
* In Hard PHY this is a 2-bit control:
* 0: AFI Mux Select
* 1: DDIO Mux Select
*/
writel(0x2, &phy_mgr_cfg->mux_sel);
/* Start tracking manager. */
writel(ctrl_cfg, &sdr_ctrl->ctrl_cfg);
return pass;
}
/**
* debug_mem_calibrate() - Report result of memory calibration
* @pass: Value indicating whether calibration passed or failed
*
* This function reports the results of the memory calibration
* and writes debug information into the register file.
*/
static void debug_mem_calibrate(struct socfpga_sdrseq *seq, int pass)
{
u32 debug_info;
if (pass) {
debug("%s: CALIBRATION PASSED\n", __FILE__);
seq->gbl.fom_in /= 2;
seq->gbl.fom_out /= 2;
if (seq->gbl.fom_in > 0xff)
seq->gbl.fom_in = 0xff;
if (seq->gbl.fom_out > 0xff)
seq->gbl.fom_out = 0xff;
/* Update the FOM in the register file */
debug_info = seq->gbl.fom_in;
debug_info |= seq->gbl.fom_out << 8;
writel(debug_info, &sdr_reg_file->fom);
writel(debug_info, &phy_mgr_cfg->cal_debug_info);
writel(PHY_MGR_CAL_SUCCESS, &phy_mgr_cfg->cal_status);
} else {
debug("%s: CALIBRATION FAILED\n", __FILE__);
debug_info = seq->gbl.error_stage;
debug_info |= seq->gbl.error_substage << 8;
debug_info |= seq->gbl.error_group << 16;
writel(debug_info, &sdr_reg_file->failing_stage);
writel(debug_info, &phy_mgr_cfg->cal_debug_info);
writel(PHY_MGR_CAL_FAIL, &phy_mgr_cfg->cal_status);
/* Update the failing group/stage in the register file */
debug_info = seq->gbl.error_stage;
debug_info |= seq->gbl.error_substage << 8;
debug_info |= seq->gbl.error_group << 16;
writel(debug_info, &sdr_reg_file->failing_stage);
}
debug("%s: Calibration complete\n", __FILE__);
}
/**
* hc_initialize_rom_data() - Initialize ROM data
*
* Initialize ROM data.
*/
static void hc_initialize_rom_data(void)
{
unsigned int nelem = 0;
const u32 *rom_init;
u32 i, addr;
socfpga_get_seq_inst_init(&rom_init, &nelem);
addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_INST_ROM_WRITE_OFFSET;
for (i = 0; i < nelem; i++)
writel(rom_init[i], addr + (i << 2));
socfpga_get_seq_ac_init(&rom_init, &nelem);
addr = SDR_PHYGRP_RWMGRGRP_ADDRESS | RW_MGR_AC_ROM_WRITE_OFFSET;
for (i = 0; i < nelem; i++)
writel(rom_init[i], addr + (i << 2));
}
/**
* initialize_reg_file() - Initialize SDR register file
*
* Initialize SDR register file.
*/
static void initialize_reg_file(struct socfpga_sdrseq *seq)
{
/* Initialize the register file with the correct data */
writel(seq->misccfg->reg_file_init_seq_signature,
&sdr_reg_file->signature);
writel(0, &sdr_reg_file->debug_data_addr);
writel(0, &sdr_reg_file->cur_stage);
writel(0, &sdr_reg_file->fom);
writel(0, &sdr_reg_file->failing_stage);
writel(0, &sdr_reg_file->debug1);
writel(0, &sdr_reg_file->debug2);
}
/**
* initialize_hps_phy() - Initialize HPS PHY
*
* Initialize HPS PHY.
*/
static void initialize_hps_phy(void)
{
u32 reg;
/*
* Tracking also gets configured here because it's in the
* same register.
*/
u32 trk_sample_count = 7500;
u32 trk_long_idle_sample_count = (10 << 16) | 100;
/*
* Format is number of outer loops in the 16 MSB, sample
* count in 16 LSB.
*/
reg = 0;
reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ACDELAYEN_SET(2);
reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQDELAYEN_SET(1);
reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSDELAYEN_SET(1);
reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_DQSLOGICDELAYEN_SET(1);
reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_RESETDELAYEN_SET(0);
reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_LPDDRDIS_SET(1);
/*
* This field selects the intrinsic latency to RDATA_EN/FULL path.
* 00-bypass, 01- add 5 cycles, 10- add 10 cycles, 11- add 15 cycles.
*/
reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_ADDLATSEL_SET(0);
reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_SET(
trk_sample_count);
writel(reg, &sdr_ctrl->phy_ctrl0);
reg = 0;
reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_SAMPLECOUNT_31_20_SET(
trk_sample_count >>
SDR_CTRLGRP_PHYCTRL_PHYCTRL_0_SAMPLECOUNT_19_0_WIDTH);
reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_SET(
trk_long_idle_sample_count);
writel(reg, &sdr_ctrl->phy_ctrl1);
reg = 0;
reg |= SDR_CTRLGRP_PHYCTRL_PHYCTRL_2_LONGIDLESAMPLECOUNT_31_20_SET(
trk_long_idle_sample_count >>
SDR_CTRLGRP_PHYCTRL_PHYCTRL_1_LONGIDLESAMPLECOUNT_19_0_WIDTH);
writel(reg, &sdr_ctrl->phy_ctrl2);
}
/**
* initialize_tracking() - Initialize tracking
*
* Initialize the register file with usable initial data.
*/
static void initialize_tracking(struct socfpga_sdrseq *seq)
{
/*
* Initialize the register file with the correct data.
* Compute usable version of value in case we skip full
* computation later.
*/
writel(DIV_ROUND_UP(seq->iocfg->delay_per_opa_tap,
seq->iocfg->delay_per_dchain_tap) - 1,
&sdr_reg_file->dtaps_per_ptap);
/* trk_sample_count */
writel(7500, &sdr_reg_file->trk_sample_count);
/* longidle outer loop [15:0] */
writel((10 << 16) | (100 << 0), &sdr_reg_file->trk_longidle);
/*
* longidle sample count [31:24]
* trfc, worst case of 933Mhz 4Gb [23:16]
* trcd, worst case [15:8]
* vfifo wait [7:0]
*/
writel((243 << 24) | (14 << 16) | (10 << 8) | (4 << 0),
&sdr_reg_file->delays);
/* mux delay */
if (dram_is_ddr(2)) {
writel(0, &sdr_reg_file->trk_rw_mgr_addr);
} else if (dram_is_ddr(3)) {
writel((seq->rwcfg->idle << 24) |
(seq->rwcfg->activate_1 << 16) |
(seq->rwcfg->sgle_read << 8) |
(seq->rwcfg->precharge_all << 0),
&sdr_reg_file->trk_rw_mgr_addr);
}
writel(seq->rwcfg->mem_if_read_dqs_width,
&sdr_reg_file->trk_read_dqs_width);
/* trefi [7:0] */
if (dram_is_ddr(2)) {
writel(1000 << 0, &sdr_reg_file->trk_rfsh);
} else if (dram_is_ddr(3)) {
writel((seq->rwcfg->refresh_all << 24) | (1000 << 0),
&sdr_reg_file->trk_rfsh);
}
}
int sdram_calibration_full(struct socfpga_sdr *sdr)
{
u32 pass;
struct socfpga_sdrseq seq;
/*
* For size reasons, this file uses hard coded addresses.
* Check if we are called with the correct address.
*/
if (sdr != (struct socfpga_sdr *)SOCFPGA_SDR_ADDRESS)
return -ENODEV;
memset(&seq, 0, sizeof(seq));
seq.rwcfg = socfpga_get_sdram_rwmgr_config();
seq.iocfg = socfpga_get_sdram_io_config();
seq.misccfg = socfpga_get_sdram_misc_config();
/* Set the calibration enabled by default */
seq.gbl.phy_debug_mode_flags |= PHY_DEBUG_ENABLE_CAL_RPT;
/*
* Only sweep all groups (regardless of fail state) by default
* Set enabled read test by default.
*/
#if DISABLE_GUARANTEED_READ
seq.gbl.phy_debug_mode_flags |= PHY_DEBUG_DISABLE_GUARANTEED_READ;
#endif
/* Initialize the register file */
initialize_reg_file(&seq);
/* Initialize any PHY CSR */
initialize_hps_phy();
scc_mgr_initialize();
initialize_tracking(&seq);
debug("%s: Preparing to start memory calibration\n", __FILE__);
debug("%s:%d\n", __func__, __LINE__);
debug_cond(DLEVEL >= 1,
"DDR3 FULL_RATE ranks=%u cs/dimm=%u dq/dqs=%u,%u vg/dqs=%u,%u ",
seq.rwcfg->mem_number_of_ranks,
seq.rwcfg->mem_number_of_cs_per_dimm,
seq.rwcfg->mem_dq_per_read_dqs,
seq.rwcfg->mem_dq_per_write_dqs,
seq.rwcfg->mem_virtual_groups_per_read_dqs,
seq.rwcfg->mem_virtual_groups_per_write_dqs);
debug_cond(DLEVEL >= 1,
"dqs=%u,%u dq=%u dm=%u ptap_delay=%u dtap_delay=%u ",
seq.rwcfg->mem_if_read_dqs_width,
seq.rwcfg->mem_if_write_dqs_width,
seq.rwcfg->mem_data_width, seq.rwcfg->mem_data_mask_width,
seq.iocfg->delay_per_opa_tap,
seq.iocfg->delay_per_dchain_tap);
debug_cond(DLEVEL >= 1, "dtap_dqsen_delay=%u, dll=%u",
seq.iocfg->delay_per_dqs_en_dchain_tap,
seq.iocfg->dll_chain_length);
debug_cond(DLEVEL >= 1,
"max values: en_p=%u dqdqs_p=%u en_d=%u dqs_in_d=%u ",
seq.iocfg->dqs_en_phase_max, seq.iocfg->dqdqs_out_phase_max,
seq.iocfg->dqs_en_delay_max, seq.iocfg->dqs_in_delay_max);
debug_cond(DLEVEL >= 1, "io_in_d=%u io_out1_d=%u io_out2_d=%u ",
seq.iocfg->io_in_delay_max, seq.iocfg->io_out1_delay_max,
seq.iocfg->io_out2_delay_max);
debug_cond(DLEVEL >= 1, "dqs_in_reserve=%u dqs_out_reserve=%u\n",
seq.iocfg->dqs_in_reserve, seq.iocfg->dqs_out_reserve);
hc_initialize_rom_data();
/* update info for sims */
reg_file_set_stage(CAL_STAGE_NIL);
reg_file_set_group(0);
/*
* Load global needed for those actions that require
* some dynamic calibration support.
*/
seq.dyn_calib_steps = STATIC_CALIB_STEPS;
/*
* Load global to allow dynamic selection of delay loop settings
* based on calibration mode.
*/
if (!(seq.dyn_calib_steps & CALIB_SKIP_DELAY_LOOPS))
seq.skip_delay_mask = 0xff;
else
seq.skip_delay_mask = 0x0;
pass = run_mem_calibrate(&seq);
debug_mem_calibrate(&seq, pass);
return pass;
}
Reference in a new issue View git blame Copy permalink