1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 2022 MediaTek Inc. All rights reserved. 4 * 5 * Author: Weijie Gao <weijie.gao@mediatek.com> 6 */ 7 8#include <log.h> 9#include <nand.h> 10#include <malloc.h> 11#include <asm/addrspace.h> 12#include <linux/io.h> 13#include <linux/iopoll.h> 14#include <linux/sizes.h> 15#include <linux/bitops.h> 16#include <linux/bitfield.h> 17#include "mt7621_nand.h" 18 19/* NFI core registers */ 20#define NFI_CNFG 0x000 21#define CNFG_OP_MODE GENMASK(14, 12) 22#define CNFG_OP_CUSTOM 6 23#define CNFG_AUTO_FMT_EN BIT(9) 24#define CNFG_HW_ECC_EN BIT(8) 25#define CNFG_BYTE_RW BIT(6) 26#define CNFG_READ_MODE BIT(1) 27 28#define NFI_PAGEFMT 0x004 29#define PAGEFMT_FDM_ECC GENMASK(15, 12) 30#define PAGEFMT_FDM GENMASK(11, 8) 31#define PAGEFMT_SPARE GENMASK(5, 4) 32#define PAGEFMT_PAGE GENMASK(1, 0) 33 34#define NFI_CON 0x008 35#define CON_NFI_SEC GENMASK(15, 12) 36#define CON_NFI_BWR BIT(9) 37#define CON_NFI_BRD BIT(8) 38#define CON_NFI_RST BIT(1) 39#define CON_FIFO_FLUSH BIT(0) 40 41#define NFI_ACCCON 0x00c 42#define ACCCON_POECS GENMASK(31, 28) 43#define ACCCON_POECS_DEF 3 44#define ACCCON_PRECS GENMASK(27, 22) 45#define ACCCON_PRECS_DEF 3 46#define ACCCON_C2R GENMASK(21, 16) 47#define ACCCON_C2R_DEF 7 48#define ACCCON_W2R GENMASK(15, 12) 49#define ACCCON_W2R_DEF 7 50#define ACCCON_WH GENMASK(11, 8) 51#define ACCCON_WH_DEF 15 52#define ACCCON_WST GENMASK(7, 4) 53#define ACCCON_WST_DEF 15 54#define ACCCON_WST_MIN 3 55#define ACCCON_RLT GENMASK(3, 0) 56#define ACCCON_RLT_DEF 15 57#define ACCCON_RLT_MIN 3 58 59#define NFI_CMD 0x020 60 61#define NFI_ADDRNOB 0x030 62#define ADDR_ROW_NOB GENMASK(6, 4) 63#define ADDR_COL_NOB GENMASK(2, 0) 64 65#define NFI_COLADDR 0x034 66#define NFI_ROWADDR 0x038 67 68#define NFI_STRDATA 0x040 69#define STR_DATA BIT(0) 70 71#define NFI_CNRNB 0x044 72#define CB2R_TIME GENMASK(7, 4) 73#define STR_CNRNB BIT(0) 74 75#define NFI_DATAW 0x050 76#define NFI_DATAR 0x054 77 78#define NFI_PIO_DIRDY 0x058 79#define PIO_DIRDY BIT(0) 80 81#define NFI_STA 0x060 82#define STA_NFI_FSM GENMASK(19, 16) 83#define STA_FSM_CUSTOM_DATA 14 84#define STA_BUSY BIT(8) 85#define STA_ADDR BIT(1) 86#define STA_CMD BIT(0) 87 88#define NFI_ADDRCNTR 0x070 89#define SEC_CNTR GENMASK(15, 12) 90#define SEC_ADDR GENMASK(9, 0) 91 92#define NFI_CSEL 0x090 93#define CSEL GENMASK(1, 0) 94 95#define NFI_FDM0L 0x0a0 96#define NFI_FDML(n) (0x0a0 + ((n) << 3)) 97 98#define NFI_FDM0M 0x0a4 99#define NFI_FDMM(n) (0x0a4 + ((n) << 3)) 100 101#define NFI_MASTER_STA 0x210 102#define MAS_ADDR GENMASK(11, 9) 103#define MAS_RD GENMASK(8, 6) 104#define MAS_WR GENMASK(5, 3) 105#define MAS_RDDLY GENMASK(2, 0) 106 107/* ECC engine registers */ 108#define ECC_ENCCON 0x000 109#define ENC_EN BIT(0) 110 111#define ECC_ENCCNFG 0x004 112#define ENC_CNFG_MSG GENMASK(28, 16) 113#define ENC_MODE GENMASK(5, 4) 114#define ENC_MODE_NFI 1 115#define ENC_TNUM GENMASK(2, 0) 116 117#define ECC_ENCIDLE 0x00c 118#define ENC_IDLE BIT(0) 119 120#define ECC_DECCON 0x100 121#define DEC_EN BIT(0) 122 123#define ECC_DECCNFG 0x104 124#define DEC_EMPTY_EN BIT(31) 125#define DEC_CS GENMASK(28, 16) 126#define DEC_CON GENMASK(13, 12) 127#define DEC_CON_EL 2 128#define DEC_MODE GENMASK(5, 4) 129#define DEC_MODE_NFI 1 130#define DEC_TNUM GENMASK(2, 0) 131 132#define ECC_DECIDLE 0x10c 133#define DEC_IDLE BIT(1) 134 135#define ECC_DECENUM 0x114 136#define ERRNUM_S 2 137#define ERRNUM_M GENMASK(3, 0) 138 139#define ECC_DECDONE 0x118 140#define DEC_DONE7 BIT(7) 141#define DEC_DONE6 BIT(6) 142#define DEC_DONE5 BIT(5) 143#define DEC_DONE4 BIT(4) 144#define DEC_DONE3 BIT(3) 145#define DEC_DONE2 BIT(2) 146#define DEC_DONE1 BIT(1) 147#define DEC_DONE0 BIT(0) 148 149#define ECC_DECEL(n) (0x11c + (n) * 4) 150#define DEC_EL_ODD_S 16 151#define DEC_EL_M 0x1fff 152#define DEC_EL_BYTE_POS_S 3 153#define DEC_EL_BIT_POS_M GENMASK(2, 0) 154 155#define ECC_FDMADDR 0x13c 156 157/* ENCIDLE and DECIDLE */ 158#define ECC_IDLE BIT(0) 159 160#define ACCTIMING(tpoecs, tprecs, tc2r, tw2r, twh, twst, trlt) \ 161 (FIELD_PREP(ACCCON_POECS, tpoecs) | \ 162 FIELD_PREP(ACCCON_PRECS, tprecs) | \ 163 FIELD_PREP(ACCCON_C2R, tc2r) | \ 164 FIELD_PREP(ACCCON_W2R, tw2r) | \ 165 FIELD_PREP(ACCCON_WH, twh) | \ 166 FIELD_PREP(ACCCON_WST, twst) | \ 167 FIELD_PREP(ACCCON_RLT, trlt)) 168 169#define MASTER_STA_MASK (MAS_ADDR | MAS_RD | MAS_WR | \ 170 MAS_RDDLY) 171#define NFI_RESET_TIMEOUT 1000000 172#define NFI_CORE_TIMEOUT 500000 173#define ECC_ENGINE_TIMEOUT 500000 174 175#define ECC_SECTOR_SIZE 512 176#define ECC_PARITY_BITS 13 177 178#define NFI_FDM_SIZE 8 179 180/* Register base */ 181#define NFI_BASE 0x1e003000 182#define NFI_ECC_BASE 0x1e003800 183 184static struct mt7621_nfc nfc_dev; 185 186static const u16 mt7621_nfi_page_size[] = { SZ_512, SZ_2K, SZ_4K }; 187static const u8 mt7621_nfi_spare_size[] = { 16, 26, 27, 28 }; 188static const u8 mt7621_ecc_strength[] = { 4, 6, 8, 10, 12 }; 189 190static inline u32 nfi_read32(struct mt7621_nfc *nfc, u32 reg) 191{ 192 return readl(nfc->nfi_regs + reg); 193} 194 195static inline void nfi_write32(struct mt7621_nfc *nfc, u32 reg, u32 val) 196{ 197 writel(val, nfc->nfi_regs + reg); 198} 199 200static inline u16 nfi_read16(struct mt7621_nfc *nfc, u32 reg) 201{ 202 return readw(nfc->nfi_regs + reg); 203} 204 205static inline void nfi_write16(struct mt7621_nfc *nfc, u32 reg, u16 val) 206{ 207 writew(val, nfc->nfi_regs + reg); 208} 209 210static inline void ecc_write16(struct mt7621_nfc *nfc, u32 reg, u16 val) 211{ 212 writew(val, nfc->ecc_regs + reg); 213} 214 215static inline u32 ecc_read32(struct mt7621_nfc *nfc, u32 reg) 216{ 217 return readl(nfc->ecc_regs + reg); 218} 219 220static inline void ecc_write32(struct mt7621_nfc *nfc, u32 reg, u32 val) 221{ 222 return writel(val, nfc->ecc_regs + reg); 223} 224 225static inline u8 *oob_fdm_ptr(struct nand_chip *nand, int sect) 226{ 227 return nand->oob_poi + sect * NFI_FDM_SIZE; 228} 229 230static inline u8 *oob_ecc_ptr(struct mt7621_nfc *nfc, int sect) 231{ 232 struct nand_chip *nand = &nfc->nand; 233 234 return nand->oob_poi + nand->ecc.steps * NFI_FDM_SIZE + 235 sect * (nfc->spare_per_sector - NFI_FDM_SIZE); 236} 237 238static inline u8 *page_data_ptr(struct nand_chip *nand, const u8 *buf, 239 int sect) 240{ 241 return (u8 *)buf + sect * nand->ecc.size; 242} 243 244static int mt7621_ecc_wait_idle(struct mt7621_nfc *nfc, u32 reg) 245{ 246 u32 val; 247 int ret; 248 249 ret = readw_poll_timeout(nfc->ecc_regs + reg, val, val & ECC_IDLE, 250 ECC_ENGINE_TIMEOUT); 251 if (ret) { 252 pr_warn("ECC engine timed out entering idle mode\n"); 253 return -EIO; 254 } 255 256 return 0; 257} 258 259static int mt7621_ecc_decoder_wait_done(struct mt7621_nfc *nfc, u32 sect) 260{ 261 u32 val; 262 int ret; 263 264 ret = readw_poll_timeout(nfc->ecc_regs + ECC_DECDONE, val, 265 val & (1 << sect), ECC_ENGINE_TIMEOUT); 266 if (ret) { 267 pr_warn("ECC decoder for sector %d timed out\n", sect); 268 return -ETIMEDOUT; 269 } 270 271 return 0; 272} 273 274static void mt7621_ecc_encoder_op(struct mt7621_nfc *nfc, bool enable) 275{ 276 mt7621_ecc_wait_idle(nfc, ECC_ENCIDLE); 277 ecc_write16(nfc, ECC_ENCCON, enable ? ENC_EN : 0); 278} 279 280static void mt7621_ecc_decoder_op(struct mt7621_nfc *nfc, bool enable) 281{ 282 mt7621_ecc_wait_idle(nfc, ECC_DECIDLE); 283 ecc_write16(nfc, ECC_DECCON, enable ? DEC_EN : 0); 284} 285 286static int mt7621_ecc_correct_check(struct mt7621_nfc *nfc, u8 *sector_buf, 287 u8 *fdm_buf, u32 sect) 288{ 289 struct nand_chip *nand = &nfc->nand; 290 u32 decnum, num_error_bits, fdm_end_bits; 291 u32 error_locations, error_bit_loc; 292 u32 error_byte_pos, error_bit_pos; 293 int bitflips = 0; 294 u32 i; 295 296 decnum = ecc_read32(nfc, ECC_DECENUM); 297 num_error_bits = (decnum >> (sect << ERRNUM_S)) & ERRNUM_M; 298 fdm_end_bits = (nand->ecc.size + NFI_FDM_SIZE) << 3; 299 300 if (!num_error_bits) 301 return 0; 302 303 if (num_error_bits == ERRNUM_M) 304 return -1; 305 306 for (i = 0; i < num_error_bits; i++) { 307 error_locations = ecc_read32(nfc, ECC_DECEL(i / 2)); 308 error_bit_loc = (error_locations >> ((i % 2) * DEC_EL_ODD_S)) & 309 DEC_EL_M; 310 error_byte_pos = error_bit_loc >> DEC_EL_BYTE_POS_S; 311 error_bit_pos = error_bit_loc & DEC_EL_BIT_POS_M; 312 313 if (error_bit_loc < (nand->ecc.size << 3)) { 314 if (sector_buf) { 315 sector_buf[error_byte_pos] ^= 316 (1 << error_bit_pos); 317 } 318 } else if (error_bit_loc < fdm_end_bits) { 319 if (fdm_buf) { 320 fdm_buf[error_byte_pos - nand->ecc.size] ^= 321 (1 << error_bit_pos); 322 } 323 } 324 325 bitflips++; 326 } 327 328 return bitflips; 329} 330 331static int mt7621_nfc_wait_write_completion(struct mt7621_nfc *nfc, 332 struct nand_chip *nand) 333{ 334 u16 val; 335 int ret; 336 337 ret = readw_poll_timeout(nfc->nfi_regs + NFI_ADDRCNTR, val, 338 FIELD_GET(SEC_CNTR, val) >= nand->ecc.steps, 339 NFI_CORE_TIMEOUT); 340 341 if (ret) { 342 pr_warn("NFI core write operation timed out\n"); 343 return -ETIMEDOUT; 344 } 345 346 return ret; 347} 348 349static void mt7621_nfc_hw_reset(struct mt7621_nfc *nfc) 350{ 351 u32 val; 352 int ret; 353 354 /* reset all registers and force the NFI master to terminate */ 355 nfi_write16(nfc, NFI_CON, CON_FIFO_FLUSH | CON_NFI_RST); 356 357 /* wait for the master to finish the last transaction */ 358 ret = readw_poll_timeout(nfc->nfi_regs + NFI_MASTER_STA, val, 359 !(val & MASTER_STA_MASK), NFI_RESET_TIMEOUT); 360 if (ret) { 361 pr_warn("Failed to reset NFI master in %dms\n", 362 NFI_RESET_TIMEOUT); 363 } 364 365 /* ensure any status register affected by the NFI master is reset */ 366 nfi_write16(nfc, NFI_CON, CON_FIFO_FLUSH | CON_NFI_RST); 367 nfi_write16(nfc, NFI_STRDATA, 0); 368} 369 370static inline void mt7621_nfc_hw_init(struct mt7621_nfc *nfc) 371{ 372 u32 acccon; 373 374 /* 375 * CNRNB: nand ready/busy register 376 * ------------------------------- 377 * 7:4: timeout register for polling the NAND busy/ready signal 378 * 0 : poll the status of the busy/ready signal after [7:4]*16 cycles. 379 */ 380 nfi_write16(nfc, NFI_CNRNB, CB2R_TIME | STR_CNRNB); 381 382 mt7621_nfc_hw_reset(nfc); 383 384 /* Apply default access timing */ 385 acccon = ACCTIMING(ACCCON_POECS_DEF, ACCCON_PRECS_DEF, ACCCON_C2R_DEF, 386 ACCCON_W2R_DEF, ACCCON_WH_DEF, ACCCON_WST_DEF, 387 ACCCON_RLT_DEF); 388 389 nfi_write32(nfc, NFI_ACCCON, acccon); 390} 391 392static int mt7621_nfc_send_command(struct mt7621_nfc *nfc, u8 command) 393{ 394 u32 val; 395 int ret; 396 397 nfi_write32(nfc, NFI_CMD, command); 398 399 ret = readl_poll_timeout(nfc->nfi_regs + NFI_STA, val, !(val & STA_CMD), 400 NFI_CORE_TIMEOUT); 401 if (ret) { 402 pr_warn("NFI core timed out entering command mode\n"); 403 return -EIO; 404 } 405 406 return 0; 407} 408 409static int mt7621_nfc_send_address_byte(struct mt7621_nfc *nfc, int addr) 410{ 411 u32 val; 412 int ret; 413 414 nfi_write32(nfc, NFI_COLADDR, addr); 415 nfi_write32(nfc, NFI_ROWADDR, 0); 416 nfi_write16(nfc, NFI_ADDRNOB, 1); 417 418 ret = readl_poll_timeout(nfc->nfi_regs + NFI_STA, val, 419 !(val & STA_ADDR), NFI_CORE_TIMEOUT); 420 if (ret) { 421 pr_warn("NFI core timed out entering address mode\n"); 422 return -EIO; 423 } 424 425 return 0; 426} 427 428static void mt7621_nfc_cmd_ctrl(struct mtd_info *mtd, int dat, 429 unsigned int ctrl) 430{ 431 struct mt7621_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); 432 433 if (ctrl & NAND_ALE) { 434 mt7621_nfc_send_address_byte(nfc, dat & 0xff); 435 } else if (ctrl & NAND_CLE) { 436 mt7621_nfc_hw_reset(nfc); 437 nfi_write16(nfc, NFI_CNFG, 438 FIELD_PREP(CNFG_OP_MODE, CNFG_OP_CUSTOM)); 439 mt7621_nfc_send_command(nfc, dat); 440 } 441} 442 443static int mt7621_nfc_dev_ready(struct mtd_info *mtd) 444{ 445 struct mt7621_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); 446 447 if (nfi_read32(nfc, NFI_STA) & STA_BUSY) 448 return 0; 449 450 return 1; 451} 452 453static void mt7621_nfc_select_chip(struct mtd_info *mtd, int chipnr) 454{ 455 struct mt7621_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); 456 457 nfi_write16(nfc, NFI_CSEL, 0); 458} 459 460static void mt7621_nfc_wait_pio_ready(struct mt7621_nfc *nfc) 461{ 462 int ret; 463 u16 val; 464 465 ret = readw_poll_timeout(nfc->nfi_regs + NFI_PIO_DIRDY, val, 466 val & PIO_DIRDY, NFI_CORE_TIMEOUT); 467 if (ret < 0) 468 pr_err("NFI core PIO mode not ready\n"); 469} 470 471static u32 mt7621_nfc_pio_read(struct mt7621_nfc *nfc, bool br) 472{ 473 u32 reg, fsm; 474 475 /* after each byte read, the NFI_STA reg is reset by the hardware */ 476 reg = nfi_read32(nfc, NFI_STA); 477 fsm = FIELD_GET(STA_NFI_FSM, reg); 478 479 if (fsm != STA_FSM_CUSTOM_DATA) { 480 reg = nfi_read16(nfc, NFI_CNFG); 481 reg |= CNFG_READ_MODE | CNFG_BYTE_RW; 482 if (!br) 483 reg &= ~CNFG_BYTE_RW; 484 nfi_write16(nfc, NFI_CNFG, reg); 485 486 /* 487 * set to max sector to allow the HW to continue reading over 488 * unaligned accesses 489 */ 490 nfi_write16(nfc, NFI_CON, CON_NFI_SEC | CON_NFI_BRD); 491 492 /* trigger to fetch data */ 493 nfi_write16(nfc, NFI_STRDATA, STR_DATA); 494 } 495 496 mt7621_nfc_wait_pio_ready(nfc); 497 498 return nfi_read32(nfc, NFI_DATAR); 499} 500 501static void mt7621_nfc_read_data(struct mt7621_nfc *nfc, u8 *buf, u32 len) 502{ 503 while (((uintptr_t)buf & 3) && len) { 504 *buf = mt7621_nfc_pio_read(nfc, true); 505 buf++; 506 len--; 507 } 508 509 while (len >= 4) { 510 *(u32 *)buf = mt7621_nfc_pio_read(nfc, false); 511 buf += 4; 512 len -= 4; 513 } 514 515 while (len) { 516 *buf = mt7621_nfc_pio_read(nfc, true); 517 buf++; 518 len--; 519 } 520} 521 522static void mt7621_nfc_read_data_discard(struct mt7621_nfc *nfc, u32 len) 523{ 524 while (len >= 4) { 525 mt7621_nfc_pio_read(nfc, false); 526 len -= 4; 527 } 528 529 while (len) { 530 mt7621_nfc_pio_read(nfc, true); 531 len--; 532 } 533} 534 535static void mt7621_nfc_pio_write(struct mt7621_nfc *nfc, u32 val, bool bw) 536{ 537 u32 reg, fsm; 538 539 reg = nfi_read32(nfc, NFI_STA); 540 fsm = FIELD_GET(STA_NFI_FSM, reg); 541 542 if (fsm != STA_FSM_CUSTOM_DATA) { 543 reg = nfi_read16(nfc, NFI_CNFG); 544 reg &= ~(CNFG_READ_MODE | CNFG_BYTE_RW); 545 if (bw) 546 reg |= CNFG_BYTE_RW; 547 nfi_write16(nfc, NFI_CNFG, reg); 548 549 nfi_write16(nfc, NFI_CON, CON_NFI_SEC | CON_NFI_BWR); 550 nfi_write16(nfc, NFI_STRDATA, STR_DATA); 551 } 552 553 mt7621_nfc_wait_pio_ready(nfc); 554 nfi_write32(nfc, NFI_DATAW, val); 555} 556 557static void mt7621_nfc_write_data(struct mt7621_nfc *nfc, const u8 *buf, 558 u32 len) 559{ 560 while (((uintptr_t)buf & 3) && len) { 561 mt7621_nfc_pio_write(nfc, *buf, true); 562 buf++; 563 len--; 564 } 565 566 while (len >= 4) { 567 mt7621_nfc_pio_write(nfc, *(const u32 *)buf, false); 568 buf += 4; 569 len -= 4; 570 } 571 572 while (len) { 573 mt7621_nfc_pio_write(nfc, *buf, true); 574 buf++; 575 len--; 576 } 577} 578 579static void mt7621_nfc_write_data_empty(struct mt7621_nfc *nfc, u32 len) 580{ 581 while (len >= 4) { 582 mt7621_nfc_pio_write(nfc, 0xffffffff, false); 583 len -= 4; 584 } 585 586 while (len) { 587 mt7621_nfc_pio_write(nfc, 0xff, true); 588 len--; 589 } 590} 591 592static void mt7621_nfc_write_byte(struct mtd_info *mtd, u8 byte) 593{ 594 struct mt7621_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); 595 596 mt7621_nfc_pio_write(nfc, byte, true); 597} 598 599static void mt7621_nfc_write_buf(struct mtd_info *mtd, const u8 *buf, int len) 600{ 601 struct mt7621_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); 602 603 return mt7621_nfc_write_data(nfc, buf, len); 604} 605 606static u8 mt7621_nfc_read_byte(struct mtd_info *mtd) 607{ 608 struct mt7621_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); 609 610 return mt7621_nfc_pio_read(nfc, true); 611} 612 613static void mt7621_nfc_read_buf(struct mtd_info *mtd, u8 *buf, int len) 614{ 615 struct mt7621_nfc *nfc = nand_get_controller_data(mtd_to_nand(mtd)); 616 617 mt7621_nfc_read_data(nfc, buf, len); 618} 619 620static int mt7621_nfc_calc_ecc_strength(struct mt7621_nfc *nfc, 621 u32 avail_ecc_bytes) 622{ 623 struct nand_chip *nand = &nfc->nand; 624 struct mtd_info *mtd = nand_to_mtd(nand); 625 u32 strength; 626 int i; 627 628 strength = avail_ecc_bytes * 8 / ECC_PARITY_BITS; 629 630 /* Find the closest supported ecc strength */ 631 for (i = ARRAY_SIZE(mt7621_ecc_strength) - 1; i >= 0; i--) { 632 if (mt7621_ecc_strength[i] <= strength) 633 break; 634 } 635 636 if (unlikely(i < 0)) { 637 pr_err("OOB size (%u) is not supported\n", mtd->oobsize); 638 return -EINVAL; 639 } 640 641 nand->ecc.strength = mt7621_ecc_strength[i]; 642 nand->ecc.bytes = DIV_ROUND_UP(nand->ecc.strength * ECC_PARITY_BITS, 8); 643 644 pr_debug("ECC strength adjusted to %u bits\n", nand->ecc.strength); 645 646 return i; 647} 648 649static int mt7621_nfc_set_spare_per_sector(struct mt7621_nfc *nfc) 650{ 651 struct nand_chip *nand = &nfc->nand; 652 struct mtd_info *mtd = nand_to_mtd(nand); 653 u32 size; 654 int i; 655 656 size = nand->ecc.bytes + NFI_FDM_SIZE; 657 658 /* Find the closest supported spare size */ 659 for (i = 0; i < ARRAY_SIZE(mt7621_nfi_spare_size); i++) { 660 if (mt7621_nfi_spare_size[i] >= size) 661 break; 662 } 663 664 if (unlikely(i >= ARRAY_SIZE(mt7621_nfi_spare_size))) { 665 pr_err("OOB size (%u) is not supported\n", mtd->oobsize); 666 return -EINVAL; 667 } 668 669 nfc->spare_per_sector = mt7621_nfi_spare_size[i]; 670 671 return i; 672} 673 674static int mt7621_nfc_ecc_init(struct mt7621_nfc *nfc) 675{ 676 struct nand_chip *nand = &nfc->nand; 677 struct mtd_info *mtd = nand_to_mtd(nand); 678 u32 avail_ecc_bytes, encode_block_size, decode_block_size; 679 u32 ecc_enccfg, ecc_deccfg; 680 int ecc_cap; 681 682 nand->ecc.options |= NAND_ECC_CUSTOM_PAGE_ACCESS; 683 684 nand->ecc.size = ECC_SECTOR_SIZE; 685 nand->ecc.steps = mtd->writesize / nand->ecc.size; 686 687 avail_ecc_bytes = mtd->oobsize / nand->ecc.steps - NFI_FDM_SIZE; 688 689 ecc_cap = mt7621_nfc_calc_ecc_strength(nfc, avail_ecc_bytes); 690 if (ecc_cap < 0) 691 return ecc_cap; 692 693 /* Sector + FDM */ 694 encode_block_size = (nand->ecc.size + NFI_FDM_SIZE) * 8; 695 ecc_enccfg = ecc_cap | FIELD_PREP(ENC_MODE, ENC_MODE_NFI) | 696 FIELD_PREP(ENC_CNFG_MSG, encode_block_size); 697 698 /* Sector + FDM + ECC parity bits */ 699 decode_block_size = ((nand->ecc.size + NFI_FDM_SIZE) * 8) + 700 nand->ecc.strength * ECC_PARITY_BITS; 701 ecc_deccfg = ecc_cap | FIELD_PREP(DEC_MODE, DEC_MODE_NFI) | 702 FIELD_PREP(DEC_CS, decode_block_size) | 703 FIELD_PREP(DEC_CON, DEC_CON_EL) | DEC_EMPTY_EN; 704 705 mt7621_ecc_encoder_op(nfc, false); 706 ecc_write32(nfc, ECC_ENCCNFG, ecc_enccfg); 707 708 mt7621_ecc_decoder_op(nfc, false); 709 ecc_write32(nfc, ECC_DECCNFG, ecc_deccfg); 710 711 return 0; 712} 713 714static int mt7621_nfc_set_page_format(struct mt7621_nfc *nfc) 715{ 716 struct nand_chip *nand = &nfc->nand; 717 struct mtd_info *mtd = nand_to_mtd(nand); 718 int i, spare_size; 719 u32 pagefmt; 720 721 spare_size = mt7621_nfc_set_spare_per_sector(nfc); 722 if (spare_size < 0) 723 return spare_size; 724 725 for (i = 0; i < ARRAY_SIZE(mt7621_nfi_page_size); i++) { 726 if (mt7621_nfi_page_size[i] == mtd->writesize) 727 break; 728 } 729 730 if (unlikely(i >= ARRAY_SIZE(mt7621_nfi_page_size))) { 731 pr_err("Page size (%u) is not supported\n", mtd->writesize); 732 return -EINVAL; 733 } 734 735 pagefmt = FIELD_PREP(PAGEFMT_PAGE, i) | 736 FIELD_PREP(PAGEFMT_SPARE, spare_size) | 737 FIELD_PREP(PAGEFMT_FDM, NFI_FDM_SIZE) | 738 FIELD_PREP(PAGEFMT_FDM_ECC, NFI_FDM_SIZE); 739 740 nfi_write16(nfc, NFI_PAGEFMT, pagefmt); 741 742 return 0; 743} 744 745static int mt7621_nfc_attach_chip(struct nand_chip *nand) 746{ 747 struct mt7621_nfc *nfc = nand_get_controller_data(nand); 748 int ret; 749 750 if (nand->options & NAND_BUSWIDTH_16) { 751 pr_err("16-bit buswidth is not supported"); 752 return -EINVAL; 753 } 754 755 ret = mt7621_nfc_ecc_init(nfc); 756 if (ret) 757 return ret; 758 759 return mt7621_nfc_set_page_format(nfc); 760} 761 762static void mt7621_nfc_write_fdm(struct mt7621_nfc *nfc) 763{ 764 struct nand_chip *nand = &nfc->nand; 765 u32 vall, valm; 766 u8 *oobptr; 767 int i, j; 768 769 for (i = 0; i < nand->ecc.steps; i++) { 770 vall = 0; 771 valm = 0; 772 oobptr = oob_fdm_ptr(nand, i); 773 774 for (j = 0; j < 4; j++) 775 vall |= (u32)oobptr[j] << (j * 8); 776 777 for (j = 0; j < 4; j++) 778 valm |= (u32)oobptr[j + 4] << (j * 8); 779 780 nfi_write32(nfc, NFI_FDML(i), vall); 781 nfi_write32(nfc, NFI_FDMM(i), valm); 782 } 783} 784 785static void mt7621_nfc_read_sector_fdm(struct mt7621_nfc *nfc, u32 sect) 786{ 787 struct nand_chip *nand = &nfc->nand; 788 u32 vall, valm; 789 u8 *oobptr; 790 int i; 791 792 vall = nfi_read32(nfc, NFI_FDML(sect)); 793 valm = nfi_read32(nfc, NFI_FDMM(sect)); 794 oobptr = oob_fdm_ptr(nand, sect); 795 796 for (i = 0; i < 4; i++) 797 oobptr[i] = (vall >> (i * 8)) & 0xff; 798 799 for (i = 0; i < 4; i++) 800 oobptr[i + 4] = (valm >> (i * 8)) & 0xff; 801} 802 803static int mt7621_nfc_read_page_hwecc(struct mtd_info *mtd, 804 struct nand_chip *nand, uint8_t *buf, 805 int oob_required, int page) 806{ 807 struct mt7621_nfc *nfc = nand_get_controller_data(nand); 808 int bitflips = 0, ret = 0; 809 int rc, i; 810 811 nand_read_page_op(nand, page, 0, NULL, 0); 812 813 nfi_write16(nfc, NFI_CNFG, FIELD_PREP(CNFG_OP_MODE, CNFG_OP_CUSTOM) | 814 CNFG_READ_MODE | CNFG_AUTO_FMT_EN | CNFG_HW_ECC_EN); 815 816 mt7621_ecc_decoder_op(nfc, true); 817 818 nfi_write16(nfc, NFI_CON, FIELD_PREP(CON_NFI_SEC, nand->ecc.steps) | 819 CON_NFI_BRD); 820 821 for (i = 0; i < nand->ecc.steps; i++) { 822 if (buf) 823 mt7621_nfc_read_data(nfc, page_data_ptr(nand, buf, i), 824 nand->ecc.size); 825 else 826 mt7621_nfc_read_data_discard(nfc, nand->ecc.size); 827 828 rc = mt7621_ecc_decoder_wait_done(nfc, i); 829 830 mt7621_nfc_read_sector_fdm(nfc, i); 831 832 if (rc < 0) { 833 ret = -EIO; 834 continue; 835 } 836 837 rc = mt7621_ecc_correct_check(nfc, 838 buf ? page_data_ptr(nand, buf, i) : NULL, 839 oob_fdm_ptr(nand, i), i); 840 841 if (rc < 0) { 842 pr_warn("Uncorrectable ECC error at page %d step %d\n", 843 page, i); 844 bitflips = nand->ecc.strength + 1; 845 mtd->ecc_stats.failed++; 846 } else { 847 if (rc > bitflips) 848 bitflips = rc; 849 mtd->ecc_stats.corrected += rc; 850 } 851 } 852 853 mt7621_ecc_decoder_op(nfc, false); 854 855 nfi_write16(nfc, NFI_CON, 0); 856 857 if (ret < 0) 858 return ret; 859 860 return bitflips; 861} 862 863static int mt7621_nfc_read_page_raw(struct mtd_info *mtd, 864 struct nand_chip *nand, uint8_t *buf, 865 int oob_required, int page) 866{ 867 struct mt7621_nfc *nfc = nand_get_controller_data(nand); 868 int i; 869 870 nand_read_page_op(nand, page, 0, NULL, 0); 871 872 nfi_write16(nfc, NFI_CNFG, FIELD_PREP(CNFG_OP_MODE, CNFG_OP_CUSTOM) | 873 CNFG_READ_MODE); 874 875 nfi_write16(nfc, NFI_CON, FIELD_PREP(CON_NFI_SEC, nand->ecc.steps) | 876 CON_NFI_BRD); 877 878 for (i = 0; i < nand->ecc.steps; i++) { 879 /* Read data */ 880 if (buf) 881 mt7621_nfc_read_data(nfc, page_data_ptr(nand, buf, i), 882 nand->ecc.size); 883 else 884 mt7621_nfc_read_data_discard(nfc, nand->ecc.size); 885 886 /* Read FDM */ 887 mt7621_nfc_read_data(nfc, oob_fdm_ptr(nand, i), NFI_FDM_SIZE); 888 889 /* Read ECC parity data */ 890 mt7621_nfc_read_data(nfc, oob_ecc_ptr(nfc, i), 891 nfc->spare_per_sector - NFI_FDM_SIZE); 892 } 893 894 nfi_write16(nfc, NFI_CON, 0); 895 896 return 0; 897} 898 899static int mt7621_nfc_read_oob_hwecc(struct mtd_info *mtd, 900 struct nand_chip *nand, int page) 901{ 902 return mt7621_nfc_read_page_hwecc(mtd, nand, NULL, 1, page); 903} 904 905static int mt7621_nfc_read_oob_raw(struct mtd_info *mtd, 906 struct nand_chip *nand, int page) 907{ 908 return mt7621_nfc_read_page_raw(mtd, nand, NULL, 1, page); 909} 910 911static int mt7621_nfc_check_empty_page(struct nand_chip *nand, const u8 *buf) 912{ 913 struct mtd_info *mtd = nand_to_mtd(nand); 914 u8 *oobptr; 915 u32 i, j; 916 917 if (buf) { 918 for (i = 0; i < mtd->writesize; i++) 919 if (buf[i] != 0xff) 920 return 0; 921 } 922 923 for (i = 0; i < nand->ecc.steps; i++) { 924 oobptr = oob_fdm_ptr(nand, i); 925 for (j = 0; j < NFI_FDM_SIZE; j++) 926 if (oobptr[j] != 0xff) 927 return 0; 928 } 929 930 return 1; 931} 932 933static int mt7621_nfc_write_page_hwecc(struct mtd_info *mtd, 934 struct nand_chip *nand, 935 const u8 *buf, int oob_required, 936 int page) 937{ 938 struct mt7621_nfc *nfc = nand_get_controller_data(nand); 939 940 if (mt7621_nfc_check_empty_page(nand, buf)) { 941 /* 942 * MT7621 ECC engine always generates parity code for input 943 * pages, even for empty pages. Doing so will write back ECC 944 * parity code to the oob region, which means such pages will 945 * no longer be empty pages. 946 * 947 * To avoid this, stop write operation if current page is an 948 * empty page. 949 */ 950 return 0; 951 } 952 953 nand_prog_page_begin_op(nand, page, 0, NULL, 0); 954 955 nfi_write16(nfc, NFI_CNFG, FIELD_PREP(CNFG_OP_MODE, CNFG_OP_CUSTOM) | 956 CNFG_AUTO_FMT_EN | CNFG_HW_ECC_EN); 957 958 mt7621_ecc_encoder_op(nfc, true); 959 960 mt7621_nfc_write_fdm(nfc); 961 962 nfi_write16(nfc, NFI_CON, FIELD_PREP(CON_NFI_SEC, nand->ecc.steps) | 963 CON_NFI_BWR); 964 965 if (buf) 966 mt7621_nfc_write_data(nfc, buf, mtd->writesize); 967 else 968 mt7621_nfc_write_data_empty(nfc, mtd->writesize); 969 970 mt7621_nfc_wait_write_completion(nfc, nand); 971 972 mt7621_ecc_encoder_op(nfc, false); 973 974 nfi_write16(nfc, NFI_CON, 0); 975 976 return nand_prog_page_end_op(nand); 977} 978 979static int mt7621_nfc_write_page_raw(struct mtd_info *mtd, 980 struct nand_chip *nand, 981 const u8 *buf, int oob_required, 982 int page) 983{ 984 struct mt7621_nfc *nfc = nand_get_controller_data(nand); 985 int i; 986 987 nand_prog_page_begin_op(nand, page, 0, NULL, 0); 988 989 nfi_write16(nfc, NFI_CNFG, FIELD_PREP(CNFG_OP_MODE, CNFG_OP_CUSTOM)); 990 991 nfi_write16(nfc, NFI_CON, FIELD_PREP(CON_NFI_SEC, nand->ecc.steps) | 992 CON_NFI_BWR); 993 994 for (i = 0; i < nand->ecc.steps; i++) { 995 /* Write data */ 996 if (buf) 997 mt7621_nfc_write_data(nfc, page_data_ptr(nand, buf, i), 998 nand->ecc.size); 999 else 1000 mt7621_nfc_write_data_empty(nfc, nand->ecc.size); 1001 1002 /* Write FDM */ 1003 mt7621_nfc_write_data(nfc, oob_fdm_ptr(nand, i), 1004 NFI_FDM_SIZE); 1005 1006 /* Write dummy ECC parity data */ 1007 mt7621_nfc_write_data_empty(nfc, nfc->spare_per_sector - 1008 NFI_FDM_SIZE); 1009 } 1010 1011 mt7621_nfc_wait_write_completion(nfc, nand); 1012 1013 nfi_write16(nfc, NFI_CON, 0); 1014 1015 return nand_prog_page_end_op(nand); 1016} 1017 1018static int mt7621_nfc_write_oob_hwecc(struct mtd_info *mtd, 1019 struct nand_chip *nand, int page) 1020{ 1021 return mt7621_nfc_write_page_hwecc(mtd, nand, NULL, 1, page); 1022} 1023 1024static int mt7621_nfc_write_oob_raw(struct mtd_info *mtd, 1025 struct nand_chip *nand, int page) 1026{ 1027 return mt7621_nfc_write_page_raw(mtd, nand, NULL, 1, page); 1028} 1029 1030static int mt7621_nfc_ooblayout_free(struct mtd_info *mtd, int section, 1031 struct mtd_oob_region *oob_region) 1032{ 1033 struct nand_chip *nand = mtd_to_nand(mtd); 1034 1035 if (section >= nand->ecc.steps) 1036 return -ERANGE; 1037 1038 oob_region->length = NFI_FDM_SIZE - 1; 1039 oob_region->offset = section * NFI_FDM_SIZE + 1; 1040 1041 return 0; 1042} 1043 1044static int mt7621_nfc_ooblayout_ecc(struct mtd_info *mtd, int section, 1045 struct mtd_oob_region *oob_region) 1046{ 1047 struct nand_chip *nand = mtd_to_nand(mtd); 1048 1049 if (section) 1050 return -ERANGE; 1051 1052 oob_region->offset = NFI_FDM_SIZE * nand->ecc.steps; 1053 oob_region->length = mtd->oobsize - oob_region->offset; 1054 1055 return 0; 1056} 1057 1058static const struct mtd_ooblayout_ops mt7621_nfc_ooblayout_ops = { 1059 .rfree = mt7621_nfc_ooblayout_free, 1060 .ecc = mt7621_nfc_ooblayout_ecc, 1061}; 1062 1063/* 1064 * This function will override the default one which is not supposed to be 1065 * used for ECC syndrome based pages. 1066 */ 1067static int mt7621_nfc_block_bad(struct mtd_info *mtd, loff_t ofs) 1068{ 1069 struct nand_chip *nand = mtd_to_nand(mtd); 1070 struct mtd_oob_ops ops; 1071 int ret, i = 0; 1072 u16 bad; 1073 1074 memset(&ops, 0, sizeof(ops)); 1075 ops.oobbuf = (uint8_t *)&bad; 1076 ops.ooboffs = nand->badblockpos; 1077 if (nand->options & NAND_BUSWIDTH_16) { 1078 ops.ooboffs &= ~0x01; 1079 ops.ooblen = 2; 1080 } else { 1081 ops.ooblen = 1; 1082 } 1083 ops.mode = MTD_OPS_RAW; 1084 1085 /* Read from first/last page(s) if necessary */ 1086 if (nand->bbt_options & NAND_BBT_SCANLASTPAGE) 1087 ofs += mtd->erasesize - mtd->writesize; 1088 1089 do { 1090 ret = mtd_read_oob(mtd, ofs, &ops); 1091 if (ret) 1092 return ret; 1093 1094 if (likely(nand->badblockbits == 8)) 1095 ret = bad != 0xFF; 1096 else 1097 ret = hweight8(bad) < nand->badblockbits; 1098 1099 i++; 1100 ofs += mtd->writesize; 1101 } while (!ret && (nand->bbt_options & NAND_BBT_SCAN2NDPAGE) && i < 2); 1102 1103 return ret; 1104} 1105 1106static void mt7621_nfc_init_chip(struct mt7621_nfc *nfc) 1107{ 1108 struct nand_chip *nand = &nfc->nand; 1109 struct mtd_info *mtd; 1110 int ret; 1111 1112 nand_set_controller_data(nand, nfc); 1113 1114 nand->options |= NAND_NO_SUBPAGE_WRITE; 1115 1116 nand->ecc.mode = NAND_ECC_HW_SYNDROME; 1117 nand->ecc.read_page = mt7621_nfc_read_page_hwecc; 1118 nand->ecc.read_page_raw = mt7621_nfc_read_page_raw; 1119 nand->ecc.write_page = mt7621_nfc_write_page_hwecc; 1120 nand->ecc.write_page_raw = mt7621_nfc_write_page_raw; 1121 nand->ecc.read_oob = mt7621_nfc_read_oob_hwecc; 1122 nand->ecc.read_oob_raw = mt7621_nfc_read_oob_raw; 1123 nand->ecc.write_oob = mt7621_nfc_write_oob_hwecc; 1124 nand->ecc.write_oob_raw = mt7621_nfc_write_oob_raw; 1125 1126 nand->dev_ready = mt7621_nfc_dev_ready; 1127 nand->select_chip = mt7621_nfc_select_chip; 1128 nand->write_byte = mt7621_nfc_write_byte; 1129 nand->write_buf = mt7621_nfc_write_buf; 1130 nand->read_byte = mt7621_nfc_read_byte; 1131 nand->read_buf = mt7621_nfc_read_buf; 1132 nand->cmd_ctrl = mt7621_nfc_cmd_ctrl; 1133 nand->block_bad = mt7621_nfc_block_bad; 1134 1135 mtd = nand_to_mtd(nand); 1136 mtd_set_ooblayout(mtd, &mt7621_nfc_ooblayout_ops); 1137 1138 /* Reset NFI master */ 1139 mt7621_nfc_hw_init(nfc); 1140 1141 ret = nand_scan_ident(mtd, 1, NULL); 1142 if (ret) 1143 return; 1144 1145 mt7621_nfc_attach_chip(nand); 1146 1147 ret = nand_scan_tail(mtd); 1148 if (ret) 1149 return; 1150 1151 nand_register(0, mtd); 1152} 1153 1154static void mt7621_nfc_set_regs(struct mt7621_nfc *nfc) 1155{ 1156 nfc->nfi_regs = (void __iomem *)CKSEG1ADDR(NFI_BASE); 1157 nfc->ecc_regs = (void __iomem *)CKSEG1ADDR(NFI_ECC_BASE); 1158} 1159 1160void mt7621_nfc_spl_init(struct mt7621_nfc *nfc) 1161{ 1162 struct nand_chip *nand = &nfc->nand; 1163 1164 mt7621_nfc_set_regs(nfc); 1165 1166 nand_set_controller_data(nand, nfc); 1167 1168 nand->options |= NAND_NO_SUBPAGE_WRITE; 1169 1170 nand->ecc.mode = NAND_ECC_HW_SYNDROME; 1171 nand->ecc.read_page = mt7621_nfc_read_page_hwecc; 1172 1173 nand->dev_ready = mt7621_nfc_dev_ready; 1174 nand->select_chip = mt7621_nfc_select_chip; 1175 nand->read_byte = mt7621_nfc_read_byte; 1176 nand->read_buf = mt7621_nfc_read_buf; 1177 nand->cmd_ctrl = mt7621_nfc_cmd_ctrl; 1178 1179 /* Reset NFI master */ 1180 mt7621_nfc_hw_init(nfc); 1181} 1182 1183int mt7621_nfc_spl_post_init(struct mt7621_nfc *nfc) 1184{ 1185 struct nand_chip *nand = &nfc->nand; 1186 int nand_maf_id, nand_dev_id; 1187 int ret; 1188 1189 ret = nand_detect(nand, &nand_maf_id, &nand_dev_id, NULL); 1190 1191 if (ret) 1192 return ret; 1193 1194 nand->numchips = 1; 1195 nand->mtd.size = nand->chipsize; 1196 1197 return mt7621_nfc_attach_chip(nand); 1198} 1199 1200void board_nand_init(void) 1201{ 1202 mt7621_nfc_set_regs(&nfc_dev); 1203 mt7621_nfc_init_chip(&nfc_dev); 1204} 1205