667 lines
20 KiB
C
667 lines
20 KiB
C
// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
|
|
/*
|
|
* Copyright (c) 2019 Amlogic, Inc. All rights reserved.
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/module.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/err.h>
|
|
#include <linux/clk.h>
|
|
#include <linux/io.h>
|
|
#include <linux/hw_random.h>
|
|
#include <linux/platform_device.h>
|
|
|
|
#include <linux/device.h>
|
|
#include <linux/init.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/irq.h>
|
|
#include <linux/scatterlist.h>
|
|
#include <linux/dma-mapping.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/crypto.h>
|
|
#include <linux/amlogic/arm-smccc.h>
|
|
#include <crypto/scatterwalk.h>
|
|
#include <crypto/algapi.h>
|
|
#include <crypto/internal/hash.h>
|
|
#include "aml-crypto-dma.h"
|
|
|
|
#define AML_CRYPTO_DMA_DBG (0)
|
|
|
|
u32 swap_ulong32(u32 val)
|
|
{
|
|
u32 res = 0;
|
|
|
|
res = ((val & 0xff) << 24) + (((val >> 8) & 0xff) << 16) +
|
|
(((val >> 16) & 0xff) << 8) + ((val >> 24) & 0xff);
|
|
return res;
|
|
}
|
|
EXPORT_SYMBOL_GPL(swap_ulong32);
|
|
|
|
void aml_write_crypto_reg(struct aml_dma_dev *dd, u32 addr, u64 data)
|
|
{
|
|
if (dd && cryptoreg) {
|
|
u32 low = (u32)(0xffffffff & data);
|
|
|
|
/* If 64 bits mode and wants to write thread reg */
|
|
if (dd->dma_bus64 && addr == dd->thread) {
|
|
u32 high = (u32)(0xff & (data >> 32));
|
|
|
|
writel(high, cryptoreg + (addr << 2) + 0x80);
|
|
}
|
|
writel(low, cryptoreg + (addr << 2));
|
|
} else {
|
|
pr_err("crypto reg mapping is not initialized\n");
|
|
}
|
|
}
|
|
EXPORT_SYMBOL_GPL(aml_write_crypto_reg);
|
|
|
|
u32 aml_read_crypto_reg(u32 addr)
|
|
{
|
|
if (!cryptoreg) {
|
|
pr_err("crypto reg mapping is not initialized\n");
|
|
return 0;
|
|
}
|
|
return readl(cryptoreg + (addr << 2));
|
|
}
|
|
EXPORT_SYMBOL_GPL(aml_read_crypto_reg);
|
|
|
|
void aml_dma_debug_v1(void *descriptor, u32 nents, const char *msg,
|
|
u32 thread, u32 status, u8 dma_bus64)
|
|
{
|
|
u32 i = 0;
|
|
struct dma_dsc *dsc = descriptor;
|
|
struct dma_dsc_64 *dsc_64 = descriptor;
|
|
|
|
dbgp(0, "begin %s\n", msg);
|
|
dbgp(0, "reg(%u) = 0x%8x\n", thread,
|
|
aml_read_crypto_reg(thread));
|
|
dbgp(0, "reg(%u) = 0x%8x\n", status,
|
|
aml_read_crypto_reg(status));
|
|
dbgp(0, "bus64 = %u\n", dma_bus64);
|
|
if (dma_bus64) {
|
|
for (i = 0; i < nents; i++) {
|
|
dbgp(0, "desc (%4x) (len) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.length);
|
|
dbgp(0, "desc (%4x) (irq) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.irq);
|
|
dbgp(0, "desc (%4x) (eoc) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.eoc);
|
|
dbgp(0, "desc (%4x) (lop) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.loop);
|
|
dbgp(0, "desc (%4x) (mod) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.mode);
|
|
dbgp(0, "desc (%4x) (beg) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.begin);
|
|
dbgp(0, "desc (%4x) (end) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.end);
|
|
dbgp(0, "desc (%4x) (opm) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.op_mode);
|
|
dbgp(0, "desc (%4x) (enc) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.enc_sha_only);
|
|
dbgp(0, "desc (%4x) (blk) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.block);
|
|
dbgp(0, "desc (%4x) (lnk_err) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.link_error);
|
|
dbgp(0, "desc (%4x) (own) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.owner);
|
|
dbgp(0, "desc (%4x) (src) = 0x%16llx\n", i,
|
|
dsc_64[i].src_addr);
|
|
dbgp(0, "desc (%4x) (tgt) = 0x%16llx\n", i,
|
|
dsc_64[i].tgt_addr);
|
|
}
|
|
} else {
|
|
for (i = 0; i < nents; i++) {
|
|
dbgp(0, "desc (%4x) (len) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.length);
|
|
dbgp(0, "desc (%4x) (irq) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.irq);
|
|
dbgp(0, "desc (%4x) (eoc) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.eoc);
|
|
dbgp(0, "desc (%4x) (lop) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.loop);
|
|
dbgp(0, "desc (%4x) (mod) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.mode);
|
|
dbgp(0, "desc (%4x) (beg) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.begin);
|
|
dbgp(0, "desc (%4x) (end) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.end);
|
|
dbgp(0, "desc (%4x) (opm) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.op_mode);
|
|
dbgp(0, "desc (%4x) (enc) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.enc_sha_only);
|
|
dbgp(0, "desc (%4x) (blk) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.block);
|
|
dbgp(0, "desc (%4x) (lnk_err) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.link_error);
|
|
dbgp(0, "desc (%4x) (own) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.owner);
|
|
dbgp(0, "desc (%4x) (src) = 0x%8x\n", i,
|
|
dsc[i].src_addr);
|
|
dbgp(0, "desc (%4x) (tgt) = 0x%8x\n", i,
|
|
dsc[i].tgt_addr);
|
|
}
|
|
}
|
|
dbgp(0, "end %s\n", msg);
|
|
}
|
|
|
|
void aml_dma_debug_v2(void *descriptor, u32 nents, const char *msg,
|
|
u32 thread, u32 status, u8 dma_bus64)
|
|
{
|
|
u32 i = 0;
|
|
struct dma_dsc_v2 *dsc = descriptor;
|
|
struct dma_dsc_v2_64 *dsc_64 = descriptor;
|
|
|
|
dbgp(0, "begin %s\n", msg);
|
|
dbgp(0, "reg(%u) = 0x%8x\n", thread,
|
|
aml_read_crypto_reg(thread));
|
|
dbgp(0, "reg(%u) = 0x%8x\n", status,
|
|
aml_read_crypto_reg(status));
|
|
dbgp(0, "bus64 = %u\n", dma_bus64);
|
|
if (dma_bus64) {
|
|
for (i = 0; i < nents; i++) {
|
|
dbgp(0, "desc (%4x) (len) = 0x%8x\n", i,
|
|
dsc_64[i].length);
|
|
dbgp(0, "desc (%4x) (skp) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.skip_block_count);
|
|
dbgp(0, "desc (%4x) (crp) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.crypt_block_count);
|
|
dbgp(0, "desc (%4x) (irq) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.irq);
|
|
dbgp(0, "desc (%4x) (eoc) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.eoc);
|
|
dbgp(0, "desc (%4x) (lop) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.loop);
|
|
dbgp(0, "desc (%4x) (mod) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.mode);
|
|
dbgp(0, "desc (%4x) (beg) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.begin);
|
|
dbgp(0, "desc (%4x) (end) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.end);
|
|
dbgp(0, "desc (%4x) (opm) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.op_mode);
|
|
dbgp(0, "desc (%4x) (enc) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.enc_sha_only);
|
|
dbgp(0, "desc (%4x) (lnk_err) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.link_error);
|
|
dbgp(0, "desc (%4x) (own) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.owner);
|
|
dbgp(0, "desc (%4x) (src) = 0x%16llx\n", i,
|
|
dsc_64[i].src_addr);
|
|
dbgp(0, "desc (%4x) (tgt) = 0x%16llx\n", i,
|
|
dsc_64[i].tgt_addr);
|
|
}
|
|
} else {
|
|
for (i = 0; i < nents; i++) {
|
|
dbgp(0, "desc (%4x) (len) = 0x%8x\n", i,
|
|
dsc[i].length);
|
|
dbgp(0, "desc (%4x) (skp) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.skip_block_count);
|
|
dbgp(0, "desc (%4x) (crp) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.crypt_block_count);
|
|
dbgp(0, "desc (%4x) (irq) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.irq);
|
|
dbgp(0, "desc (%4x) (eoc) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.eoc);
|
|
dbgp(0, "desc (%4x) (lop) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.loop);
|
|
dbgp(0, "desc (%4x) (mod) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.mode);
|
|
dbgp(0, "desc (%4x) (beg) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.begin);
|
|
dbgp(0, "desc (%4x) (end) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.end);
|
|
dbgp(0, "desc (%4x) (opm) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.op_mode);
|
|
dbgp(0, "desc (%4x) (enc) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.enc_sha_only);
|
|
dbgp(0, "desc (%4x) (lnk_err) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.link_error);
|
|
dbgp(0, "desc (%4x) (own) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.owner);
|
|
dbgp(0, "desc (%4x) (src) = 0x%8x\n", i,
|
|
dsc[i].src_addr);
|
|
dbgp(0, "desc (%4x) (tgt) = 0x%8x\n", i,
|
|
dsc[i].tgt_addr);
|
|
}
|
|
}
|
|
dbgp(0, "end %s\n", msg);
|
|
}
|
|
|
|
void aml_dma_debug(void *descriptor, u32 nents, const char *msg,
|
|
u32 thread, u32 status, struct aml_dma_dev *dma)
|
|
{
|
|
if (dma->dma_version == DMA_VERSION_1)
|
|
return aml_dma_debug_v1(descriptor, nents, msg, thread, status, dma->dma_bus64);
|
|
else if (dma->dma_version == DMA_VERSION_2)
|
|
return aml_dma_debug_v2(descriptor, nents, msg, thread, status, dma->dma_bus64);
|
|
else
|
|
return;
|
|
}
|
|
EXPORT_SYMBOL_GPL(aml_dma_debug);
|
|
|
|
void aml_dma_link_debug(void *descriptor, dma_addr_t dma_dsc,
|
|
u32 nents, const char *msg, u8 dma_bus64)
|
|
{
|
|
struct dma_sg_dsc *dsc = descriptor;
|
|
struct dma_sg_dsc_64 *dsc_64 = descriptor;
|
|
u32 i = 0;
|
|
|
|
dbgp(0, "link begin %s\n", msg);
|
|
dbgp(0, "link dma_addr 0x%lx\n", (unsigned long)dma_dsc);
|
|
if (dma_bus64) {
|
|
for (i = 0; i < nents; i++) {
|
|
dbgp(0, "sg desc (%4x) (vld) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.valid);
|
|
dbgp(0, "sg desc (%4x) (eoc) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.eoc);
|
|
dbgp(0, "sg desc (%4x) (int) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.intr);
|
|
dbgp(0, "sg desc (%4x) (act) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.act);
|
|
dbgp(0, "sg desc (%4x) (len) = 0x%8x\n", i,
|
|
dsc_64[i].dsc_cfg.b.length);
|
|
dbgp(0, "sg desc (%4x) (adr) = 0x%16llx\n", i,
|
|
dsc_64[i].addr);
|
|
}
|
|
} else {
|
|
for (i = 0; i < nents; i++) {
|
|
dbgp(0, "sg desc (%4x) (vld) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.valid);
|
|
dbgp(0, "sg desc (%4x) (eoc) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.eoc);
|
|
dbgp(0, "sg desc (%4x) (int) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.intr);
|
|
dbgp(0, "sg desc (%4x) (act) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.act);
|
|
dbgp(0, "sg desc (%4x) (len) = 0x%8x\n", i,
|
|
dsc[i].dsc_cfg.b.length);
|
|
dbgp(0, "sg desc (%4x) (adr) = 0x%8x\n", i,
|
|
dsc[i].addr);
|
|
}
|
|
}
|
|
dbgp(0, "link end %s\n", msg);
|
|
}
|
|
EXPORT_SYMBOL_GPL(aml_dma_link_debug);
|
|
|
|
irqreturn_t aml_crypto_dma_irq(int irq, void *dev_id)
|
|
{
|
|
struct aml_dma_dev *dd = dev_id;
|
|
struct device *dev = dd->dev;
|
|
u32 status = aml_read_crypto_reg(dd->status);
|
|
|
|
if (status) {
|
|
if (status == 0x1)
|
|
dev_err(dev, "irq overwrite\n");
|
|
if (dd->dma_busy) {
|
|
complete(&dd->done);
|
|
return IRQ_HANDLED;
|
|
} else {
|
|
return IRQ_NONE;
|
|
}
|
|
}
|
|
|
|
return IRQ_NONE;
|
|
}
|
|
|
|
u32 aml_dma_do_hw_crypto(struct aml_dma_dev *dd,
|
|
void *descriptor,
|
|
u32 dsc_len,
|
|
dma_addr_t dsc_addr,
|
|
u8 polling, u8 dma_flags)
|
|
{
|
|
u32 status = 0;
|
|
struct dma_dsc_v2_64 dsc;
|
|
|
|
spin_lock_irqsave(&dd->dma_lock, dd->irq_flags);
|
|
dd->dma_busy |= dma_flags;
|
|
reinit_completion(&dd->done);
|
|
spin_unlock_irqrestore(&dd->dma_lock, dd->irq_flags);
|
|
aml_write_crypto_reg(dd, dd->status, 0xffff);
|
|
aml_write_crypto_reg(dd, dd->thread, (uintptr_t)dsc_addr | 2);
|
|
if (polling) {
|
|
while (aml_read_crypto_reg(dd->status) == 0)
|
|
;
|
|
} else {
|
|
wait_for_completion(&dd->done);
|
|
}
|
|
status = aml_read_crypto_reg(dd->status);
|
|
if (status & DMA_STATUS_KEY_ERROR) {
|
|
/* If error occurs, report it and don't wait owner bit */
|
|
pr_err("Error returned by DMA(0x%8x)\n", status);
|
|
} else {
|
|
aml_dma_dsc_reader(descriptor, dsc_len - 1, &dsc, 0, dd);
|
|
WARN_ON(!dsc.dsc_cfg.b.eoc);
|
|
if (dsc.dsc_cfg.b.eoc == 1) {
|
|
while (dsc.dsc_cfg.b.owner != 0) {
|
|
cpu_relax();
|
|
aml_dma_dsc_reader(descriptor, dsc_len - 1, &dsc, 0, dd);
|
|
}
|
|
}
|
|
}
|
|
spin_lock_irqsave(&dd->dma_lock, dd->irq_flags);
|
|
dd->dma_busy &= ~dma_flags;
|
|
spin_unlock_irqrestore(&dd->dma_lock, dd->irq_flags);
|
|
|
|
return status;
|
|
}
|
|
EXPORT_SYMBOL_GPL(aml_dma_do_hw_crypto);
|
|
|
|
void aml_dma_finish_hw_crypto(struct aml_dma_dev *dd, u8 dma_flags)
|
|
{
|
|
spin_lock_irqsave(&dd->dma_lock, dd->irq_flags);
|
|
aml_write_crypto_reg(dd, dd->status, 0xffff);
|
|
dd->dma_busy &= ~dma_flags;
|
|
spin_unlock_irqrestore(&dd->dma_lock, dd->irq_flags);
|
|
}
|
|
EXPORT_SYMBOL_GPL(aml_dma_finish_hw_crypto);
|
|
|
|
int aml_dma_crypto_enqueue_req(struct aml_dma_dev *dd,
|
|
struct crypto_async_request *req)
|
|
{
|
|
s32 ret = 0;
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(&dd->queue_lock, flags);
|
|
ret = crypto_enqueue_request(&dd->queue, req);
|
|
spin_unlock_irqrestore(&dd->queue_lock, flags);
|
|
wake_up_process(dd->kthread);
|
|
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL_GPL(aml_dma_crypto_enqueue_req);
|
|
|
|
noinline int aml_dma_call_smc(u64 func_id, u64 arg0, u64 arg1, u64 arg2)
|
|
{
|
|
struct arm_smccc_res res;
|
|
|
|
arm_smccc_smc((unsigned long)func_id,
|
|
(unsigned long)arg0,
|
|
(unsigned long)arg1,
|
|
(unsigned long)arg2,
|
|
0, 0, 0, 0, &res);
|
|
return res.a0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(aml_dma_call_smc);
|
|
|
|
inline void aml_dma_dsc_writer_v1(void *base, u32 i, void *dsc,
|
|
u8 dma_bus64, u8 is_sg_dsc)
|
|
{
|
|
u8 use_block_mode = 0;
|
|
|
|
if (dma_bus64) {
|
|
if (!is_sg_dsc) {
|
|
struct dma_dsc_64 *dsc_base = base;
|
|
struct dma_dsc_v2_64 *dsc_to_write = dsc;
|
|
|
|
if (!(dsc_to_write->length % DMA_BLOCK_MODE_SIZE))
|
|
use_block_mode = 1;
|
|
|
|
dsc_base[i].src_addr = dsc_to_write->src_addr;
|
|
dsc_base[i].tgt_addr = dsc_to_write->tgt_addr;
|
|
dsc_base[i].dsc_cfg.d32 = 0;
|
|
dsc_base[i].dsc_cfg.b.irq = dsc_to_write->dsc_cfg.b.irq;
|
|
dsc_base[i].dsc_cfg.b.eoc = dsc_to_write->dsc_cfg.b.eoc;
|
|
dsc_base[i].dsc_cfg.b.loop = dsc_to_write->dsc_cfg.b.loop;
|
|
dsc_base[i].dsc_cfg.b.mode = dsc_to_write->dsc_cfg.b.mode;
|
|
dsc_base[i].dsc_cfg.b.begin = dsc_to_write->dsc_cfg.b.begin;
|
|
dsc_base[i].dsc_cfg.b.end = dsc_to_write->dsc_cfg.b.end;
|
|
dsc_base[i].dsc_cfg.b.op_mode = dsc_to_write->dsc_cfg.b.op_mode;
|
|
dsc_base[i].dsc_cfg.b.enc_sha_only = dsc_to_write->dsc_cfg.b.enc_sha_only;
|
|
if (use_block_mode) {
|
|
dsc_base[i].dsc_cfg.b.length =
|
|
dsc_to_write->length / DMA_BLOCK_MODE_SIZE;
|
|
dsc_base[i].dsc_cfg.b.block = 1;
|
|
} else {
|
|
dsc_base[i].dsc_cfg.b.length = dsc_to_write->length;
|
|
WARN_ON(dsc_base[i].dsc_cfg.b.length != dsc_to_write->length);
|
|
}
|
|
dsc_base[i].dsc_cfg.b.link_error = dsc_to_write->dsc_cfg.b.link_error;
|
|
dsc_base[i].dsc_cfg.b.owner = dsc_to_write->dsc_cfg.b.owner;
|
|
} else {
|
|
struct dma_sg_dsc_64 *dsc_base = base;
|
|
struct dma_sg_dsc_64 *dsc_to_write = dsc;
|
|
|
|
dsc_base[i] = *dsc_to_write;
|
|
}
|
|
} else {
|
|
if (!is_sg_dsc) {
|
|
struct dma_dsc *dsc_base = base;
|
|
struct dma_dsc_v2_64 *dsc_to_write = dsc;
|
|
|
|
if (!(dsc_to_write->length % DMA_BLOCK_MODE_SIZE))
|
|
use_block_mode = 1;
|
|
|
|
dsc_base[i].src_addr = dsc_to_write->src_addr;
|
|
dsc_base[i].tgt_addr = dsc_to_write->tgt_addr;
|
|
dsc_base[i].dsc_cfg.d32 = 0;
|
|
dsc_base[i].dsc_cfg.b.length = dsc_to_write->length;
|
|
dsc_base[i].dsc_cfg.b.irq = dsc_to_write->dsc_cfg.b.irq;
|
|
dsc_base[i].dsc_cfg.b.eoc = dsc_to_write->dsc_cfg.b.eoc;
|
|
dsc_base[i].dsc_cfg.b.loop = dsc_to_write->dsc_cfg.b.loop;
|
|
dsc_base[i].dsc_cfg.b.mode = dsc_to_write->dsc_cfg.b.mode;
|
|
dsc_base[i].dsc_cfg.b.begin = dsc_to_write->dsc_cfg.b.begin;
|
|
dsc_base[i].dsc_cfg.b.end = dsc_to_write->dsc_cfg.b.end;
|
|
dsc_base[i].dsc_cfg.b.op_mode = dsc_to_write->dsc_cfg.b.op_mode;
|
|
dsc_base[i].dsc_cfg.b.enc_sha_only = dsc_to_write->dsc_cfg.b.enc_sha_only;
|
|
if (use_block_mode) {
|
|
dsc_base[i].dsc_cfg.b.length =
|
|
dsc_to_write->length / DMA_BLOCK_MODE_SIZE;
|
|
dsc_base[i].dsc_cfg.b.block = 1;
|
|
} else {
|
|
dsc_base[i].dsc_cfg.b.length = dsc_to_write->length;
|
|
}
|
|
dsc_base[i].dsc_cfg.b.link_error = dsc_to_write->dsc_cfg.b.link_error;
|
|
dsc_base[i].dsc_cfg.b.owner = dsc_to_write->dsc_cfg.b.owner;
|
|
} else {
|
|
struct dma_sg_dsc *dsc_base = base;
|
|
struct dma_sg_dsc_64 *dsc_to_write = dsc;
|
|
|
|
dsc_base[i].dsc_cfg = dsc_to_write->dsc_cfg;
|
|
dsc_base[i].addr = dsc_to_write->addr;
|
|
}
|
|
}
|
|
}
|
|
|
|
inline void aml_dma_dsc_writer_v2(void *base, u32 i, void *dsc,
|
|
u8 dma_bus64, u8 is_sg_dsc)
|
|
{
|
|
if (dma_bus64) {
|
|
if (!is_sg_dsc) {
|
|
struct dma_dsc_v2_64 *dsc_base = base;
|
|
struct dma_dsc_v2_64 *dsc_to_write = dsc;
|
|
|
|
dsc_base[i] = *dsc_to_write;
|
|
} else {
|
|
struct dma_sg_dsc_64 *dsc_base = base;
|
|
struct dma_sg_dsc_64 *dsc_to_write = dsc;
|
|
|
|
dsc_base[i] = *dsc_to_write;
|
|
}
|
|
} else {
|
|
if (!is_sg_dsc) {
|
|
struct dma_dsc_v2 *dsc_base = base;
|
|
struct dma_dsc_v2_64 *dsc_to_write = dsc;
|
|
|
|
dsc_base[i].dsc_cfg = dsc_to_write->dsc_cfg;
|
|
dsc_base[i].src_addr = dsc_to_write->src_addr;
|
|
dsc_base[i].tgt_addr = dsc_to_write->tgt_addr;
|
|
} else {
|
|
struct dma_sg_dsc *dsc_base = base;
|
|
struct dma_sg_dsc_64 *dsc_to_write = dsc;
|
|
|
|
dsc_base[i].dsc_cfg = dsc_to_write->dsc_cfg;
|
|
dsc_base[i].addr = dsc_to_write->addr;
|
|
}
|
|
}
|
|
}
|
|
|
|
inline void aml_dma_dsc_writer(void *base, u32 i, void *dsc, u8 is_sg_dsc, struct aml_dma_dev *dma)
|
|
{
|
|
if (dma->dma_version == DMA_VERSION_1)
|
|
return aml_dma_dsc_writer_v1(base, i, dsc, dma->dma_bus64, is_sg_dsc);
|
|
else if (dma->dma_version == DMA_VERSION_2)
|
|
return aml_dma_dsc_writer_v2(base, i, dsc, dma->dma_bus64, is_sg_dsc);
|
|
else
|
|
return;
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(aml_dma_dsc_writer);
|
|
|
|
inline void aml_dma_dsc_reader_v1(void *base, u32 i, void *dsc,
|
|
u8 dma_bus64, u8 is_sg_dsc)
|
|
{
|
|
if (dma_bus64) {
|
|
if (!is_sg_dsc) {
|
|
struct dma_dsc_64 *dsc_base = base;
|
|
struct dma_dsc_v2_64 *dsc_read = dsc;
|
|
|
|
if (dsc_base[i].dsc_cfg.b.block)
|
|
dsc_read->length =
|
|
dsc_base[i].dsc_cfg.b.length * DMA_BLOCK_MODE_SIZE;
|
|
else
|
|
dsc_read->length = dsc_base[i].dsc_cfg.b.length;
|
|
dsc_read->src_addr = dsc_base[i].src_addr;
|
|
dsc_read->tgt_addr = dsc_base[i].tgt_addr;
|
|
dsc_read->dsc_cfg.d32 = 0;
|
|
dsc_read->dsc_cfg.b.irq = dsc_base[i].dsc_cfg.b.irq;
|
|
dsc_read->dsc_cfg.b.eoc = dsc_base[i].dsc_cfg.b.eoc;
|
|
dsc_read->dsc_cfg.b.loop = dsc_base[i].dsc_cfg.b.loop;
|
|
dsc_read->dsc_cfg.b.mode = dsc_base[i].dsc_cfg.b.mode;
|
|
dsc_read->dsc_cfg.b.begin = dsc_base[i].dsc_cfg.b.begin;
|
|
dsc_read->dsc_cfg.b.end = dsc_base[i].dsc_cfg.b.end;
|
|
dsc_read->dsc_cfg.b.op_mode = dsc_base[i].dsc_cfg.b.op_mode;
|
|
dsc_read->dsc_cfg.b.enc_sha_only = dsc_base[i].dsc_cfg.b.enc_sha_only;
|
|
dsc_read->dsc_cfg.b.link_error = dsc_base[i].dsc_cfg.b.link_error;
|
|
dsc_read->dsc_cfg.b.owner = dsc_base[i].dsc_cfg.b.owner;
|
|
} else {
|
|
struct dma_sg_dsc_64 *dsc_base = base;
|
|
struct dma_sg_dsc_64 *dsc_read = dsc;
|
|
|
|
*dsc_read = dsc_base[i];
|
|
}
|
|
} else {
|
|
if (!is_sg_dsc) {
|
|
struct dma_dsc *dsc_base = base;
|
|
struct dma_dsc_v2_64 *dsc_read = dsc;
|
|
|
|
if (dsc_base[i].dsc_cfg.b.block)
|
|
dsc_read->length =
|
|
dsc_base[i].dsc_cfg.b.length * DMA_BLOCK_MODE_SIZE;
|
|
else
|
|
dsc_read->length = dsc_base[i].dsc_cfg.b.length;
|
|
dsc_read->src_addr = dsc_base[i].src_addr;
|
|
dsc_read->tgt_addr = dsc_base[i].tgt_addr;
|
|
dsc_read->dsc_cfg.d32 = 0;
|
|
dsc_read->dsc_cfg.b.irq = dsc_base[i].dsc_cfg.b.irq;
|
|
dsc_read->dsc_cfg.b.eoc = dsc_base[i].dsc_cfg.b.eoc;
|
|
dsc_read->dsc_cfg.b.loop = dsc_base[i].dsc_cfg.b.loop;
|
|
dsc_read->dsc_cfg.b.mode = dsc_base[i].dsc_cfg.b.mode;
|
|
dsc_read->dsc_cfg.b.begin = dsc_base[i].dsc_cfg.b.begin;
|
|
dsc_read->dsc_cfg.b.end = dsc_base[i].dsc_cfg.b.end;
|
|
dsc_read->dsc_cfg.b.op_mode = dsc_base[i].dsc_cfg.b.op_mode;
|
|
dsc_read->dsc_cfg.b.enc_sha_only = dsc_base[i].dsc_cfg.b.enc_sha_only;
|
|
dsc_read->dsc_cfg.b.link_error = dsc_base[i].dsc_cfg.b.link_error;
|
|
dsc_read->dsc_cfg.b.owner = dsc_base[i].dsc_cfg.b.owner;
|
|
} else {
|
|
struct dma_sg_dsc *dsc_base = base;
|
|
struct dma_sg_dsc_64 *dsc_read = dsc;
|
|
|
|
dsc_read->dsc_cfg = dsc_base[i].dsc_cfg;
|
|
dsc_read->addr = dsc_base[i].addr;
|
|
}
|
|
}
|
|
}
|
|
|
|
inline void aml_dma_dsc_reader_v2(void *base, u32 i, void *dsc,
|
|
u8 dma_bus64, u8 is_sg_dsc)
|
|
{
|
|
if (dma_bus64) {
|
|
if (!is_sg_dsc) {
|
|
struct dma_dsc_v2_64 *dsc_base = base;
|
|
struct dma_dsc_v2_64 *dsc_read = dsc;
|
|
|
|
*dsc_read = dsc_base[i];
|
|
} else {
|
|
struct dma_sg_dsc_64 *dsc_base = base;
|
|
struct dma_sg_dsc_64 *dsc_read = dsc;
|
|
|
|
*dsc_read = dsc_base[i];
|
|
}
|
|
} else {
|
|
if (!is_sg_dsc) {
|
|
struct dma_dsc_v2 *dsc_base = base;
|
|
struct dma_dsc_v2_64 *dsc_read = dsc;
|
|
|
|
dsc_read->dsc_cfg = dsc_base[i].dsc_cfg;
|
|
dsc_read->src_addr = dsc_base[i].src_addr;
|
|
dsc_read->tgt_addr = dsc_base[i].tgt_addr;
|
|
} else {
|
|
struct dma_sg_dsc *dsc_base = base;
|
|
struct dma_sg_dsc_64 *dsc_read = dsc;
|
|
|
|
dsc_read->dsc_cfg = dsc_base[i].dsc_cfg;
|
|
dsc_read->addr = dsc_base[i].addr;
|
|
}
|
|
}
|
|
}
|
|
|
|
inline void aml_dma_dsc_reader(void *base, u32 i, void *dsc, u8 is_sg_dsc, struct aml_dma_dev *dma)
|
|
{
|
|
if (dma->dma_version == DMA_VERSION_1)
|
|
return aml_dma_dsc_reader_v1(base, i, dsc, dma->dma_bus64, is_sg_dsc);
|
|
else if (dma->dma_version == DMA_VERSION_2)
|
|
return aml_dma_dsc_reader_v2(base, i, dsc, dma->dma_bus64, is_sg_dsc);
|
|
else
|
|
return;
|
|
}
|
|
EXPORT_SYMBOL_GPL(aml_dma_dsc_reader);
|
|
|
|
inline u32 aml_dma_get_dsc_sz(u8 is_sg_dsc, struct aml_dma_dev *dma)
|
|
{
|
|
if (dma->dma_version == DMA_VERSION_1) {
|
|
if (!is_sg_dsc)
|
|
return dma->dma_bus64 ? sizeof(struct dma_dsc_64) :
|
|
sizeof(struct dma_dsc);
|
|
else
|
|
return dma->dma_bus64 ? sizeof(struct dma_sg_dsc_64) :
|
|
sizeof(struct dma_sg_dsc);
|
|
} else if (dma->dma_version == DMA_VERSION_2) {
|
|
if (!is_sg_dsc)
|
|
return dma->dma_bus64 ? sizeof(struct dma_dsc_v2_64) :
|
|
sizeof(struct dma_dsc_v2);
|
|
else
|
|
return dma->dma_bus64 ? sizeof(struct dma_sg_dsc_64) :
|
|
sizeof(struct dma_sg_dsc);
|
|
} else {
|
|
return 0;
|
|
}
|
|
}
|
|
EXPORT_SYMBOL_GPL(aml_dma_get_dsc_sz);
|
|
|
|
#if AML_CRYPTO_DMA_DBG
|
|
void dump_buf(const s8 *name, u8 *buf, const s32 length)
|
|
{
|
|
s32 i = 0;
|
|
s32 j = length;
|
|
|
|
pr_err("\nstart dump(%s, %d)================\n", name, length);
|
|
while (j >= 8) {
|
|
pr_err("%#02x ~ %#02x:\n", i, i + 7);
|
|
pr_err("%#02x, %#02x, %#02x, %#02x, %#02x, %#02x, %#02x, %#02x\n",
|
|
buf[i], buf[i + 1], buf[i + 2], buf[i + 3],
|
|
buf[i + 4], buf[i + 5], buf[i + 6], buf[i + 7]);
|
|
i += 8;
|
|
j -= 8;
|
|
};
|
|
|
|
while (j > 0) {
|
|
pr_err("%#02x ", buf[i]);
|
|
j--;
|
|
i++;
|
|
};
|
|
pr_err("\nend dump(%s, %d)================\n", name, length);
|
|
}
|
|
#endif
|