raspi/linux
1
0
Fork 1
mirror of https://github.com/raspberrypi/linux.git synced 2025-12-06 01:49:46 +00:00
Code Issues Projects Releases Wiki Activity

drivers: pcie: Hailo: Remove Hailo PCIe driver

Browse Source 
This driver will be installed through DKMS going forward.

Signed-off-by: Naushir Patuck <naush@raspberrypi.com>
This commit is contained in:
Naushir Patuck
2025-09-30 09:58:09 +01:00
committed by Dom Cobley
parent cbe9e7a96e
commit 6677154cb2
42 changed files with 1 additions and 9097 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/media/pci/Kconfig
View File

@@ -74,7 +74,6 @@ config VIDEO_PCI_SKELETON
when developing new drivers.
source "drivers/media/pci/intel/Kconfig"
source "drivers/media/pci/hailo/Kconfig"
endif #MEDIA_PCI_SUPPORT
endif #PCI

3
drivers/media/pci/Makefile
View File

@@ -17,8 +17,7 @@ obj-y += ttpci/ \
saa7146/ \
smipcie/ \
netup_unidvb/ \
intel/ \
hailo/
intel/
# Please keep it alphabetically sorted by Kconfig name
# (e. g. LC_ALL=C sort Makefile)

6
drivers/media/pci/hailo/Kconfig
View File

@@ -1,6 +0,0 @@
config MEDIA_PCI_HAILO
tristate "Hailo AI accelerator PCIe driver"
depends on PCI
help
Enable build of Hailo AI accelerator PCIe driver.

34
drivers/media/pci/hailo/Makefile
View File

@@ -1,34 +0,0 @@
# SPDX-License-Identifier: GPL-2.0
COMMON_SRC_DIRECTORY=common
VDMA_SRC_DIRECTORY=vdma
UTILS_SRC_DIRECTORY=utils
obj-$(CONFIG_MEDIA_PCI_HAILO) := hailo_pci.o
hailo_pci-objs += src/pcie.o
hailo_pci-objs += src/fops.o
hailo_pci-objs += src/sysfs.o
hailo_pci-objs += src/nnc.o
hailo_pci-objs += src/soc.o
hailo_pci-objs += $(COMMON_SRC_DIRECTORY)/fw_validation.o
hailo_pci-objs += $(COMMON_SRC_DIRECTORY)/fw_operation.o
hailo_pci-objs += $(COMMON_SRC_DIRECTORY)/pcie_common.o
hailo_pci-objs += $(COMMON_SRC_DIRECTORY)/vdma_common.o
hailo_pci-objs += $(COMMON_SRC_DIRECTORY)/hailo_resource.o
hailo_pci-objs += $(UTILS_SRC_DIRECTORY)/logs.o
hailo_pci-objs += $(UTILS_SRC_DIRECTORY)/integrated_nnc_utils.o
hailo_pci-objs += $(VDMA_SRC_DIRECTORY)/vdma.o
hailo_pci-objs += $(VDMA_SRC_DIRECTORY)/memory.o
hailo_pci-objs += $(VDMA_SRC_DIRECTORY)/ioctl.o
ccflags-y += -Werror
ccflags-y += -DHAILO_RASBERRY_PIE
ccflags-y += -I $(src)
ccflags-y += -I $(src)/include
ccflags-y += -I $(src)/common
clean-files := $(hailo_pci-objs)

147
drivers/media/pci/hailo/common/fw_operation.c
View File

@@ -1,147 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#include "fw_operation.h"
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/bug.h>
typedef struct {
u32 host_offset;
u32 chip_offset;
} FW_DEBUG_BUFFER_HEADER_t;
#define DEBUG_BUFFER_DATA_SIZE (DEBUG_BUFFER_TOTAL_SIZE - sizeof(FW_DEBUG_BUFFER_HEADER_t))
#define PCIE_D2H_NOTIFICATION_SRAM_OFFSET (0x640 + 0x640)
#define PCIE_APP_CPU_DEBUG_OFFSET (8*1024)
#define PCIE_CORE_CPU_DEBUG_OFFSET (PCIE_APP_CPU_DEBUG_OFFSET + DEBUG_BUFFER_TOTAL_SIZE)
int hailo_read_firmware_notification(struct hailo_resource *resource, struct hailo_d2h_notification *notification)
{
hailo_d2h_buffer_details_t d2h_buffer_details = {0, 0};
hailo_resource_read_buffer(resource, 0, sizeof(d2h_buffer_details),
&d2h_buffer_details);
if ((sizeof(notification->buffer) < d2h_buffer_details.buffer_len) || (0 == d2h_buffer_details.is_buffer_in_use)) {
return -EINVAL;
}
notification->buffer_len = d2h_buffer_details.buffer_len;
hailo_resource_read_buffer(resource, sizeof(d2h_buffer_details), notification->buffer_len, notification->buffer);
// Write is_buffer_in_use = false
hailo_resource_write16(resource, 0, 0);
return 0;
}
int hailo_pcie_read_firmware_notification(struct hailo_resource *resource,
struct hailo_d2h_notification *notification)
{
struct hailo_resource notification_resource;
if (PCIE_D2H_NOTIFICATION_SRAM_OFFSET > resource->size) {
return -EINVAL;
}
notification_resource.address = resource->address + PCIE_D2H_NOTIFICATION_SRAM_OFFSET,
notification_resource.size = sizeof(struct hailo_d2h_notification);
return hailo_read_firmware_notification(&notification_resource, notification);
}
static inline size_t calculate_log_ready_to_read(FW_DEBUG_BUFFER_HEADER_t *header)
{
size_t ready_to_read = 0;
size_t host_offset = header->host_offset;
size_t chip_offset = header->chip_offset;
if (chip_offset >= host_offset) {
ready_to_read = chip_offset - host_offset;
} else {
ready_to_read = DEBUG_BUFFER_DATA_SIZE - (host_offset - chip_offset);
}
return ready_to_read;
}
long hailo_read_firmware_log(struct hailo_resource *fw_logger_resource, struct hailo_read_log_params *params)
{
FW_DEBUG_BUFFER_HEADER_t debug_buffer_header = {0};
size_t read_offset = 0;
size_t ready_to_read = 0;
size_t size_to_read = 0;
uintptr_t user_buffer = (uintptr_t)params->buffer;
if (params->buffer_size > ARRAY_SIZE(params->buffer)) {
return -EINVAL;
}
hailo_resource_read_buffer(fw_logger_resource, 0, sizeof(debug_buffer_header),
&debug_buffer_header);
/* Point to the start of the data buffer. */
ready_to_read = calculate_log_ready_to_read(&debug_buffer_header);
if (0 == ready_to_read) {
params->read_bytes = 0;
return 0;
}
/* If ready to read is bigger than the buffer size, read only buffer size bytes. */
ready_to_read = min(ready_to_read, params->buffer_size);
/* Point to the data that is read to be read by the host. */
read_offset = sizeof(debug_buffer_header) + debug_buffer_header.host_offset;
/* Check if the offset should cycle back to beginning. */
if (DEBUG_BUFFER_DATA_SIZE <= debug_buffer_header.host_offset + ready_to_read) {
size_to_read = DEBUG_BUFFER_DATA_SIZE - debug_buffer_header.host_offset;
hailo_resource_read_buffer(fw_logger_resource, read_offset, size_to_read, (void*)user_buffer);
user_buffer += size_to_read;
size_to_read = ready_to_read - size_to_read;
/* Point back to the beginning of the data buffer. */
read_offset -= debug_buffer_header.host_offset;
}
else {
size_to_read = ready_to_read;
}
/* size_to_read may become 0 if the read reached DEBUG_BUFFER_DATA_SIZE exactly */
hailo_resource_read_buffer(fw_logger_resource, read_offset, size_to_read, (void*)user_buffer);
/* Change current_offset to represent the new host offset. */
read_offset += size_to_read;
hailo_resource_write32(fw_logger_resource, offsetof(FW_DEBUG_BUFFER_HEADER_t, host_offset),
(u32)(read_offset - sizeof(debug_buffer_header)));
params->read_bytes = ready_to_read;
return 0;
}
long hailo_pcie_read_firmware_log(struct hailo_resource *resource, struct hailo_read_log_params *params)
{
long err = 0;
struct hailo_resource log_resource = {resource->address, DEBUG_BUFFER_TOTAL_SIZE};
if (HAILO_CPU_ID_CPU0 == params->cpu_id) {
log_resource.address += PCIE_APP_CPU_DEBUG_OFFSET;
} else if (HAILO_CPU_ID_CPU1 == params->cpu_id) {
log_resource.address += PCIE_CORE_CPU_DEBUG_OFFSET;
} else {
return -EINVAL;
}
if (0 == params->buffer_size) {
params->read_bytes = 0;
return 0;
}
err = hailo_read_firmware_log(&log_resource, params);
if (0 != err) {
return err;
}
return 0;
}

29
drivers/media/pci/hailo/common/fw_operation.h
View File

@@ -1,29 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_COMMON_FIRMWARE_OPERATION_H_
#define _HAILO_COMMON_FIRMWARE_OPERATION_H_
#include "hailo_resource.h"
#define DEBUG_BUFFER_TOTAL_SIZE (4*1024)
#ifdef __cplusplus
extern "C" {
#endif
int hailo_read_firmware_notification(struct hailo_resource *resource, struct hailo_d2h_notification *notification);
int hailo_pcie_read_firmware_notification(struct hailo_resource *resource, struct hailo_d2h_notification *notification);
long hailo_read_firmware_log(struct hailo_resource *fw_logger_resource, struct hailo_read_log_params *params);
long hailo_pcie_read_firmware_log(struct hailo_resource *resource, struct hailo_read_log_params *params);
#ifdef __cplusplus
}
#endif
#endif /* _HAILO_COMMON_FIRMWARE_OPERATION_H_ */

114
drivers/media/pci/hailo/common/fw_validation.c
View File

@@ -1,114 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#include "fw_validation.h"
#include <linux/errno.h>
#include <linux/types.h>
/* when reading the firmware we don't want to read past the firmware_size,
so we have a consumed_firmware_offset that is updated _before_ accessing data at that offset
of firmware_base_address */
#define CONSUME_FIRMWARE(__size, __err) do { \
consumed_firmware_offset += (u32) (__size); \
if ((firmware_size < (__size)) || (firmware_size < consumed_firmware_offset)) { \
err = __err; \
goto exit; \
} \
} while(0)
int FW_VALIDATION__validate_fw_header(uintptr_t firmware_base_address,
size_t firmware_size, u32 max_code_size, u32 *outer_consumed_firmware_offset,
firmware_header_t **out_firmware_header, enum hailo_board_type board_type)
{
int err = -EINVAL;
firmware_header_t *firmware_header = NULL;
u32 consumed_firmware_offset = *outer_consumed_firmware_offset;
u32 expected_firmware_magic = 0;
firmware_header = (firmware_header_t *) (firmware_base_address + consumed_firmware_offset);
CONSUME_FIRMWARE(sizeof(firmware_header_t), -EINVAL);
switch (board_type) {
case HAILO_BOARD_TYPE_HAILO8:
expected_firmware_magic = FIRMWARE_HEADER_MAGIC_HAILO8;
break;
case HAILO_BOARD_TYPE_HAILO10H_LEGACY:
case HAILO_BOARD_TYPE_HAILO15:
case HAILO_BOARD_TYPE_HAILO10H:
expected_firmware_magic = FIRMWARE_HEADER_MAGIC_HAILO15;
break;
case HAILO_BOARD_TYPE_HAILO15L:
expected_firmware_magic = FIRMWARE_HEADER_MAGIC_HAILO15L;
break;
default:
err = -EINVAL;
goto exit;
}
if (expected_firmware_magic != firmware_header->magic) {
err = -EINVAL;
goto exit;
}
/* Validate that the firmware header version is supported */
switch(firmware_header->header_version) {
case FIRMWARE_HEADER_VERSION_INITIAL:
break;
default:
err = -EINVAL;
goto exit;
break;
}
if (MINIMUM_FIRMWARE_CODE_SIZE > firmware_header->code_size) {
err = -EINVAL;
goto exit;
}
if (max_code_size < firmware_header->code_size) {
err = -EINVAL;
goto exit;
}
CONSUME_FIRMWARE(firmware_header->code_size, -EINVAL);
*outer_consumed_firmware_offset = consumed_firmware_offset;
*out_firmware_header = firmware_header;
err = 0;
exit:
return err;
}
int FW_VALIDATION__validate_cert_header(uintptr_t firmware_base_address,
size_t firmware_size, u32 *outer_consumed_firmware_offset, secure_boot_certificate_header_t **out_firmware_cert)
{
secure_boot_certificate_header_t *firmware_cert = NULL;
int err = -EINVAL;
u32 consumed_firmware_offset = *outer_consumed_firmware_offset;
firmware_cert = (secure_boot_certificate_header_t *) (firmware_base_address + consumed_firmware_offset);
CONSUME_FIRMWARE(sizeof(secure_boot_certificate_header_t), -EINVAL);
if ((MAXIMUM_FIRMWARE_CERT_KEY_SIZE < firmware_cert->key_size) ||
(MAXIMUM_FIRMWARE_CERT_CONTENT_SIZE < firmware_cert->content_size)) {
err = -EINVAL;
goto exit;
}
CONSUME_FIRMWARE(firmware_cert->key_size, -EINVAL);
CONSUME_FIRMWARE(firmware_cert->content_size, -EINVAL);
*outer_consumed_firmware_offset = consumed_firmware_offset;
*out_firmware_cert = firmware_cert;
err = 0;
exit:
return err;
}

58
drivers/media/pci/hailo/common/fw_validation.h
View File

@@ -1,58 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef PCIE_COMMON_FIRMWARE_HEADER_UTILS_H_
#define PCIE_COMMON_FIRMWARE_HEADER_UTILS_H_
#include "hailo_ioctl_common.h"
#include <linux/types.h>
#define FIRMWARE_HEADER_MAGIC_HAILO8 (0x1DD89DE0)
#define FIRMWARE_HEADER_MAGIC_HAILO15 (0xE905DAAB)
#define FIRMWARE_HEADER_MAGIC_HAILO15L (0xF94739AB)
typedef enum {
FIRMWARE_HEADER_VERSION_INITIAL = 0,
/* MUST BE LAST */
FIRMWARE_HEADER_VERSION_COUNT
} firmware_header_version_t;
typedef struct {
u32 magic;
u32 header_version;
u32 firmware_major;
u32 firmware_minor;
u32 firmware_revision;
u32 code_size;
} firmware_header_t;
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable:4200)
#endif /* _MSC_VER */
typedef struct {
u32 key_size;
u32 content_size;
} secure_boot_certificate_header_t;
#ifdef _MSC_VER
#pragma warning(pop)
#endif /* _MSC_VER */
#define MINIMUM_FIRMWARE_CODE_SIZE (20*4)
#define MAXIMUM_FIRMWARE_CERT_KEY_SIZE (0x1000)
#define MAXIMUM_FIRMWARE_CERT_CONTENT_SIZE (0x1000)
int FW_VALIDATION__validate_fw_header(uintptr_t firmware_base_address,
size_t firmware_size, u32 max_code_size, u32 *outer_consumed_firmware_offset,
firmware_header_t **out_firmware_header, enum hailo_board_type board_type);
int FW_VALIDATION__validate_cert_header(uintptr_t firmware_base_address,
size_t firmware_size, u32 *outer_consumed_firmware_offset, secure_boot_certificate_header_t **out_firmware_cert);
#endif

685
drivers/media/pci/hailo/common/hailo_ioctl_common.h
View File

@@ -1,685 +0,0 @@
// SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) AND MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_IOCTL_COMMON_H_
#define _HAILO_IOCTL_COMMON_H_
#define HAILO_DRV_VER_MAJOR 4
#define HAILO_DRV_VER_MINOR 20
#define HAILO_DRV_VER_REVISION 0
#define _STRINGIFY_EXPANDED( x ) #x
#define _STRINGIFY_NUMBER( x ) _STRINGIFY_EXPANDED(x)
#define HAILO_DRV_VER _STRINGIFY_NUMBER(HAILO_DRV_VER_MAJOR) "." _STRINGIFY_NUMBER(HAILO_DRV_VER_MINOR) "." _STRINGIFY_NUMBER(HAILO_DRV_VER_REVISION)
// This value is not easily changeable.
// For example: the channel interrupts ioctls assume we have up to 32 channels
#define MAX_VDMA_CHANNELS_PER_ENGINE (32)
#define VDMA_CHANNELS_PER_ENGINE_PER_DIRECTION (16)
#define MAX_VDMA_ENGINES (3)
#define SIZE_OF_VDMA_DESCRIPTOR (16)
#define VDMA_DEST_CHANNELS_START (16)
#define MAX_SG_DESCS_COUNT (64 * 1024u)
#define HAILO_VDMA_MAX_ONGOING_TRANSFERS (128)
#define HAILO_VDMA_MAX_ONGOING_TRANSFERS_MASK (HAILO_VDMA_MAX_ONGOING_TRANSFERS - 1)
#define CHANNEL_IRQ_TIMESTAMPS_SIZE (HAILO_VDMA_MAX_ONGOING_TRANSFERS * 2)
#define CHANNEL_IRQ_TIMESTAMPS_SIZE_MASK (CHANNEL_IRQ_TIMESTAMPS_SIZE - 1)
#define INVALID_DRIVER_HANDLE_VALUE ((uintptr_t)-1)
// Used by windows and unix driver to raise the right CPU control handle to the FW. The same as in pcie_service FW
#define FW_ACCESS_CORE_CPU_CONTROL_SHIFT (1)
#define FW_ACCESS_CORE_CPU_CONTROL_MASK (1 << FW_ACCESS_CORE_CPU_CONTROL_SHIFT)
#define FW_ACCESS_CONTROL_INTERRUPT_SHIFT (0)
#define FW_ACCESS_APP_CPU_CONTROL_MASK (1 << FW_ACCESS_CONTROL_INTERRUPT_SHIFT)
#define FW_ACCESS_DRIVER_SHUTDOWN_SHIFT (2)
#define FW_ACCESS_DRIVER_SHUTDOWN_MASK (1 << FW_ACCESS_DRIVER_SHUTDOWN_SHIFT)
// HRT-15790 TODO: separate nnc interrupts and soc interrupts
#define FW_ACCESS_SOFT_RESET_SHIFT (3)
#define FW_ACCESS_SOFT_RESET_MASK (1 << FW_ACCESS_SOFT_RESET_SHIFT)
#define FW_ACCESS_SOC_CONTROL_SHIFT (3)
#define FW_ACCESS_SOC_CONTROL_MASK (1 << FW_ACCESS_SOC_CONTROL_SHIFT)
#define INVALID_VDMA_CHANNEL (0xff)
#if !defined(__cplusplus) && defined(NTDDI_VERSION)
#include <wdm.h>
typedef ULONG uint32_t;
typedef UCHAR uint8_t;
typedef USHORT uint16_t;
typedef ULONGLONG uint64_t;
#endif /* !defined(__cplusplus) && defined(NTDDI_VERSION) */
#ifdef _MSC_VER
#include <initguid.h>
#if !defined(bool) && !defined(__cplusplus)
typedef uint8_t bool;
#endif // !defined(bool) && !defined(__cplusplus)
#if !defined(INT_MAX)
#define INT_MAX 0x7FFFFFFF
#endif // !defined(INT_MAX)
#if !defined(ECONNRESET)
#define ECONNRESET 104 /* Connection reset by peer */
#endif // !defined(ECONNRESET)
// {d88d31f1-fede-4e71-ac2a-6ce0018c1501}
DEFINE_GUID (GUID_DEVINTERFACE_HailoKM_NNC,
0xd88d31f1,0xfede,0x4e71,0xac,0x2a,0x6c,0xe0,0x01,0x8c,0x15,0x01);
// {7f16047d-64b8-207a-0092-e970893970a2}
DEFINE_GUID (GUID_DEVINTERFACE_HailoKM_SOC,
0x7f16047d,0x64b8,0x207a,0x00,0x92,0xe9,0x70,0x89,0x39,0x70,0xa2);
#define HAILO_GENERAL_IOCTL_MAGIC 0
#define HAILO_VDMA_IOCTL_MAGIC 1
#define HAILO_SOC_IOCTL_MAGIC 2
#define HAILO_PCI_EP_IOCTL_MAGIC 3
#define HAILO_NNC_IOCTL_MAGIC 4
#define HAILO_IOCTL_COMPATIBLE CTL_CODE(FILE_DEVICE_UNKNOWN, 0x802, METHOD_BUFFERED, FILE_ANY_ACCESS)
typedef struct tCompatibleHailoIoctlParam
{
union {
struct {
ULONG Size : 16;
ULONG Code : 8;
ULONG Type : 6;
ULONG Read : 1;
ULONG Write : 1;
} bits;
ULONG value;
} u;
} tCompatibleHailoIoctlParam;
static ULONG FORCEINLINE _IOC_(ULONG nr, ULONG type, ULONG size, bool read, bool write)
{
struct tCompatibleHailoIoctlParam param;
param.u.bits.Code = nr;
param.u.bits.Size = size;
param.u.bits.Type = type;
param.u.bits.Read = read ? 1 : 0;
param.u.bits.Write = write ? 1 : 0;
return param.u.value;
}
#define _IOW_(type,nr,size) _IOC_(nr, type, sizeof(size), true, false)
#define _IOR_(type,nr,size) _IOC_(nr, type, sizeof(size), false, true)
#define _IOWR_(type,nr,size) _IOC_(nr, type, sizeof(size), true, true)
#define _IO_(type,nr) _IOC_(nr, type, 0, false, false)
#elif defined(__linux__) // #ifdef _MSC_VER
#ifndef __KERNEL__
// include the userspace headers only if this file is included by user space program
// It is discourged to include them when compiling the driver (https://lwn.net/Articles/113349/)
#include <stdint.h>
#include <sys/types.h>
#else
#include <linux/types.h>
#include <linux/limits.h>
#include <linux/kernel.h>
#endif // ifndef __KERNEL__
#include <linux/ioctl.h>
#define _IOW_ _IOW
#define _IOR_ _IOR
#define _IOWR_ _IOWR
#define _IO_ _IO
#define HAILO_GENERAL_IOCTL_MAGIC 'g'
#define HAILO_VDMA_IOCTL_MAGIC 'v'
#define HAILO_SOC_IOCTL_MAGIC 's'
#define HAILO_NNC_IOCTL_MAGIC 'n'
#define HAILO_PCI_EP_IOCTL_MAGIC 'p'
#elif defined(__QNX__) // #ifdef _MSC_VER
#include <devctl.h>
#include <stdint.h>
#include <sys/types.h>
#include <sys/mman.h>
#include <stdbool.h>
// defines for devctl
#define _IOW_ __DIOF
#define _IOR_ __DIOT
#define _IOWR_ __DIOTF
#define _IO_ __DION
#define HAILO_GENERAL_IOCTL_MAGIC _DCMD_ALL
#define HAILO_VDMA_IOCTL_MAGIC _DCMD_MISC
#else // #ifdef _MSC_VER
#error "unsupported platform!"
#endif
#pragma pack(push, 1)
struct hailo_channel_interrupt_timestamp {
uint64_t timestamp_ns;
uint16_t desc_num_processed;
};
typedef struct {
uint16_t is_buffer_in_use;
uint16_t buffer_len;
} hailo_d2h_buffer_details_t;
// This struct is the same as `enum dma_data_direction` (defined in linux/dma-direction)
enum hailo_dma_data_direction {
HAILO_DMA_BIDIRECTIONAL = 0,
HAILO_DMA_TO_DEVICE = 1,
HAILO_DMA_FROM_DEVICE = 2,
HAILO_DMA_NONE = 3,
/** Max enum value to maintain ABI Integrity */
HAILO_DMA_MAX_ENUM = INT_MAX,
};
// Enum that states what type of buffer we are working with in the driver
enum hailo_dma_buffer_type {
HAILO_DMA_USER_PTR_BUFFER = 0,
HAILO_DMA_DMABUF_BUFFER = 1,
/** Max enum value to maintain ABI Integrity */
HAILO_DMA_BUFFER_MAX_ENUM = INT_MAX,
};
// Enum that determines if buffer should be allocated from user space or from driver
enum hailo_allocation_mode {
HAILO_ALLOCATION_MODE_USERSPACE = 0,
HAILO_ALLOCATION_MODE_DRIVER = 1,
/** Max enum value to maintain ABI Integrity */
HAILO_ALLOCATION_MODE_MAX_ENUM = INT_MAX,
};
enum hailo_vdma_interrupts_domain {
HAILO_VDMA_INTERRUPTS_DOMAIN_NONE = 0,
HAILO_VDMA_INTERRUPTS_DOMAIN_DEVICE = (1 << 0),
HAILO_VDMA_INTERRUPTS_DOMAIN_HOST = (1 << 1),
/** Max enum value to maintain ABI Integrity */
HAILO_VDMA_INTERRUPTS_DOMAIN_MAX_ENUM = INT_MAX,
};
/* structure used in ioctl HAILO_VDMA_BUFFER_MAP */
struct hailo_vdma_buffer_map_params {
#if defined(__linux__) || defined(_MSC_VER)
uintptr_t user_address; // in
#elif defined(__QNX__)
shm_handle_t shared_memory_handle; // in
#else
#error "unsupported platform!"
#endif // __linux__
size_t size; // in
enum hailo_dma_data_direction data_direction; // in
enum hailo_dma_buffer_type buffer_type; // in
uintptr_t allocated_buffer_handle; // in
size_t mapped_handle; // out
};
/* structure used in ioctl HAILO_VDMA_BUFFER_UNMAP */
struct hailo_vdma_buffer_unmap_params {
size_t mapped_handle;
};
/* structure used in ioctl HAILO_DESC_LIST_CREATE */
struct hailo_desc_list_create_params {
size_t desc_count; // in
uint16_t desc_page_size; // in
bool is_circular; // in
uintptr_t desc_handle; // out
uint64_t dma_address; // out
};
/* structure used in ioctl HAILO_DESC_LIST_RELEASE */
struct hailo_desc_list_release_params {
uintptr_t desc_handle; // in
};
struct hailo_write_action_list_params {
uint8_t *data; // in
size_t size; // in
uint64_t dma_address; // out
};
/* structure used in ioctl HAILO_DESC_LIST_BIND_VDMA_BUFFER */
struct hailo_desc_list_program_params {
size_t buffer_handle; // in
size_t buffer_size; // in
size_t buffer_offset; // in
uintptr_t desc_handle; // in
uint8_t channel_index; // in
uint32_t starting_desc; // in
bool should_bind; // in
enum hailo_vdma_interrupts_domain last_interrupts_domain; // in
bool is_debug; // in
};
/* structure used in ioctl HAILO_VDMA_ENABLE_CHANNELS */
struct hailo_vdma_enable_channels_params {
uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES]; // in
bool enable_timestamps_measure; // in
};
/* structure used in ioctl HAILO_VDMA_DISABLE_CHANNELS */
struct hailo_vdma_disable_channels_params {
uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES]; // in
};
/* structure used in ioctl HAILO_VDMA_INTERRUPTS_WAIT */
struct hailo_vdma_interrupts_channel_data {
uint8_t engine_index;
uint8_t channel_index;
bool is_active; // If not activate, num_processed is ignored.
uint8_t transfers_completed; // Number of transfers completed.
uint8_t host_error; // Channel errors bits on source side
uint8_t device_error; // Channel errors bits on dest side
bool validation_success; // If the validation of the channel was successful
};
struct hailo_vdma_interrupts_wait_params {
uint32_t channels_bitmap_per_engine[MAX_VDMA_ENGINES]; // in
uint8_t channels_count; // out
struct hailo_vdma_interrupts_channel_data
irq_data[MAX_VDMA_CHANNELS_PER_ENGINE * MAX_VDMA_ENGINES]; // out
};
/* structure used in ioctl HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS */
struct hailo_vdma_interrupts_read_timestamp_params {
uint8_t engine_index; // in
uint8_t channel_index; // in
uint32_t timestamps_count; // out
struct hailo_channel_interrupt_timestamp timestamps[CHANNEL_IRQ_TIMESTAMPS_SIZE]; // out
};
/* structure used in ioctl HAILO_FW_CONTROL */
#define MAX_CONTROL_LENGTH (1500)
#define PCIE_EXPECTED_MD5_LENGTH (16)
/* structure used in ioctl HAILO_FW_CONTROL and HAILO_READ_LOG */
enum hailo_cpu_id {
HAILO_CPU_ID_CPU0 = 0,
HAILO_CPU_ID_CPU1,
HAILO_CPU_ID_NONE,
/** Max enum value to maintain ABI Integrity */
HAILO_CPU_MAX_ENUM = INT_MAX,
};
struct hailo_fw_control {
// expected_md5+buffer_len+buffer must be in this order at the start of the struct
uint8_t expected_md5[PCIE_EXPECTED_MD5_LENGTH];
uint32_t buffer_len;
uint8_t buffer[MAX_CONTROL_LENGTH];
uint32_t timeout_ms;
enum hailo_cpu_id cpu_id;
};
/* structure used in ioctl HAILO_MEMORY_TRANSFER */
// Max bar transfer size gotten from ATR0_TABLE_SIZE
#define MAX_MEMORY_TRANSFER_LENGTH (4096)
enum hailo_transfer_direction {
TRANSFER_READ = 0,
TRANSFER_WRITE,
/** Max enum value to maintain ABI Integrity */
TRANSFER_MAX_ENUM = INT_MAX,
};
enum hailo_transfer_memory_type {
HAILO_TRANSFER_DEVICE_DIRECT_MEMORY,
// vDMA memories
HAILO_TRANSFER_MEMORY_VDMA0 = 0x100,
HAILO_TRANSFER_MEMORY_VDMA1,
HAILO_TRANSFER_MEMORY_VDMA2,
// PCIe driver memories
HAILO_TRANSFER_MEMORY_PCIE_BAR0 = 0x200,
HAILO_TRANSFER_MEMORY_PCIE_BAR2 = 0x202,
HAILO_TRANSFER_MEMORY_PCIE_BAR4 = 0x204,
// DRAM DMA driver memories
HAILO_TRANSFER_MEMORY_DMA_ENGINE0 = 0x300,
HAILO_TRANSFER_MEMORY_DMA_ENGINE1,
HAILO_TRANSFER_MEMORY_DMA_ENGINE2,
// PCIe EP driver memories
HAILO_TRANSFER_MEMORY_PCIE_EP_CONFIG = 0x400,
HAILO_TRANSFER_MEMORY_PCIE_EP_BRIDGE,
/** Max enum value to maintain ABI Integrity */
HAILO_TRANSFER_MEMORY_MAX_ENUM = INT_MAX,
};
struct hailo_memory_transfer_params {
enum hailo_transfer_direction transfer_direction; // in
enum hailo_transfer_memory_type memory_type; // in
uint64_t address; // in
size_t count; // in
uint8_t buffer[MAX_MEMORY_TRANSFER_LENGTH]; // in/out
};
/* structure used in ioctl HAILO_VDMA_BUFFER_SYNC */
enum hailo_vdma_buffer_sync_type {
HAILO_SYNC_FOR_CPU,
HAILO_SYNC_FOR_DEVICE,
/** Max enum value to maintain ABI Integrity */
HAILO_SYNC_MAX_ENUM = INT_MAX,
};
struct hailo_vdma_buffer_sync_params {
size_t handle; // in
enum hailo_vdma_buffer_sync_type sync_type; // in
size_t offset; // in
size_t count; // in
};
/* structure used in ioctl HAILO_READ_NOTIFICATION */
#define MAX_NOTIFICATION_LENGTH (1500)
struct hailo_d2h_notification {
size_t buffer_len; // out
uint8_t buffer[MAX_NOTIFICATION_LENGTH]; // out
};
enum hailo_board_type {
HAILO_BOARD_TYPE_HAILO8 = 0,
HAILO_BOARD_TYPE_HAILO15,
HAILO_BOARD_TYPE_HAILO15L,
HAILO_BOARD_TYPE_HAILO10H,
HAILO_BOARD_TYPE_HAILO10H_LEGACY,
HAILO_BOARD_TYPE_COUNT,
/** Max enum value to maintain ABI Integrity */
HAILO_BOARD_TYPE_MAX_ENUM = INT_MAX
};
enum hailo_accelerator_type {
HAILO_ACCELERATOR_TYPE_NNC,
HAILO_ACCELERATOR_TYPE_SOC,
/** Max enum value to maintain ABI Integrity */
HAILO_ACCELERATOR_TYPE_MAX_ENUM = INT_MAX
};
enum hailo_dma_type {
HAILO_DMA_TYPE_PCIE,
HAILO_DMA_TYPE_DRAM,
HAILO_DMA_TYPE_PCI_EP,
/** Max enum value to maintain ABI Integrity */
HAILO_DMA_TYPE_MAX_ENUM = INT_MAX,
};
struct hailo_device_properties {
uint16_t desc_max_page_size;
enum hailo_board_type board_type;
enum hailo_allocation_mode allocation_mode;
enum hailo_dma_type dma_type;
size_t dma_engines_count;
bool is_fw_loaded;
#ifdef __QNX__
pid_t resource_manager_pid;
#endif // __QNX__
};
struct hailo_driver_info {
uint32_t major_version;
uint32_t minor_version;
uint32_t revision_version;
};
/* structure used in ioctl HAILO_READ_LOG */
#define MAX_FW_LOG_BUFFER_LENGTH (512)
struct hailo_read_log_params {
enum hailo_cpu_id cpu_id; // in
uint8_t buffer[MAX_FW_LOG_BUFFER_LENGTH]; // out
size_t buffer_size; // in
size_t read_bytes; // out
};
/* structure used in ioctl HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC */
struct hailo_allocate_low_memory_buffer_params {
size_t buffer_size; // in
uintptr_t buffer_handle; // out
};
/* structure used in ioctl HAILO_VDMA_LOW_MEMORY_BUFFER_FREE */
struct hailo_free_low_memory_buffer_params {
uintptr_t buffer_handle; // in
};
struct hailo_mark_as_in_use_params {
bool in_use; // out
};
/* structure used in ioctl HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC */
struct hailo_allocate_continuous_buffer_params {
size_t buffer_size; // in
uintptr_t buffer_handle; // out
uint64_t dma_address; // out
};
/* structure used in ioctl HAILO_VDMA_CONTINUOUS_BUFFER_FREE */
struct hailo_free_continuous_buffer_params {
uintptr_t buffer_handle; // in
};
/* structures used in ioctl HAILO_VDMA_LAUNCH_TRANSFER */
struct hailo_vdma_transfer_buffer {
size_t mapped_buffer_handle; // in
uint32_t offset; // in
uint32_t size; // in
};
// We allow maximum 2 buffers per transfer since we may have an extra buffer
// to make sure each buffer is aligned to page size.
#define HAILO_MAX_BUFFERS_PER_SINGLE_TRANSFER (2)
struct hailo_vdma_launch_transfer_params {
uint8_t engine_index; // in
uint8_t channel_index; // in
uintptr_t desc_handle; // in
uint32_t starting_desc; // in
bool should_bind; // in, if false, assumes buffer already bound.
uint8_t buffers_count; // in
struct hailo_vdma_transfer_buffer
buffers[HAILO_MAX_BUFFERS_PER_SINGLE_TRANSFER]; // in
enum hailo_vdma_interrupts_domain first_interrupts_domain; // in
enum hailo_vdma_interrupts_domain last_interrupts_domain; // in
bool is_debug; // in, if set program hw to send
// more info (e.g desc complete status)
uint32_t descs_programed; // out, amount of descriptors programed.
int launch_transfer_status; // out, status of the launch transfer call. (only used in case of error)
};
/* structure used in ioctl HAILO_SOC_CONNECT */
struct hailo_soc_connect_params {
uint16_t port_number; // in
uint8_t input_channel_index; // out
uint8_t output_channel_index; // out
uintptr_t input_desc_handle; // in
uintptr_t output_desc_handle; // in
};
/* structure used in ioctl HAILO_SOC_CLOSE */
struct hailo_soc_close_params {
uint8_t input_channel_index; // in
uint8_t output_channel_index; // in
};
/* structure used in ioctl HAILO_PCI_EP_ACCEPT */
struct hailo_pci_ep_accept_params {
uint16_t port_number; // in
uint8_t input_channel_index; // out
uint8_t output_channel_index; // out
uintptr_t input_desc_handle; // in
uintptr_t output_desc_handle; // in
};
/* structure used in ioctl HAILO_PCI_EP_CLOSE */
struct hailo_pci_ep_close_params {
uint8_t input_channel_index; // in
uint8_t output_channel_index; // in
};
#ifdef _MSC_VER
struct tCompatibleHailoIoctlData
{
tCompatibleHailoIoctlParam Parameters;
ULONG_PTR Value;
union {
struct hailo_memory_transfer_params MemoryTransfer;
struct hailo_vdma_enable_channels_params VdmaEnableChannels;
struct hailo_vdma_disable_channels_params VdmaDisableChannels;
struct hailo_vdma_interrupts_read_timestamp_params VdmaInterruptsReadTimestamps;
struct hailo_vdma_interrupts_wait_params VdmaInterruptsWait;
struct hailo_vdma_buffer_sync_params VdmaBufferSync;
struct hailo_fw_control FirmwareControl;
struct hailo_vdma_buffer_map_params VdmaBufferMap;
struct hailo_vdma_buffer_unmap_params VdmaBufferUnmap;
struct hailo_desc_list_create_params DescListCreate;
struct hailo_desc_list_release_params DescListReleaseParam;
struct hailo_desc_list_program_params DescListProgram;
struct hailo_d2h_notification D2HNotification;
struct hailo_device_properties DeviceProperties;
struct hailo_driver_info DriverInfo;
struct hailo_read_log_params ReadLog;
struct hailo_mark_as_in_use_params MarkAsInUse;
struct hailo_vdma_launch_transfer_params LaunchTransfer;
struct hailo_soc_connect_params ConnectParams;
struct hailo_soc_close_params SocCloseParams;
struct hailo_pci_ep_accept_params AcceptParams;
struct hailo_pci_ep_close_params PciEpCloseParams;
struct hailo_write_action_list_params WriteActionListParams;
} Buffer;
};
#endif // _MSC_VER
#pragma pack(pop)
enum hailo_general_ioctl_code {
HAILO_MEMORY_TRANSFER_CODE,
HAILO_QUERY_DEVICE_PROPERTIES_CODE,
HAILO_QUERY_DRIVER_INFO_CODE,
// Must be last
HAILO_GENERAL_IOCTL_MAX_NR,
};
#define HAILO_MEMORY_TRANSFER _IOWR_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_MEMORY_TRANSFER_CODE, struct hailo_memory_transfer_params)
#define HAILO_QUERY_DEVICE_PROPERTIES _IOW_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_QUERY_DEVICE_PROPERTIES_CODE, struct hailo_device_properties)
#define HAILO_QUERY_DRIVER_INFO _IOW_(HAILO_GENERAL_IOCTL_MAGIC, HAILO_QUERY_DRIVER_INFO_CODE, struct hailo_driver_info)
enum hailo_vdma_ioctl_code {
HAILO_VDMA_ENABLE_CHANNELS_CODE,
HAILO_VDMA_DISABLE_CHANNELS_CODE,
HAILO_VDMA_INTERRUPTS_WAIT_CODE,
HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE,
HAILO_VDMA_BUFFER_MAP_CODE,
HAILO_VDMA_BUFFER_UNMAP_CODE,
HAILO_VDMA_BUFFER_SYNC_CODE,
HAILO_DESC_LIST_CREATE_CODE,
HAILO_DESC_LIST_RELEASE_CODE,
HAILO_DESC_LIST_PROGRAM_CODE,
HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC_CODE,
HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE,
HAILO_MARK_AS_IN_USE_CODE,
HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC_CODE,
HAILO_VDMA_CONTINUOUS_BUFFER_FREE_CODE,
HAILO_VDMA_LAUNCH_TRANSFER_CODE,
// Must be last
HAILO_VDMA_IOCTL_MAX_NR,
};
#define HAILO_VDMA_ENABLE_CHANNELS _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_ENABLE_CHANNELS_CODE, struct hailo_vdma_enable_channels_params)
#define HAILO_VDMA_DISABLE_CHANNELS _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_DISABLE_CHANNELS_CODE, struct hailo_vdma_disable_channels_params)
#define HAILO_VDMA_INTERRUPTS_WAIT _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_WAIT_CODE, struct hailo_vdma_interrupts_wait_params)
#define HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS_CODE, struct hailo_vdma_interrupts_read_timestamp_params)
#define HAILO_VDMA_BUFFER_MAP _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_MAP_CODE, struct hailo_vdma_buffer_map_params)
#define HAILO_VDMA_BUFFER_UNMAP _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_UNMAP_CODE, struct hailo_vdma_buffer_unmap_params)
#define HAILO_VDMA_BUFFER_SYNC _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_BUFFER_SYNC_CODE, struct hailo_vdma_buffer_sync_params)
#define HAILO_DESC_LIST_CREATE _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_CREATE_CODE, struct hailo_desc_list_create_params)
#define HAILO_DESC_LIST_RELEASE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_RELEASE_CODE, struct hailo_desc_list_release_params)
#define HAILO_DESC_LIST_PROGRAM _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_DESC_LIST_PROGRAM_CODE, struct hailo_desc_list_program_params)
#define HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC_CODE, struct hailo_allocate_low_memory_buffer_params)
#define HAILO_VDMA_LOW_MEMORY_BUFFER_FREE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LOW_MEMORY_BUFFER_FREE_CODE, struct hailo_free_low_memory_buffer_params)
#define HAILO_MARK_AS_IN_USE _IOW_(HAILO_VDMA_IOCTL_MAGIC, HAILO_MARK_AS_IN_USE_CODE, struct hailo_mark_as_in_use_params)
#define HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC_CODE, struct hailo_allocate_continuous_buffer_params)
#define HAILO_VDMA_CONTINUOUS_BUFFER_FREE _IOR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_CONTINUOUS_BUFFER_FREE_CODE, struct hailo_free_continuous_buffer_params)
#define HAILO_VDMA_LAUNCH_TRANSFER _IOWR_(HAILO_VDMA_IOCTL_MAGIC, HAILO_VDMA_LAUNCH_TRANSFER_CODE, struct hailo_vdma_launch_transfer_params)
enum hailo_nnc_ioctl_code {
HAILO_FW_CONTROL_CODE,
HAILO_READ_NOTIFICATION_CODE,
HAILO_DISABLE_NOTIFICATION_CODE,
HAILO_READ_LOG_CODE,
HAILO_RESET_NN_CORE_CODE,
HAILO_WRITE_ACTION_LIST_CODE,
// Must be last
HAILO_NNC_IOCTL_MAX_NR
};
#define HAILO_FW_CONTROL _IOWR_(HAILO_NNC_IOCTL_MAGIC, HAILO_FW_CONTROL_CODE, struct hailo_fw_control)
#define HAILO_READ_NOTIFICATION _IOW_(HAILO_NNC_IOCTL_MAGIC, HAILO_READ_NOTIFICATION_CODE, struct hailo_d2h_notification)
#define HAILO_DISABLE_NOTIFICATION _IO_(HAILO_NNC_IOCTL_MAGIC, HAILO_DISABLE_NOTIFICATION_CODE)
#define HAILO_READ_LOG _IOWR_(HAILO_NNC_IOCTL_MAGIC, HAILO_READ_LOG_CODE, struct hailo_read_log_params)
#define HAILO_RESET_NN_CORE _IO_(HAILO_NNC_IOCTL_MAGIC, HAILO_RESET_NN_CORE_CODE)
#define HAILO_WRITE_ACTION_LIST _IOW_(HAILO_NNC_IOCTL_MAGIC, HAILO_WRITE_ACTION_LIST_CODE, struct hailo_write_action_list_params)
enum hailo_soc_ioctl_code {
HAILO_SOC_IOCTL_CONNECT_CODE,
HAILO_SOC_IOCTL_CLOSE_CODE,
// Must be last
HAILO_SOC_IOCTL_MAX_NR,
};
#define HAILO_SOC_CONNECT _IOWR_(HAILO_SOC_IOCTL_MAGIC, HAILO_SOC_IOCTL_CONNECT_CODE, struct hailo_soc_connect_params)
#define HAILO_SOC_CLOSE _IOR_(HAILO_SOC_IOCTL_MAGIC, HAILO_SOC_IOCTL_CLOSE_CODE, struct hailo_soc_close_params)
enum hailo_pci_ep_ioctl_code {
HAILO_PCI_EP_ACCEPT_CODE,
HAILO_PCI_EP_CLOSE_CODE,
// Must be last
HAILO_PCI_EP_IOCTL_MAX_NR,
};
#define HAILO_PCI_EP_ACCEPT _IOWR_(HAILO_PCI_EP_IOCTL_MAGIC, HAILO_PCI_EP_ACCEPT_CODE, struct hailo_pci_ep_accept_params)
#define HAILO_PCI_EP_CLOSE _IOR_(HAILO_PCI_EP_IOCTL_MAGIC, HAILO_PCI_EP_CLOSE_CODE, struct hailo_pci_ep_close_params)
#endif /* _HAILO_IOCTL_COMMON_H_ */

13
drivers/media/pci/hailo/common/hailo_pcie_version.h
View File

@@ -1,13 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2022 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_COMMON_PCIE_VERSION_H_
#define _HAILO_COMMON_PCIE_VERSION_H_
#define HAILO_DRV_VER_MAJOR 4
#define HAILO_DRV_VER_MINOR 17
#define HAILO_DRV_VER_REVISION 0
#endif /* _HAILO_COMMON_PCIE_VERSION_H_ */

141
drivers/media/pci/hailo/common/hailo_resource.c
View File

@@ -1,141 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#include "hailo_resource.h"
#include "utils.h"
#include <linux/io.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/kernel.h>
#define ALIGN_TO_32_BIT(addr) ((addr) & (~((uintptr_t)0x3)))
u8 hailo_resource_read8(struct hailo_resource *resource, size_t offset)
{
u32 val = ioread32((u8*)ALIGN_TO_32_BIT(resource->address + offset));
u64 offset_in_bits = BITS_IN_BYTE * ((resource->address + offset) - ALIGN_TO_32_BIT(resource->address + offset));
return (u8)READ_BITS_AT_OFFSET(BYTE_SIZE * BITS_IN_BYTE, offset_in_bits, val);
}
u16 hailo_resource_read16(struct hailo_resource *resource, size_t offset)
{
u32 val = ioread32((u8*)ALIGN_TO_32_BIT(resource->address + offset));
u64 offset_in_bits = BITS_IN_BYTE * ((resource->address + offset) - ALIGN_TO_32_BIT(resource->address + offset));
return (u16)READ_BITS_AT_OFFSET(WORD_SIZE * BITS_IN_BYTE, offset_in_bits, val);
}
u32 hailo_resource_read32(struct hailo_resource *resource, size_t offset)
{
return ioread32((u8*)resource->address + offset);
}
void hailo_resource_write8(struct hailo_resource *resource, size_t offset, u8 value)
{
u32 initial_val = ioread32((u8*)ALIGN_TO_32_BIT(resource->address + offset));
u64 offset_in_bits = BITS_IN_BYTE * ((resource->address + offset) - ALIGN_TO_32_BIT(resource->address + offset));
iowrite32(WRITE_BITS_AT_OFFSET(BYTE_SIZE * BITS_IN_BYTE, offset_in_bits, initial_val, value),
(u8*)ALIGN_TO_32_BIT(resource->address + offset));
}
void hailo_resource_write16(struct hailo_resource *resource, size_t offset, u16 value)
{
u32 initial_val = ioread32((u8*)ALIGN_TO_32_BIT(resource->address + offset));
u64 offset_in_bits = BITS_IN_BYTE * ((resource->address + offset) - ALIGN_TO_32_BIT(resource->address + offset));
iowrite32(WRITE_BITS_AT_OFFSET(WORD_SIZE * BITS_IN_BYTE, offset_in_bits, initial_val, value),
(u8*)ALIGN_TO_32_BIT(resource->address + offset));
}
void hailo_resource_write32(struct hailo_resource *resource, size_t offset, u32 value)
{
iowrite32(value, (u8*)resource->address + offset);
}
void hailo_resource_read_buffer(struct hailo_resource *resource, size_t offset, size_t count, void *to)
{
// Copied and modified from linux aarch64 (using ioread32 instead of readq that does not work all the time)
uintptr_t to_ptr = (uintptr_t)to;
while ((count > 0) && (!IS_ALIGNED(to_ptr, 4) || !IS_ALIGNED((uintptr_t)resource->address + offset, 4))) {
*(u8*)to_ptr = hailo_resource_read8(resource, offset);
to_ptr++;
offset++;
count--;
}
while (count >= 4) {
*(u32*)to_ptr = hailo_resource_read32(resource, offset);
to_ptr += 4;
offset += 4;
count -= 4;
}
while (count > 0) {
*(u8*)to_ptr = hailo_resource_read8(resource, offset);
to_ptr++;
offset++;
count--;
}
}
int hailo_resource_write_buffer(struct hailo_resource *resource, size_t offset, size_t count, const void *from)
{
// read the bytes after writing them for flushing the data. This function also checks if the pcie link
// is broken.
uintptr_t from_ptr = (uintptr_t)from;
while (count && (!IS_ALIGNED(resource->address + offset, 4) || !IS_ALIGNED(from_ptr, 4))) {
hailo_resource_write8(resource, offset, *(u8*)from_ptr);
if (hailo_resource_read8(resource, offset) != *(u8*)from_ptr) {
return -EIO;
}
from_ptr++;
offset++;
count--;
}
while (count >= 4) {
hailo_resource_write32(resource, offset, *(u32*)from_ptr);
if (hailo_resource_read32(resource, offset) != *(u32*)from_ptr) {
return -EIO;
}
from_ptr += 4;
offset += 4;
count -= 4;
}
while (count) {
hailo_resource_write8(resource, offset, *(u8*)from_ptr);
if (hailo_resource_read8(resource, offset) != *(u8*)from_ptr) {
return -EIO;
}
from_ptr++;
offset++;
count--;
}
return 0;
}
int hailo_resource_transfer(struct hailo_resource *resource, struct hailo_memory_transfer_params *transfer)
{
// Check for transfer size (address is in resources address-space)
if ((transfer->address + transfer->count) > (u64)resource->size) {
return -EINVAL;
}
if (transfer->count > ARRAY_SIZE(transfer->buffer)) {
return -EINVAL;
}
switch (transfer->transfer_direction) {
case TRANSFER_READ:
hailo_resource_read_buffer(resource, (u32)transfer->address, transfer->count, transfer->buffer);
return 0;
case TRANSFER_WRITE:
return hailo_resource_write_buffer(resource, (u32)transfer->address, transfer->count, transfer->buffer);
default:
return -EINVAL;
}
}

39
drivers/media/pci/hailo/common/hailo_resource.h
View File

@@ -1,39 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_COMMON_HAILO_RESOURCE_H_
#define _HAILO_COMMON_HAILO_RESOURCE_H_
#include "hailo_ioctl_common.h"
#include <linux/types.h>
struct hailo_resource {
uintptr_t address;
size_t size;
};
#ifdef __cplusplus
extern "C" {
#endif
// Implemented by the specific platform
u32 hailo_resource_read32(struct hailo_resource *resource, size_t offset);
u16 hailo_resource_read16(struct hailo_resource *resource, size_t offset);
u8 hailo_resource_read8(struct hailo_resource *resource, size_t offset);
void hailo_resource_write32(struct hailo_resource *resource, size_t offset, u32 value);
void hailo_resource_write16(struct hailo_resource *resource, size_t offset, u16 value);
void hailo_resource_write8(struct hailo_resource *resource, size_t offset, u8 value);
void hailo_resource_read_buffer(struct hailo_resource *resource, size_t offset, size_t count, void *to);
int hailo_resource_write_buffer(struct hailo_resource *resource, size_t offset, size_t count, const void *from);
// Transfer (read/write) the given resource into/from transfer params.
int hailo_resource_transfer(struct hailo_resource *resource, struct hailo_memory_transfer_params *transfer);
#ifdef __cplusplus
}
#endif
#endif /* _HAILO_COMMON_HAILO_RESOURCE_H_ */

913
drivers/media/pci/hailo/common/pcie_common.c
View File

@@ -1,913 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#include "pcie_common.h"
#include "fw_operation.h"
#include "soc_structs.h"
#include <linux/errno.h>
#include <linux/bug.h>
#include <linux/delay.h>
#include <linux/kernel.h>
#include <linux/printk.h>
#include <linux/device.h>
#define BSC_IMASK_HOST (0x0188)
#define BCS_ISTATUS_HOST (0x018C)
#define BCS_SOURCE_INTERRUPT_PER_CHANNEL (0x400)
#define BCS_DESTINATION_INTERRUPT_PER_CHANNEL (0x500)
#define PO2_ROUND_UP(size, alignment) ((size + alignment-1) & ~(alignment-1))
#define ATR_PARAM (0x17)
#define ATR_SRC_ADDR (0x0)
#define ATR_TRSL_PARAM (6)
#define ATR_TABLE_SIZE (0x1000u)
#define ATR_TABLE_SIZE_MASK (0x1000u - 1)
#define ATR0_PCIE_BRIDGE_OFFSET (0x700)
#define ATR_PCIE_BRIDGE_OFFSET(atr_index) (ATR0_PCIE_BRIDGE_OFFSET + (atr_index * 0x20))
#define MAXIMUM_APP_FIRMWARE_CODE_SIZE (0x40000)
#define MAXIMUM_CORE_FIRMWARE_CODE_SIZE (0x20000)
#define FIRMWARE_LOAD_WAIT_MAX_RETRIES (100)
#define FIRMWARE_LOAD_SLEEP_MS (50)
#define PCIE_REQUEST_SIZE_OFFSET (0x640)
#define PCIE_CONFIG_VENDOR_OFFSET (0x0098)
#define HAILO_PCIE_DMA_DEVICE_INTERRUPTS_BITMASK (1 << 4)
#define HAILO_PCIE_DMA_HOST_INTERRUPTS_BITMASK (1 << 5)
#define HAILO_PCIE_DMA_SRC_CHANNELS_BITMASK (0x0000FFFF)
#define HAILO_PCIE_MAX_ATR_TABLE_INDEX (3)
#define BOOT_STATUS_UNINITIALIZED (0x1)
#define PCIE_CONTROL_SECTION_ADDRESS_H8 (0x60000000)
#define PCIE_BLOCK_ADDRESS_ATR1 (0x200000)
#define PCIE_CONFIG_PCIE_CFG_QM_ROUTING_MODE_SET(reg_offset) \
(reg_offset) = (((reg_offset) & ~0x00000004L) | ((uint32_t)(1) << 2))
struct hailo_fw_addresses {
u32 boot_fw_header;
u32 app_fw_code_ram_base;
u32 boot_key_cert;
u32 boot_cont_cert;
u32 core_code_ram_base;
u32 core_fw_header;
u32 raise_ready_offset;
u32 boot_status;
u32 pcie_cfg_regs;
};
struct hailo_board_compatibility {
struct hailo_fw_addresses fw_addresses;
const struct hailo_pcie_loading_stage stages[MAX_LOADING_STAGES];
};
static const struct hailo_file_batch hailo10h_files_stg1[] = {
{
.filename = "hailo/hailo10h/customer_certificate.bin",
.address = 0xA0000,
.max_size = 0x8004,
.is_mandatory = true,
.has_header = false,
.has_core = false
},
{
.filename = "hailo/hailo10h/u-boot.dtb.signed",
.address = 0xA8004,
.max_size = 0x20000,
.is_mandatory = true,
.has_header = false,
.has_core = false
},
{
.filename = "hailo/hailo10h/scu_fw.bin",
.address = 0x20000,
.max_size = 0x40000,
.is_mandatory = true,
.has_header = true,
.has_core = false
},
{
.filename = NULL,
.address = 0x00,
.max_size = 0x00,
.is_mandatory = false,
.has_header = false,
.has_core = false
}
};
static const struct hailo_file_batch hailo10h_files_stg2[] = {
{
.filename = "hailo/hailo10h/u-boot-spl.bin",
.address = 0x85000000,
.max_size = 0x1000000,
.is_mandatory = true,
.has_header = false,
.has_core = false
},
{
.filename = "hailo/hailo10h/u-boot-tfa.itb",
.address = 0x86000000,
.max_size = 0x1000000,
.is_mandatory = true,
.has_header = false,
.has_core = false
},
{
.filename = "hailo/hailo10h/fitImage",
.address = 0x87000000,
.max_size = 0x1000000,
.is_mandatory = true,
.has_header = false,
.has_core = false
},
{
.filename = "hailo/hailo10h/image-fs",
#ifndef HAILO_EMULATOR
.address = 0x88000000,
#else
// TODO : HRT-15692 - merge two cases
.address = 0x89000000,
#endif /* ifndef HAILO_EMULATOR */
.max_size = 0x20000000, // Max size 512MB
.is_mandatory = true,
.has_header = false,
.has_core = false
}
};
// If loading linux from EMMC - only need few files from second batch (u-boot-spl.bin and u-boot-tfa.itb)
static const struct hailo_file_batch hailo10h_files_stg2_linux_in_emmc[] = {
{
.filename = "hailo/hailo10h/u-boot-spl.bin",
.address = 0x85000000,
.max_size = 0x1000000,
.is_mandatory = true,
.has_header = false,
.has_core = false
},
{
.filename = "hailo/hailo10h/u-boot-tfa.itb",
.address = 0x86000000,
.max_size = 0x1000000,
.is_mandatory = true,
.has_header = false,
.has_core = false
},
{
.filename = NULL,
.address = 0x00,
.max_size = 0x00,
.is_mandatory = false,
.has_header = false,
.has_core = false
},
};
static const struct hailo_file_batch hailo8_files_stg1[] = {
{
.filename = "hailo/hailo8_fw.bin",
.address = 0x20000,
.max_size = 0x50000,
.is_mandatory = true,
.has_header = true,
.has_core = true
},
{
.filename = "hailo/hailo8_board_cfg.bin",
.address = 0x60001000,
.max_size = PCIE_HAILO8_BOARD_CFG_MAX_SIZE,
.is_mandatory = false,
.has_header = false,
.has_core = false
},
{
.filename = "hailo/hailo8_fw_cfg.bin",
.address = 0x60001500,
.max_size = PCIE_HAILO8_FW_CFG_MAX_SIZE,
.is_mandatory = false,
.has_header = false,
.has_core = false
},
{
.filename = NULL,
.address = 0x00,
.max_size = 0x00,
.is_mandatory = false,
.has_header = false,
.has_core = false
}
};
static const struct hailo_file_batch hailo10h_legacy_files_stg1[] = {
{
.filename = "hailo/hailo15_fw.bin",
.address = 0x20000,
.max_size = 0x100000,
.is_mandatory = true,
.has_header = true,
.has_core = true
},
{
.filename = NULL,
.address = 0x00,
.max_size = 0x00,
.is_mandatory = false,
.has_header = false,
.has_core = false
}
};
// TODO HRT-15014 - Fix names for hailo15l legacy accelerator
static const struct hailo_file_batch hailo15l_files_stg1[] = {
{
.filename = "hailo/hailo15l_fw.bin",
.address = 0x20000,
.max_size = 0x100000,
.is_mandatory = true,
.has_header = true,
.has_core = true
},
{
.filename = NULL,
.address = 0x00,
.max_size = 0x00,
.is_mandatory = false,
.has_header = false,
.has_core = false
}
};
static const struct hailo_board_compatibility compat[HAILO_BOARD_TYPE_COUNT] = {
[HAILO_BOARD_TYPE_HAILO8] = {
.fw_addresses = {
.boot_fw_header = 0xE0030,
.boot_key_cert = 0xE0048,
.boot_cont_cert = 0xE0390,
.app_fw_code_ram_base = 0x60000,
.core_code_ram_base = 0xC0000,
.core_fw_header = 0xA0000,
.raise_ready_offset = 0x1684,
.boot_status = 0xe0000,
},
.stages = {
{
.batch = hailo8_files_stg1,
.trigger_address = 0xE0980,
.timeout = FIRMWARE_WAIT_TIMEOUT_MS,
.amount_of_files_in_stage = 3
},
},
},
[HAILO_BOARD_TYPE_HAILO10H_LEGACY] = {
.fw_addresses = {
.boot_fw_header = 0x88000,
.boot_key_cert = 0x88018,
.boot_cont_cert = 0x886a8,
.app_fw_code_ram_base = 0x20000,
.core_code_ram_base = 0x60000,
.core_fw_header = 0xC0000,
.raise_ready_offset = 0x1754,
.boot_status = 0x80000,
},
.stages = {
{
.batch = hailo10h_legacy_files_stg1,
.trigger_address = 0x88c98,
.timeout = FIRMWARE_WAIT_TIMEOUT_MS,
.amount_of_files_in_stage = 1
},
},
},
[HAILO_BOARD_TYPE_HAILO10H] = {
.fw_addresses = {
.boot_fw_header = 0x88000,
.boot_key_cert = 0x88018,
.boot_cont_cert = 0x886a8,
.app_fw_code_ram_base = 0x20000,
.core_code_ram_base = 0,
.core_fw_header = 0,
.raise_ready_offset = 0x1754,
.boot_status = 0x80000,
.pcie_cfg_regs = 0x002009dc,
},
.stages = {
{
.batch = hailo10h_files_stg1,
.trigger_address = 0x88c98,
.timeout = FIRMWARE_WAIT_TIMEOUT_MS,
.amount_of_files_in_stage = 3
},
{
.batch = hailo10h_files_stg2,
.trigger_address = 0x84000000,
.timeout = PCI_EP_WAIT_TIMEOUT_MS,
.amount_of_files_in_stage = 4
},
{
.batch = hailo10h_files_stg2_linux_in_emmc,
.trigger_address = 0x84000000,
.timeout = FIRMWARE_WAIT_TIMEOUT_MS,
.amount_of_files_in_stage = 2
},
},
},
// HRT-11344 : none of these matter except raise_ready_offset seeing as we load fw seperately - not through driver
// After implementing bootloader put correct values here
[HAILO_BOARD_TYPE_HAILO15L] = {
.fw_addresses = {
.boot_fw_header = 0x88000,
.boot_key_cert = 0x88018,
.boot_cont_cert = 0x886a8,
.app_fw_code_ram_base = 0x20000,
.core_code_ram_base = 0x60000,
.core_fw_header = 0xC0000,
// NOTE: After they update hw consts - check register fw_access_interrupt_w1s of pcie_config
.raise_ready_offset = 0x174c,
.boot_status = 0x80000,
},
.stages = {
{
.batch = hailo15l_files_stg1,
.trigger_address = 0x88c98,
.timeout = FIRMWARE_WAIT_TIMEOUT_MS,
.amount_of_files_in_stage = 1
},
},
}
};
const struct hailo_pcie_loading_stage *hailo_pcie_get_loading_stage_info(enum hailo_board_type board_type,
enum loading_stages stage)
{
return &compat[board_type].stages[stage];
}
static u32 read_and_clear_reg(struct hailo_resource *resource, u32 offset)
{
u32 value = hailo_resource_read32(resource, offset);
if (value != 0) {
hailo_resource_write32(resource, offset, value);
}
return value;
}
bool hailo_pcie_read_interrupt(struct hailo_pcie_resources *resources, struct hailo_pcie_interrupt_source *source)
{
u32 istatus_host = 0;
memset(source, 0, sizeof(*source));
istatus_host = read_and_clear_reg(&resources->config, BCS_ISTATUS_HOST);
if (0 == istatus_host) {
return false;
}
source->sw_interrupts = (istatus_host >> BCS_ISTATUS_HOST_SW_IRQ_SHIFT);
if (istatus_host & BCS_ISTATUS_HOST_VDMA_SRC_IRQ_MASK) {
source->vdma_channels_bitmap |= read_and_clear_reg(&resources->config, BCS_SOURCE_INTERRUPT_PER_CHANNEL);
}
if (istatus_host & BCS_ISTATUS_HOST_VDMA_DEST_IRQ_MASK) {
source->vdma_channels_bitmap |= read_and_clear_reg(&resources->config, BCS_DESTINATION_INTERRUPT_PER_CHANNEL);
}
return true;
}
int hailo_pcie_write_firmware_control(struct hailo_pcie_resources *resources, const struct hailo_fw_control *command)
{
int err = 0;
u32 request_size = 0;
u8 fw_access_value = FW_ACCESS_APP_CPU_CONTROL_MASK;
const struct hailo_fw_addresses *fw_addresses = &(compat[resources->board_type].fw_addresses);
if (!hailo_pcie_is_firmware_loaded(resources)) {
return -ENODEV;
}
// Copy md5 + buffer_len + buffer
request_size = sizeof(command->expected_md5) + sizeof(command->buffer_len) + command->buffer_len;
err = hailo_resource_write_buffer(&resources->fw_access, 0, PO2_ROUND_UP(request_size, FW_CODE_SECTION_ALIGNMENT),
command);
if (err < 0) {
return err;
}
// Raise the bit for the CPU that will handle the control
fw_access_value = (command->cpu_id == HAILO_CPU_ID_CPU1) ? FW_ACCESS_CORE_CPU_CONTROL_MASK :
FW_ACCESS_APP_CPU_CONTROL_MASK;
// Raise ready flag to FW
hailo_resource_write32(&resources->fw_access, fw_addresses->raise_ready_offset, (u32)fw_access_value);
return 0;
}
int hailo_pcie_read_firmware_control(struct hailo_pcie_resources *resources, struct hailo_fw_control *command)
{
u32 response_header_size = 0;
// Copy response md5 + buffer_len
response_header_size = sizeof(command->expected_md5) + sizeof(command->buffer_len);
hailo_resource_read_buffer(&resources->fw_access, PCIE_REQUEST_SIZE_OFFSET, response_header_size, command);
if (sizeof(command->buffer) < command->buffer_len) {
return -EINVAL;
}
// Copy response buffer
hailo_resource_read_buffer(&resources->fw_access, PCIE_REQUEST_SIZE_OFFSET + (size_t)response_header_size,
command->buffer_len, &command->buffer);
return 0;
}
void hailo_pcie_write_firmware_driver_shutdown(struct hailo_pcie_resources *resources)
{
const struct hailo_fw_addresses *fw_addresses = &(compat[resources->board_type].fw_addresses);
const u32 fw_access_value = FW_ACCESS_DRIVER_SHUTDOWN_MASK;
// Write shutdown flag to FW
hailo_resource_write32(&resources->fw_access, fw_addresses->raise_ready_offset, fw_access_value);
}
void hailo_pcie_write_firmware_soft_reset(struct hailo_pcie_resources *resources)
{
const struct hailo_fw_addresses *fw_addresses = &(compat[resources->board_type].fw_addresses);
const u32 fw_access_value = FW_ACCESS_SOFT_RESET_MASK;
// Write shutdown flag to FW
hailo_resource_write32(&resources->fw_access, fw_addresses->raise_ready_offset, fw_access_value);
}
int hailo_pcie_configure_atr_table(struct hailo_resource *bridge_config, u64 trsl_addr, u32 atr_index)
{
size_t offset = 0;
struct hailo_atr_config atr = {
.atr_param = (ATR_PARAM | (atr_index << 12)),
.atr_src = ATR_SRC_ADDR,
.atr_trsl_addr_1 = (u32)(trsl_addr & 0xFFFFFFFF),
.atr_trsl_addr_2 = (u32)(trsl_addr >> 32),
.atr_trsl_param = ATR_TRSL_PARAM
};
BUG_ON(HAILO_PCIE_MAX_ATR_TABLE_INDEX < atr_index);
offset = ATR_PCIE_BRIDGE_OFFSET(atr_index);
return hailo_resource_write_buffer(bridge_config, offset, sizeof(atr), (void*)&atr);
}
void hailo_pcie_read_atr_table(struct hailo_resource *bridge_config, struct hailo_atr_config *atr, u32 atr_index)
{
size_t offset = 0;
BUG_ON(HAILO_PCIE_MAX_ATR_TABLE_INDEX < atr_index);
offset = ATR_PCIE_BRIDGE_OFFSET(atr_index);
hailo_resource_read_buffer(bridge_config, offset, sizeof(*atr), (void*)atr);
}
static void write_memory_chunk(struct hailo_pcie_resources *resources,
hailo_ptr_t dest, u32 dest_offset, const void *src, u32 len)
{
u32 ATR_INDEX = 0;
BUG_ON(dest_offset + len > (u32)resources->fw_access.size);
(void)hailo_pcie_configure_atr_table(&resources->config, dest, ATR_INDEX);
(void)hailo_resource_write_buffer(&resources->fw_access, dest_offset, len, src);
}
static void read_memory_chunk(
struct hailo_pcie_resources *resources, hailo_ptr_t src, u32 src_offset, void *dest, u32 len)
{
u32 ATR_INDEX = 0;
BUG_ON(src_offset + len > (u32)resources->fw_access.size);
(void)hailo_pcie_configure_atr_table(&resources->config, src, ATR_INDEX);
(void)hailo_resource_read_buffer(&resources->fw_access, src_offset, len, dest);
}
// Note: this function modify the device ATR table (that is also used by the firmware for control and vdma).
// Use with caution, and restore the original atr if needed.
static void write_memory(struct hailo_pcie_resources *resources, hailo_ptr_t dest, const void *src, u32 len)
{
struct hailo_atr_config previous_atr = {0};
hailo_ptr_t base_address = (dest & ~ATR_TABLE_SIZE_MASK);
u32 chunk_len = 0;
u32 offset = 0;
u32 ATR_INDEX = 0;
// Store previous ATR (Read/write modify the ATR).
hailo_pcie_read_atr_table(&resources->config, &previous_atr, ATR_INDEX);
if (base_address != dest) {
// Data is not aligned, write the first chunk
chunk_len = min((u32)(base_address + ATR_TABLE_SIZE - dest), len);
write_memory_chunk(resources, base_address, (u32)(dest - base_address), src, chunk_len);
offset += chunk_len;
}
while (offset < len) {
chunk_len = min(len - offset, ATR_TABLE_SIZE);
write_memory_chunk(resources, dest + offset, 0, (const u8*)src + offset, chunk_len);
offset += chunk_len;
}
(void)hailo_pcie_configure_atr_table(&resources->config,
(((u64)(previous_atr.atr_trsl_addr_2) << 32) | previous_atr.atr_trsl_addr_1), ATR_INDEX);
}
// Note: this function modify the device ATR table (that is also used by the firmware for control and vdma).
// Use with caution, and restore the original atr if needed.
static void read_memory(struct hailo_pcie_resources *resources, hailo_ptr_t src, void *dest, u32 len)
{
struct hailo_atr_config previous_atr = {0};
hailo_ptr_t base_address = (src & ~ATR_TABLE_SIZE_MASK);
u32 chunk_len = 0;
u32 offset = 0;
u32 ATR_INDEX = 0;
// Store previous ATR (Read/write modify the ATR).
hailo_pcie_read_atr_table(&resources->config, &previous_atr, ATR_INDEX);
if (base_address != src) {
// Data is not aligned, read the first chunk
chunk_len = min((u32)(base_address + ATR_TABLE_SIZE - src), len);
read_memory_chunk(resources, base_address, (u32)(src - base_address), dest, chunk_len);
offset += chunk_len;
}
while (offset < len) {
chunk_len = min(len - offset, ATR_TABLE_SIZE);
read_memory_chunk(resources, src + offset, 0, (u8*)dest + offset, chunk_len);
offset += chunk_len;
}
(void)hailo_pcie_configure_atr_table(&resources->config,
(((u64)(previous_atr.atr_trsl_addr_2) << 32) | previous_atr.atr_trsl_addr_1), ATR_INDEX);
}
// Note: This function use for enabling the vDMA transaction host<->device by read modify write of the EP registers in the SOC - for fast boot over vDMA.
void hailo_pcie_configure_ep_registers_for_dma_transaction(struct hailo_pcie_resources *resources)
{
u32 reg_routing_mercury = 0;
BUG_ON(compat[resources->board_type].fw_addresses.pcie_cfg_regs == 0);
read_memory(resources, compat[resources->board_type].fw_addresses.pcie_cfg_regs, &reg_routing_mercury, sizeof(reg_routing_mercury));
PCIE_CONFIG_PCIE_CFG_QM_ROUTING_MODE_SET(reg_routing_mercury);
write_memory(resources, compat[resources->board_type].fw_addresses.pcie_cfg_regs, &reg_routing_mercury, sizeof(reg_routing_mercury));
}
static void hailo_write_app_firmware(struct hailo_pcie_resources *resources, firmware_header_t *fw_header,
secure_boot_certificate_header_t *fw_cert)
{
const struct hailo_fw_addresses *fw_addresses = &(compat[resources->board_type].fw_addresses);
u8 *fw_code = ((u8*)fw_header + sizeof(firmware_header_t));
u8 *key_data = ((u8*)fw_cert + sizeof(secure_boot_certificate_header_t));
u8 *content_data = key_data + fw_cert->key_size;
write_memory(resources, fw_addresses->boot_fw_header, fw_header, sizeof(firmware_header_t));
write_memory(resources, fw_addresses->app_fw_code_ram_base, fw_code, fw_header->code_size);
write_memory(resources, fw_addresses->boot_key_cert, key_data, fw_cert->key_size);
write_memory(resources, fw_addresses->boot_cont_cert, content_data, fw_cert->content_size);
}
static void hailo_write_core_firmware(struct hailo_pcie_resources *resources, firmware_header_t *fw_header)
{
const struct hailo_fw_addresses *fw_addresses = &(compat[resources->board_type].fw_addresses);
void *fw_code = (void*)((u8*)fw_header + sizeof(firmware_header_t));
write_memory(resources, fw_addresses->core_code_ram_base, fw_code, fw_header->code_size);
write_memory(resources, fw_addresses->core_fw_header, fw_header, sizeof(firmware_header_t));
}
void hailo_trigger_firmware_boot(struct hailo_pcie_resources *resources, u32 stage)
{
u32 pcie_finished = 1;
write_memory(resources, compat[resources->board_type].stages[stage].trigger_address, (void*)&pcie_finished, sizeof(pcie_finished));
}
u32 hailo_get_boot_status(struct hailo_pcie_resources *resources)
{
u32 boot_status = 0;
const struct hailo_fw_addresses *fw_addresses = &(compat[resources->board_type].fw_addresses);
read_memory(resources, fw_addresses->boot_status, &boot_status, sizeof(boot_status));
return boot_status;
}
/**
* Validates the FW headers.
* @param[in] address Address of the firmware.
* @param[in] firmware_size Size of the firmware.
* @param[out] out_app_firmware_header (optional) App firmware header
* @param[out] out_core_firmware_header (optional) Core firmware header
* @param[out] out_firmware_cert (optional) Firmware certificate header
*/
static int FW_VALIDATION__validate_fw_headers(uintptr_t firmware_base_address, size_t firmware_size,
firmware_header_t **out_app_firmware_header, firmware_header_t **out_core_firmware_header,
secure_boot_certificate_header_t **out_firmware_cert, enum hailo_board_type board_type)
{
firmware_header_t *app_firmware_header = NULL;
firmware_header_t *core_firmware_header = NULL;
secure_boot_certificate_header_t *firmware_cert = NULL;
int err = -EINVAL;
u32 consumed_firmware_offset = 0;
err = FW_VALIDATION__validate_fw_header(firmware_base_address, firmware_size, MAXIMUM_APP_FIRMWARE_CODE_SIZE,
&consumed_firmware_offset, &app_firmware_header, board_type);
if (0 != err) {
err = -EINVAL;
goto exit;
}
err = FW_VALIDATION__validate_cert_header(firmware_base_address, firmware_size,
&consumed_firmware_offset, &firmware_cert);
if (0 != err) {
err = -EINVAL;
goto exit;
}
// Not validating with HAILO10H since core firmware doesn't loaded over pcie
if (HAILO_BOARD_TYPE_HAILO10H != board_type) {
err = FW_VALIDATION__validate_fw_header(firmware_base_address, firmware_size, MAXIMUM_CORE_FIRMWARE_CODE_SIZE,
&consumed_firmware_offset, &core_firmware_header, board_type);
if (0 != err) {
err = -EINVAL;
goto exit;
}
}
if (consumed_firmware_offset != firmware_size) {
/* it is an error if there is leftover data after the last firmware header */
err = -EINVAL;
goto exit;
}
/* the out params are all optional */
if (NULL != out_app_firmware_header) {
*out_app_firmware_header = app_firmware_header;
}
if (NULL != out_firmware_cert) {
*out_firmware_cert = firmware_cert;
}
if (NULL != out_core_firmware_header) {
*out_core_firmware_header = core_firmware_header;
}
err = 0;
exit:
return err;
}
static int write_single_file(struct hailo_pcie_resources *resources, const struct hailo_file_batch *file_info, struct device *dev)
{
const struct firmware *firmware = NULL;
firmware_header_t *app_firmware_header = NULL;
secure_boot_certificate_header_t *firmware_cert = NULL;
firmware_header_t *core_firmware_header = NULL;
int err = 0;
err = request_firmware_direct(&firmware, file_info->filename, dev);
if (err < 0) {
return err;
}
if (firmware->size > file_info->max_size) {
release_firmware(firmware);
return -EFBIG;
}
if (file_info->has_header) {
err = FW_VALIDATION__validate_fw_headers((uintptr_t)firmware->data, firmware->size,
&app_firmware_header, &core_firmware_header, &firmware_cert, resources->board_type);
if (err < 0) {
release_firmware(firmware);
return err;
}
hailo_write_app_firmware(resources, app_firmware_header, firmware_cert);
if (file_info->has_core) {
hailo_write_core_firmware(resources, core_firmware_header);
}
} else {
write_memory(resources, file_info->address, (void*)firmware->data, firmware->size);
}
release_firmware(firmware);
return 0;
}
int hailo_pcie_write_firmware_batch(struct device *dev, struct hailo_pcie_resources *resources, u32 stage)
{
const struct hailo_pcie_loading_stage *stage_info = hailo_pcie_get_loading_stage_info(resources->board_type, stage);
const struct hailo_file_batch *files_batch = stage_info->batch;
const u8 amount_of_files = stage_info->amount_of_files_in_stage;
int file_index = 0;
int err = 0;
for (file_index = 0; file_index < amount_of_files; file_index++)
{
dev_notice(dev, "Writing file %s\n", files_batch[file_index].filename);
err = write_single_file(resources, &files_batch[file_index], dev);
if (err < 0) {
pr_warn("Failed to write file %s\n", files_batch[file_index].filename);
if (files_batch[file_index].is_mandatory) {
return err;
}
}
dev_notice(dev, "File %s written successfully\n", files_batch[file_index].filename);
}
hailo_trigger_firmware_boot(resources, stage);
return 0;
}
bool hailo_pcie_is_firmware_loaded(struct hailo_pcie_resources *resources)
{
u32 offset;
u32 atr_value;
if (HAILO_BOARD_TYPE_HAILO8 == resources->board_type) {
offset = ATR_PCIE_BRIDGE_OFFSET(0) + offsetof(struct hailo_atr_config, atr_trsl_addr_1);
atr_value = hailo_resource_read32(&resources->config, offset);
return (PCIE_CONTROL_SECTION_ADDRESS_H8 == atr_value);
}
else {
offset = ATR_PCIE_BRIDGE_OFFSET(1) + offsetof(struct hailo_atr_config, atr_trsl_addr_1);
atr_value = hailo_resource_read32(&resources->config, offset);
return (PCIE_BLOCK_ADDRESS_ATR1 == atr_value);
}
}
bool hailo_pcie_wait_for_firmware(struct hailo_pcie_resources *resources)
{
size_t retries;
for (retries = 0; retries < FIRMWARE_LOAD_WAIT_MAX_RETRIES; retries++) {
if (hailo_pcie_is_firmware_loaded(resources)) {
return true;
}
msleep(FIRMWARE_LOAD_SLEEP_MS);
}
return false;
}
void hailo_pcie_update_channel_interrupts_mask(struct hailo_pcie_resources* resources, u32 channels_bitmap)
{
size_t i = 0;
u32 mask = hailo_resource_read32(&resources->config, BSC_IMASK_HOST);
// Clear old channel interrupts
mask &= ~BCS_ISTATUS_HOST_VDMA_SRC_IRQ_MASK;
mask &= ~BCS_ISTATUS_HOST_VDMA_DEST_IRQ_MASK;
// Set interrupt by the bitmap
for (i = 0; i < MAX_VDMA_CHANNELS_PER_ENGINE; ++i) {
if (hailo_test_bit(i, &channels_bitmap)) {
// based on 18.5.2 "vDMA Interrupt Registers" in PLDA documentation
u32 offset = (i & 16) ? 8 : 0;
hailo_set_bit((((int)i*8) / MAX_VDMA_CHANNELS_PER_ENGINE) + offset, &mask);
}
}
hailo_resource_write32(&resources->config, BSC_IMASK_HOST, mask);
}
void hailo_pcie_enable_interrupts(struct hailo_pcie_resources *resources)
{
u32 mask = hailo_resource_read32(&resources->config, BSC_IMASK_HOST);
hailo_resource_write32(&resources->config, BCS_ISTATUS_HOST, 0xFFFFFFFF);
hailo_resource_write32(&resources->config, BCS_DESTINATION_INTERRUPT_PER_CHANNEL, 0xFFFFFFFF);
hailo_resource_write32(&resources->config, BCS_SOURCE_INTERRUPT_PER_CHANNEL, 0xFFFFFFFF);
mask |= BCS_ISTATUS_HOST_SW_IRQ_MASK;
hailo_resource_write32(&resources->config, BSC_IMASK_HOST, mask);
}
void hailo_pcie_disable_interrupts(struct hailo_pcie_resources* resources)
{
hailo_resource_write32(&resources->config, BSC_IMASK_HOST, 0);
}
static int direct_memory_transfer(struct hailo_pcie_resources *resources,
struct hailo_memory_transfer_params *params)
{
switch (params->transfer_direction) {
case TRANSFER_READ:
read_memory(resources, params->address, params->buffer, (u32)params->count);
break;
case TRANSFER_WRITE:
write_memory(resources, params->address, params->buffer, (u32)params->count);
break;
default:
return -EINVAL;
}
return 0;
}
int hailo_pcie_memory_transfer(struct hailo_pcie_resources *resources, struct hailo_memory_transfer_params *params)
{
if (params->count > ARRAY_SIZE(params->buffer)) {
return -EINVAL;
}
switch (params->memory_type) {
case HAILO_TRANSFER_DEVICE_DIRECT_MEMORY:
return direct_memory_transfer(resources, params);
case HAILO_TRANSFER_MEMORY_PCIE_BAR0:
return hailo_resource_transfer(&resources->config, params);
case HAILO_TRANSFER_MEMORY_PCIE_BAR2:
case HAILO_TRANSFER_MEMORY_VDMA0:
return hailo_resource_transfer(&resources->vdma_registers, params);
case HAILO_TRANSFER_MEMORY_PCIE_BAR4:
return hailo_resource_transfer(&resources->fw_access, params);
default:
return -EINVAL;
}
}
bool hailo_pcie_is_device_connected(struct hailo_pcie_resources *resources)
{
return PCI_VENDOR_ID_HAILO == hailo_resource_read16(&resources->config, PCIE_CONFIG_VENDOR_OFFSET);
}
int hailo_set_device_type(struct hailo_pcie_resources *resources)
{
switch(resources->board_type) {
case HAILO_BOARD_TYPE_HAILO8:
case HAILO_BOARD_TYPE_HAILO10H_LEGACY:
case HAILO_BOARD_TYPE_HAILO15L:
resources->accelerator_type = HAILO_ACCELERATOR_TYPE_NNC;
break;
case HAILO_BOARD_TYPE_HAILO10H:
resources->accelerator_type = HAILO_ACCELERATOR_TYPE_SOC;
break;
default:
return -EINVAL;
}
return 0;
}
// On PCIe, just return the start address
u64 hailo_pcie_encode_desc_dma_address_range(dma_addr_t dma_address_start, dma_addr_t dma_address_end, u32 step, u8 channel_id)
{
(void)channel_id;
(void)dma_address_end;
(void)step;
return (u64)dma_address_start;
}
struct hailo_vdma_hw hailo_pcie_vdma_hw = {
.hw_ops = {
.encode_desc_dma_address_range = hailo_pcie_encode_desc_dma_address_range,
},
.ddr_data_id = HAILO_PCIE_HOST_DMA_DATA_ID,
.device_interrupts_bitmask = HAILO_PCIE_DMA_DEVICE_INTERRUPTS_BITMASK,
.host_interrupts_bitmask = HAILO_PCIE_DMA_HOST_INTERRUPTS_BITMASK,
.src_channels_bitmask = HAILO_PCIE_DMA_SRC_CHANNELS_BITMASK,
};
void hailo_pcie_soc_write_request(struct hailo_pcie_resources *resources,
const struct hailo_pcie_soc_request *request)
{
const struct hailo_fw_addresses *fw_addresses = &(compat[resources->board_type].fw_addresses);
BUILD_BUG_ON_MSG((sizeof(*request) % sizeof(u32)) != 0, "Request must be a multiple of 4 bytes");
hailo_resource_write_buffer(&resources->fw_access, 0, sizeof(*request), (void*)request);
hailo_resource_write32(&resources->fw_access, fw_addresses->raise_ready_offset, FW_ACCESS_SOC_CONTROL_MASK);
}
void hailo_pcie_soc_read_response(struct hailo_pcie_resources *resources,
struct hailo_pcie_soc_response *response)
{
BUILD_BUG_ON_MSG((sizeof(*response) % sizeof(u32)) != 0, "Request must be a multiple of 4 bytes");
hailo_resource_read_buffer(&resources->fw_access, 0, sizeof(*response), response);
}

193
drivers/media/pci/hailo/common/pcie_common.h
View File

@@ -1,193 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_COMMON_PCIE_COMMON_H_
#define _HAILO_COMMON_PCIE_COMMON_H_
#include "hailo_resource.h"
#include "hailo_ioctl_common.h"
#include "fw_validation.h"
#include "fw_operation.h"
#include "utils.h"
#include "vdma_common.h"
#include "soc_structs.h"
#include <linux/types.h>
#include <linux/firmware.h>
#define BCS_ISTATUS_HOST_SW_IRQ_MASK (0xFF000000)
#define BCS_ISTATUS_HOST_SW_IRQ_SHIFT (24)
#define BCS_ISTATUS_HOST_VDMA_SRC_IRQ_MASK (0x000000FF)
#define BCS_ISTATUS_HOST_VDMA_DEST_IRQ_MASK (0x0000FF00)
#define PCIE_HAILO8_BOARD_CFG_MAX_SIZE (0x500)
#define PCIE_HAILO8_FW_CFG_MAX_SIZE (0x500)
#define FW_CODE_SECTION_ALIGNMENT (4)
#define HAILO_PCIE_CONFIG_BAR (0)
#define HAILO_PCIE_VDMA_REGS_BAR (2)
#define HAILO_PCIE_FW_ACCESS_BAR (4)
#define HAILO_PCIE_DMA_ENGINES_COUNT (1)
#define PCI_VDMA_ENGINE_INDEX (0)
#define MAX_FILES_PER_STAGE (4)
#define HAILO_PCIE_HOST_DMA_DATA_ID (0)
#define HAILO_PCI_EP_HOST_DMA_DATA_ID (6)
#define DRIVER_NAME "hailo"
#define PCI_VENDOR_ID_HAILO 0x1e60
#define PCI_DEVICE_ID_HAILO_HAILO8 0x2864
#define PCI_DEVICE_ID_HAILO_HAILO10H 0x45C4
#define PCI_DEVICE_ID_HAILO_HAILO15L 0x43a2
typedef u64 hailo_ptr_t;
struct hailo_pcie_resources {
struct hailo_resource config; // BAR0
struct hailo_resource vdma_registers; // BAR2
struct hailo_resource fw_access; // BAR4
enum hailo_board_type board_type;
enum hailo_accelerator_type accelerator_type;
};
struct hailo_atr_config {
u32 atr_param;
u32 atr_src;
u32 atr_trsl_addr_1;
u32 atr_trsl_addr_2;
u32 atr_trsl_param;
};
enum loading_stages {
FIRST_STAGE = 0,
SECOND_STAGE = 1,
SECOND_STAGE_LINUX_IN_EMMC = 2,
MAX_LOADING_STAGES = 3
};
enum hailo_pcie_nnc_sw_interrupt_masks {
HAILO_PCIE_NNC_FW_NOTIFICATION_IRQ = 0x2,
HAILO_PCIE_NNC_FW_CONTROL_IRQ = 0x4,
HAILO_PCIE_NNC_DRIVER_DOWN_IRQ = 0x8,
};
enum hailo_pcie_soc_sw_interrupt_masks {
HAILO_PCIE_SOC_CONTROL_IRQ = 0x10,
HAILO_PCIE_SOC_CLOSE_IRQ = 0x20,
};
enum hailo_pcie_boot_interrupt_masks {
HAILO_PCIE_BOOT_SOFT_RESET_IRQ = 0x1,
HAILO_PCIE_BOOT_IRQ = 0x2,
};
struct hailo_pcie_interrupt_source {
u32 sw_interrupts;
u32 vdma_channels_bitmap;
};
struct hailo_file_batch {
const char *filename;
u32 address;
size_t max_size;
bool is_mandatory;
bool has_header;
bool has_core;
};
struct hailo_pcie_loading_stage {
const struct hailo_file_batch *batch;
u32 trigger_address;
u32 timeout;
u8 amount_of_files_in_stage;
};
// TODO: HRT-6144 - Align Windows/Linux to QNX
#ifdef __QNX__
enum hailo_bar_index {
BAR0 = 0,
BAR2,
BAR4,
MAX_BAR
};
#else
enum hailo_bar_index {
BAR0 = 0,
BAR1,
BAR2,
BAR3,
BAR4,
BAR5,
MAX_BAR
};
#endif // ifdef (__QNX__)
#ifdef __cplusplus
extern "C" {
#endif
#ifndef HAILO_EMULATOR
#define TIME_UNTIL_REACH_BOOTLOADER (10)
#define PCI_EP_WAIT_TIMEOUT_MS (40000)
#define FIRMWARE_WAIT_TIMEOUT_MS (5000)
#else /* ifndef HAILO_EMULATOR */
// PCI EP timeout is defined to 50000000 because on Emulator the boot time + linux init time can be very long (4+ hours)
#define TIME_UNTIL_REACH_BOOTLOADER (10000)
#define PCI_EP_WAIT_TIMEOUT_MS (50000000)
#define FIRMWARE_WAIT_TIMEOUT_MS (5000000)
#endif /* ifndef HAILO_EMULATOR */
extern struct hailo_vdma_hw hailo_pcie_vdma_hw;
const struct hailo_pcie_loading_stage* hailo_pcie_get_loading_stage_info(enum hailo_board_type board_type,
enum loading_stages stage);
// Reads the interrupt source from BARs, return false if there is no interrupt.
// note - this function clears the interrupt signals.
bool hailo_pcie_read_interrupt(struct hailo_pcie_resources *resources, struct hailo_pcie_interrupt_source *source);
void hailo_pcie_update_channel_interrupts_mask(struct hailo_pcie_resources *resources, u32 channels_bitmap);
void hailo_pcie_enable_interrupts(struct hailo_pcie_resources *resources);
void hailo_pcie_disable_interrupts(struct hailo_pcie_resources *resources);
int hailo_pcie_write_firmware_control(struct hailo_pcie_resources *resources, const struct hailo_fw_control *command);
int hailo_pcie_read_firmware_control(struct hailo_pcie_resources *resources, struct hailo_fw_control *command);
int hailo_pcie_write_firmware_batch(struct device *dev, struct hailo_pcie_resources *resources, u32 stage);
bool hailo_pcie_is_firmware_loaded(struct hailo_pcie_resources *resources);
bool hailo_pcie_wait_for_firmware(struct hailo_pcie_resources *resources);
int hailo_pcie_memory_transfer(struct hailo_pcie_resources *resources, struct hailo_memory_transfer_params *params);
bool hailo_pcie_is_device_connected(struct hailo_pcie_resources *resources);
void hailo_pcie_write_firmware_driver_shutdown(struct hailo_pcie_resources *resources);
void hailo_pcie_write_firmware_soft_reset(struct hailo_pcie_resources *resources);
void hailo_pcie_configure_ep_registers_for_dma_transaction(struct hailo_pcie_resources *resources);
void hailo_trigger_firmware_boot(struct hailo_pcie_resources *resources, u32 stage);
int hailo_set_device_type(struct hailo_pcie_resources *resources);
u32 hailo_get_boot_status(struct hailo_pcie_resources *resources);
int hailo_pcie_configure_atr_table(struct hailo_resource *bridge_config, u64 trsl_addr, u32 atr_index);
void hailo_pcie_read_atr_table(struct hailo_resource *bridge_config, struct hailo_atr_config *atr, u32 atr_index);
u64 hailo_pcie_encode_desc_dma_address_range(dma_addr_t dma_address_start, dma_addr_t dma_address_end, u32 step, u8 channel_id);
void hailo_pcie_soc_write_request(struct hailo_pcie_resources *resources,
const struct hailo_pcie_soc_request *request);
void hailo_pcie_soc_read_response(struct hailo_pcie_resources *resources,
struct hailo_pcie_soc_response *response);
#ifdef __cplusplus
}
#endif
#endif /* _HAILO_COMMON_PCIE_COMMON_H_ */

79
drivers/media/pci/hailo/common/soc_structs.h
View File

@@ -1,79 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
/**
* Contains definitions for pcie soc to pcie ep communication
*/
#ifndef __HAILO_COMMON_SOC_STRUCTS__
#define __HAILO_COMMON_SOC_STRUCTS__
#include <linux/types.h>
#pragma pack(push, 1)
struct hailo_pcie_soc_connect_request {
u16 port;
};
struct hailo_pcie_soc_connect_response {
u8 input_channel_index;
u8 output_channel_index;
};
struct hailo_pcie_soc_close_request {
u32 channels_bitmap;
};
struct hailo_pcie_soc_close_response {
u8 reserved;
};
enum hailo_pcie_soc_control_code {
// Start from big initial value to ensure the right code was used (using 0
// as initiale may cause confusion if the code was not set correctly).
HAILO_PCIE_SOC_CONTROL_CODE_CONNECT = 0x100,
HAILO_PCIE_SOC_CONTROL_CODE_CLOSE,
HAILO_PCIE_SOC_CONTROL_CODE_INVALID,
};
#define HAILO_PCIE_SOC_MAX_REQUEST_SIZE_BYTES (16)
#define HAILO_PCIE_SOC_MAX_RESPONSE_SIZE_BYTES (16)
// IRQ to signal the PCIe that the EP was closed/released
#define PCI_EP_SOC_CLOSED_IRQ (0x00000020)
#define PCI_EP_SOC_CONNECT_RESPONSE (0x00000010)
struct hailo_pcie_soc_request {
u32 control_code;
union {
struct hailo_pcie_soc_connect_request connect;
struct hailo_pcie_soc_close_request close;
u8 pad[HAILO_PCIE_SOC_MAX_REQUEST_SIZE_BYTES];
};
};
struct hailo_pcie_soc_response {
u32 control_code;
s32 status;
union {
struct hailo_pcie_soc_connect_response connect;
struct hailo_pcie_soc_close_response close;
u8 pad[HAILO_PCIE_SOC_MAX_RESPONSE_SIZE_BYTES];
};
};
#pragma pack(pop)
// Compile time validate function. Don't need to call it.
static inline void __validate_soc_struct_sizes(void)
{
BUILD_BUG_ON_MSG(sizeof(struct hailo_pcie_soc_request) !=
sizeof(u32) + HAILO_PCIE_SOC_MAX_REQUEST_SIZE_BYTES, "Invalid request size");
BUILD_BUG_ON_MSG(sizeof(struct hailo_pcie_soc_response) !=
sizeof(u32) + sizeof(s32) + HAILO_PCIE_SOC_MAX_RESPONSE_SIZE_BYTES, "Invalid response size");
}
#endif /* __HAILO_COMMON_SOC_STRUCTS__ */

82
drivers/media/pci/hailo/common/utils.h
View File

@@ -1,82 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_DRIVER_UTILS_H_
#define _HAILO_DRIVER_UTILS_H_
#include <linux/bitops.h>
#define DWORD_SIZE (4)
#define WORD_SIZE (2)
#define BYTE_SIZE (1)
#define BITS_IN_BYTE (8)
#define hailo_clear_bit(bit, pval) { *(pval) &= ~(1 << bit); }
#define hailo_test_bit(pos,var_addr) ((*var_addr) & (1<<(pos)))
#define READ_BITS_AT_OFFSET(amount_bits, offset, initial_value) \
(((initial_value) >> (offset)) & ((1 << (amount_bits)) - 1))
#define WRITE_BITS_AT_OFFSET(amount_bits, offset, initial_value, value) \
(((initial_value) & ~(((1 << (amount_bits)) - 1) << (offset))) | \
(((value) & ((1 << (amount_bits)) - 1)) << (offset)))
#ifdef __cplusplus
extern "C"
{
#endif
static inline bool is_powerof2(size_t v) {
// bit trick
return (v & (v - 1)) == 0;
}
static inline void hailo_set_bit(int nr, u32* addr) {
u32 mask = BIT_MASK(nr);
u32 *p = addr + BIT_WORD(nr);
*p |= mask;
}
static inline uint8_t ceil_log2(uint32_t n)
{
uint8_t result = 0;
if (n <= 1) {
return 0;
}
while (n > 1) {
result++;
n = (n + 1) >> 1;
}
return result;
}
// Gets the nearest power of 2 >= value, for any value <= MAX_POWER_OF_2_VALUE. Otherwise POWER_OF_2_ERROR is returned.
#define MAX_POWER_OF_2_VALUE (0x80000000)
#define POWER_OF_2_ERROR ((uint32_t)-1)
static inline uint32_t get_nearest_powerof_2(uint32_t value)
{
uint32_t power_of_2 = 1;
if (value > MAX_POWER_OF_2_VALUE) {
return POWER_OF_2_ERROR;
}
while (value > power_of_2) {
power_of_2 <<= 1;
}
return power_of_2;
}
#ifndef DIV_ROUND_UP
#define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
#endif
#ifdef __cplusplus
}
#endif
#endif // _HAILO_DRIVER_UTILS_H_

876
drivers/media/pci/hailo/common/vdma_common.c
View File

@@ -1,876 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#include "vdma_common.h"
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/bug.h>
#include <linux/circ_buf.h>
#include <linux/ktime.h>
#include <linux/timekeeping.h>
#include <linux/kernel.h>
#include <linux/kconfig.h>
#include <linux/printk.h>
#define VDMA_CHANNEL_CONTROL_START (0x1)
#define VDMA_CHANNEL_CONTROL_ABORT (0b00)
#define VDMA_CHANNEL_CONTROL_ABORT_PAUSE (0b10)
#define VDMA_CHANNEL_CONTROL_START_ABORT_PAUSE_RESUME_BITMASK (0x3)
#define VDMA_CHANNEL_CONTROL_START_ABORT_BITMASK (0x1)
#define VDMA_CHANNEL_CONTROL_MASK (0xFC)
#define VDMA_CHANNEL_CONTROL_START_RESUME (0b01)
#define VDMA_CHANNEL_CONTROL_START_PAUSE (0b11)
#define VDMA_CHANNEL_CONTROL_ABORT (0b00)
#define VDMA_CHANNEL_CONTROL_ABORT_PAUSE (0b10)
#define VDMA_CHANNEL_CONTROL_START_ABORT_PAUSE_RESUME_BITMASK (0x3)
#define VDMA_CHANNEL_DESC_DEPTH_WIDTH (4)
#define VDMA_CHANNEL_DESC_DEPTH_SHIFT (11)
#define VDMA_CHANNEL_DATA_ID_SHIFT (8)
#define VDMA_CHANNEL__MAX_CHECKS_CHANNEL_IS_IDLE (10000)
#define VDMA_CHANNEL__ADDRESS_L_OFFSET (0x0A)
#define VDMA_CHANNEL__ALIGNED_ADDRESS_L_OFFSET (0x8)
#define VDMA_CHANNEL__ADDRESS_H_OFFSET (0x0C)
#define DESCRIPTOR_PAGE_SIZE_SHIFT (8)
#define DESCRIPTOR_DESC_CONTROL (0x2)
#define DESCRIPTOR_ADDR_L_MASK (0xFFFFFFC0)
#define DESCRIPTOR_LIST_MAX_DEPTH (16)
#define DESCRIPTOR_DESC_STATUS_DONE_BIT (0x0)
#define DESCRIPTOR_DESC_STATUS_ERROR_BIT (0x1)
#define DESCRIPTOR_DESC_STATUS_MASK (0xFF)
#define DESC_STATUS_REQ (1 << 0)
#define DESC_STATUS_REQ_ERR (1 << 1)
#define DESC_REQUEST_IRQ_PROCESSED (1 << 2)
#define DESC_REQUEST_IRQ_ERR (1 << 3)
#define VDMA_CHANNEL_NUM_PROCESSED_WIDTH (16)
#define VDMA_CHANNEL_NUM_PROCESSED_MASK ((1 << VDMA_CHANNEL_NUM_PROCESSED_WIDTH) - 1)
#define VDMA_CHANNEL_NUM_ONGOING_MASK VDMA_CHANNEL_NUM_PROCESSED_MASK
#define TIMESTAMPS_CIRC_SPACE(timestamp_list) \
CIRC_SPACE((timestamp_list).head, (timestamp_list).tail, CHANNEL_IRQ_TIMESTAMPS_SIZE)
#define TIMESTAMPS_CIRC_CNT(timestamp_list) \
CIRC_CNT((timestamp_list).head, (timestamp_list).tail, CHANNEL_IRQ_TIMESTAMPS_SIZE)
#define ONGOING_TRANSFERS_CIRC_SPACE(transfers_list) \
CIRC_SPACE((transfers_list).head, (transfers_list).tail, HAILO_VDMA_MAX_ONGOING_TRANSFERS)
#define ONGOING_TRANSFERS_CIRC_CNT(transfers_list) \
CIRC_CNT((transfers_list).head, (transfers_list).tail, HAILO_VDMA_MAX_ONGOING_TRANSFERS)
#ifndef for_each_sgtable_dma_sg
#define for_each_sgtable_dma_sg(sgt, sg, i) \
for_each_sg((sgt)->sgl, sg, (sgt)->nents, i)
#endif /* for_each_sgtable_dma_sg */
static int ongoing_transfer_push(struct hailo_vdma_channel *channel,
struct hailo_ongoing_transfer *ongoing_transfer)
{
struct hailo_ongoing_transfers_list *transfers = &channel->ongoing_transfers;
if (!ONGOING_TRANSFERS_CIRC_SPACE(*transfers)) {
return -EFAULT;
}
if (ongoing_transfer->dirty_descs_count > ARRAY_SIZE(ongoing_transfer->dirty_descs)) {
return -EFAULT;
}
transfers->transfers[transfers->head] = *ongoing_transfer;
transfers->head = (transfers->head + 1) & HAILO_VDMA_MAX_ONGOING_TRANSFERS_MASK;
return 0;
}
static int ongoing_transfer_pop(struct hailo_vdma_channel *channel,
struct hailo_ongoing_transfer *ongoing_transfer)
{
struct hailo_ongoing_transfers_list *transfers = &channel->ongoing_transfers;
if (!ONGOING_TRANSFERS_CIRC_CNT(*transfers)) {
return -EFAULT;
}
if (ongoing_transfer) {
*ongoing_transfer = transfers->transfers[transfers->tail];
}
transfers->tail = (transfers->tail + 1) & HAILO_VDMA_MAX_ONGOING_TRANSFERS_MASK;
return 0;
}
static void clear_dirty_desc(struct hailo_vdma_descriptors_list *desc_list, u16 desc)
{
desc_list->desc_list[desc].PageSize_DescControl =
(u32)((desc_list->desc_page_size << DESCRIPTOR_PAGE_SIZE_SHIFT) + DESCRIPTOR_DESC_CONTROL);
}
static void clear_dirty_descs(struct hailo_vdma_channel *channel,
struct hailo_ongoing_transfer *ongoing_transfer)
{
u8 i = 0;
struct hailo_vdma_descriptors_list *desc_list = channel->last_desc_list;
BUG_ON(ongoing_transfer->dirty_descs_count > ARRAY_SIZE(ongoing_transfer->dirty_descs));
for (i = 0; i < ongoing_transfer->dirty_descs_count; i++) {
clear_dirty_desc(desc_list, ongoing_transfer->dirty_descs[i]);
}
}
static bool validate_last_desc_status(struct hailo_vdma_channel *channel,
struct hailo_ongoing_transfer *ongoing_transfer)
{
u16 last_desc = ongoing_transfer->last_desc;
u32 last_desc_control = channel->last_desc_list->desc_list[last_desc].RemainingPageSize_Status &
DESCRIPTOR_DESC_STATUS_MASK;
if (!hailo_test_bit(DESCRIPTOR_DESC_STATUS_DONE_BIT, &last_desc_control)) {
pr_err("Expecting desc %d to be done\n", last_desc);
return false;
}
if (hailo_test_bit(DESCRIPTOR_DESC_STATUS_ERROR_BIT, &last_desc_control)) {
pr_err("Got unexpected error on desc %d\n", last_desc);
return false;
}
return true;
}
static void hailo_vdma_program_descriptor(struct hailo_vdma_descriptor *descriptor, u64 dma_address, size_t page_size,
u8 data_id)
{
descriptor->PageSize_DescControl = (u32)((page_size << DESCRIPTOR_PAGE_SIZE_SHIFT) +
DESCRIPTOR_DESC_CONTROL);
descriptor->AddrL_rsvd_DataID = (u32)(((dma_address & DESCRIPTOR_ADDR_L_MASK)) | data_id);
descriptor->AddrH = (u32)(dma_address >> 32);
descriptor->RemainingPageSize_Status = 0 ;
}
static u8 get_channel_id(u8 channel_index)
{
return (channel_index < MAX_VDMA_CHANNELS_PER_ENGINE) ? (channel_index & 0xF) : INVALID_VDMA_CHANNEL;
}
int hailo_vdma_program_descriptors_in_chunk(
struct hailo_vdma_hw *vdma_hw,
dma_addr_t chunk_addr,
unsigned int chunk_size,
struct hailo_vdma_descriptors_list *desc_list,
u32 desc_index,
u32 max_desc_index,
u8 channel_index,
u8 data_id)
{
const u16 page_size = desc_list->desc_page_size;
const u32 descs_to_program = DIV_ROUND_UP(chunk_size, page_size);
const u32 starting_desc_index = desc_index;
const u32 residue_size = chunk_size % page_size;
struct hailo_vdma_descriptor *dma_desc = NULL;
u64 encoded_addr = 0;
if (descs_to_program == 0) {
// Nothing to program
return 0;
}
// We iterate through descriptors [desc_index, desc_index + descs_to_program)
if (desc_index + descs_to_program > max_desc_index + 1) {
return -ERANGE;
}
encoded_addr = vdma_hw->hw_ops.encode_desc_dma_address_range(chunk_addr, chunk_addr + chunk_size, page_size, get_channel_id(channel_index));
if (INVALID_VDMA_ADDRESS == encoded_addr) {
return -EFAULT;
}
// Program all descriptors except the last one
for (desc_index = starting_desc_index; desc_index < starting_desc_index + descs_to_program - 1; desc_index++) {
// 'desc_index & desc_list_len_mask' is used instead of modulo; see hailo_vdma_descriptors_list documentation.
hailo_vdma_program_descriptor(
&desc_list->desc_list[desc_index & desc_list->desc_count_mask],
encoded_addr, page_size, data_id);
encoded_addr += page_size;
}
// Handle the last descriptor outside of the loop
// 'desc_index & desc_list_len_mask' is used instead of modulo; see hailo_vdma_descriptors_list documentation.
dma_desc = &desc_list->desc_list[desc_index & desc_list->desc_count_mask];
hailo_vdma_program_descriptor(dma_desc, encoded_addr,
(residue_size == 0) ? page_size : (u16)residue_size, data_id);
return (int)descs_to_program;
}
static unsigned long get_interrupts_bitmask(struct hailo_vdma_hw *vdma_hw,
enum hailo_vdma_interrupts_domain interrupts_domain, bool is_debug)
{
unsigned long bitmask = 0;
if (0 != (HAILO_VDMA_INTERRUPTS_DOMAIN_DEVICE & interrupts_domain)) {
bitmask |= vdma_hw->device_interrupts_bitmask;
}
if (0 != (HAILO_VDMA_INTERRUPTS_DOMAIN_HOST & interrupts_domain)) {
bitmask |= vdma_hw->host_interrupts_bitmask;
}
if (bitmask != 0) {
bitmask |= DESC_REQUEST_IRQ_PROCESSED | DESC_REQUEST_IRQ_ERR;
if (is_debug) {
bitmask |= DESC_STATUS_REQ | DESC_STATUS_REQ_ERR;
}
}
return bitmask;
}
static int bind_and_program_descriptors_list(
struct hailo_vdma_hw *vdma_hw,
struct hailo_vdma_descriptors_list *desc_list,
u32 starting_desc,
struct hailo_vdma_mapped_transfer_buffer *buffer,
u8 channel_index,
enum hailo_vdma_interrupts_domain last_desc_interrupts,
bool is_debug)
{
int desc_programmed = 0;
int descs_programmed_in_chunk = 0;
u32 max_desc_index = 0;
u32 chunk_size = 0;
struct scatterlist *sg_entry = NULL;
unsigned int i = 0;
size_t buffer_current_offset = 0;
dma_addr_t chunk_start_addr = 0;
u32 program_size = buffer->size;
if (starting_desc >= desc_list->desc_count) {
return -EFAULT;
}
if (buffer->offset % desc_list->desc_page_size != 0) {
return -EFAULT;
}
// On circular buffer, allow programming desc_count descriptors (starting
// from starting_desc). On non circular, don't allow is to pass desc_count
max_desc_index = desc_list->is_circular ?
starting_desc + desc_list->desc_count - 1 :
desc_list->desc_count - 1;
for_each_sgtable_dma_sg(buffer->sg_table, sg_entry, i) {
// Skip sg entries until we reach the right buffer offset. offset can be in the middle of an sg entry.
if (buffer_current_offset + sg_dma_len(sg_entry) < buffer->offset) {
buffer_current_offset += sg_dma_len(sg_entry);
continue;
}
chunk_start_addr = (buffer_current_offset < buffer->offset) ?
sg_dma_address(sg_entry) + (buffer->offset - buffer_current_offset) :
sg_dma_address(sg_entry);
chunk_size = (buffer_current_offset < buffer->offset) ?
(u32)(sg_dma_len(sg_entry) - (buffer->offset - buffer_current_offset)) :
(u32)(sg_dma_len(sg_entry));
chunk_size = min((u32)program_size, chunk_size);
descs_programmed_in_chunk = hailo_vdma_program_descriptors_in_chunk(vdma_hw, chunk_start_addr, chunk_size, desc_list,
starting_desc, max_desc_index, channel_index, vdma_hw->ddr_data_id);
if (descs_programmed_in_chunk < 0) {
return descs_programmed_in_chunk;
}
desc_programmed += descs_programmed_in_chunk;
starting_desc = starting_desc + descs_programmed_in_chunk;
program_size -= chunk_size;
buffer_current_offset += sg_dma_len(sg_entry);
}
if (program_size != 0) {
// We didn't program all the buffer.
return -EFAULT;
}
desc_list->desc_list[(starting_desc - 1) % desc_list->desc_count].PageSize_DescControl |=
get_interrupts_bitmask(vdma_hw, last_desc_interrupts, is_debug);
return desc_programmed;
}
static int program_last_desc(
struct hailo_vdma_hw *vdma_hw,
struct hailo_vdma_descriptors_list *desc_list,
u32 starting_desc,
struct hailo_vdma_mapped_transfer_buffer *transfer_buffer,
enum hailo_vdma_interrupts_domain last_desc_interrupts,
bool is_debug)
{
u8 control = (u8)(DESCRIPTOR_DESC_CONTROL | get_interrupts_bitmask(vdma_hw, last_desc_interrupts, is_debug));
u32 total_descs = DIV_ROUND_UP(transfer_buffer->size, desc_list->desc_page_size);
u32 last_desc = (starting_desc + total_descs - 1) % desc_list->desc_count;
u32 last_desc_size = transfer_buffer->size - (total_descs - 1) * desc_list->desc_page_size;
// Configure only last descriptor with residue size
desc_list->desc_list[last_desc].PageSize_DescControl = (u32)
((last_desc_size << DESCRIPTOR_PAGE_SIZE_SHIFT) + control);
return (int)total_descs;
}
int hailo_vdma_program_descriptors_list(
struct hailo_vdma_hw *vdma_hw,
struct hailo_vdma_descriptors_list *desc_list,
u32 starting_desc,
struct hailo_vdma_mapped_transfer_buffer *buffer,
bool should_bind,
u8 channel_index,
enum hailo_vdma_interrupts_domain last_desc_interrupts,
bool is_debug)
{
return should_bind ?
bind_and_program_descriptors_list(vdma_hw, desc_list, starting_desc,
buffer, channel_index, last_desc_interrupts, is_debug) :
program_last_desc(vdma_hw, desc_list, starting_desc, buffer,
last_desc_interrupts, is_debug);
}
static bool channel_control_reg_is_active(u8 control)
{
return (control & VDMA_CHANNEL_CONTROL_START_ABORT_BITMASK) == VDMA_CHANNEL_CONTROL_START;
}
static int validate_channel_state(struct hailo_vdma_channel *channel)
{
u32 host_regs_value = ioread32(channel->host_regs);
const u8 control = READ_BITS_AT_OFFSET(BYTE_SIZE * BITS_IN_BYTE, CHANNEL_CONTROL_OFFSET * BITS_IN_BYTE, host_regs_value);
const u16 hw_num_avail = READ_BITS_AT_OFFSET(WORD_SIZE * BITS_IN_BYTE, CHANNEL_NUM_AVAIL_OFFSET * BITS_IN_BYTE, host_regs_value);
if (!channel_control_reg_is_active(control)) {
return -ECONNRESET;
}
if (hw_num_avail != channel->state.num_avail) {
pr_err("Channel %d hw state out of sync. num available is %d, expected %d\n",
channel->index, hw_num_avail, channel->state.num_avail);
return -EFAULT;
}
return 0;
}
void hailo_vdma_set_num_avail(u8 __iomem *regs, u16 num_avail)
{
u32 regs_val = ioread32(regs);
iowrite32(WRITE_BITS_AT_OFFSET(WORD_SIZE * BITS_IN_BYTE, CHANNEL_NUM_AVAIL_OFFSET * BITS_IN_BYTE, regs_val, num_avail),
regs);
}
u16 hailo_vdma_get_num_proc(u8 __iomem *regs)
{
return READ_BITS_AT_OFFSET(WORD_SIZE * BITS_IN_BYTE, 0, ioread32(regs + CHANNEL_NUM_PROC_OFFSET));
}
int hailo_vdma_launch_transfer(
struct hailo_vdma_hw *vdma_hw,
struct hailo_vdma_channel *channel,
struct hailo_vdma_descriptors_list *desc_list,
u32 starting_desc,
u8 buffers_count,
struct hailo_vdma_mapped_transfer_buffer *buffers,
bool should_bind,
enum hailo_vdma_interrupts_domain first_interrupts_domain,
enum hailo_vdma_interrupts_domain last_desc_interrupts,
bool is_debug)
{
int ret = -EFAULT;
u32 total_descs = 0;
u32 first_desc = starting_desc;
u32 last_desc = U32_MAX;
u16 new_num_avail = 0;
struct hailo_ongoing_transfer ongoing_transfer = {0};
u8 i = 0;
channel->state.desc_count_mask = (desc_list->desc_count - 1);
if (NULL == channel->last_desc_list) {
// First transfer on this active channel, store desc list.
channel->last_desc_list = desc_list;
} else if (desc_list != channel->last_desc_list) {
// Shouldn't happen, desc list may change only after channel deactivation.
pr_err("Inconsistent desc list given to channel %d\n", channel->index);
return -EINVAL;
}
ret = validate_channel_state(channel);
if (ret < 0) {
return ret;
}
if (channel->state.num_avail != (u16)starting_desc) {
pr_err("Channel %d state out of sync. num available is %d, expected %d\n",
channel->index, channel->state.num_avail, (u16)starting_desc);
return -EFAULT;
}
if (buffers_count > HAILO_MAX_BUFFERS_PER_SINGLE_TRANSFER) {
pr_err("Too many buffers %u for single transfer\n", buffers_count);
return -EINVAL;
}
BUILD_BUG_ON_MSG((HAILO_MAX_BUFFERS_PER_SINGLE_TRANSFER + 1) != ARRAY_SIZE(ongoing_transfer.dirty_descs),
"Unexpected amount of dirty descriptors");
ongoing_transfer.dirty_descs_count = buffers_count + 1;
ongoing_transfer.dirty_descs[0] = (u16)starting_desc;
for (i = 0; i < buffers_count; i++) {
ret = hailo_vdma_program_descriptors_list(vdma_hw, desc_list,
starting_desc, &buffers[i], should_bind, channel->index,
(i == (buffers_count - 1) ? last_desc_interrupts : HAILO_VDMA_INTERRUPTS_DOMAIN_NONE),
is_debug);
total_descs += ret;
last_desc = (starting_desc + ret - 1) % desc_list->desc_count;
starting_desc = (starting_desc + ret) % desc_list->desc_count;
ongoing_transfer.dirty_descs[i+1] = (u16)last_desc;
ongoing_transfer.buffers[i] = buffers[i];
}
ongoing_transfer.buffers_count = buffers_count;
desc_list->desc_list[first_desc].PageSize_DescControl |=
get_interrupts_bitmask(vdma_hw, first_interrupts_domain, is_debug);
ongoing_transfer.last_desc = (u16)last_desc;
ongoing_transfer.is_debug = is_debug;
ret = ongoing_transfer_push(channel, &ongoing_transfer);
if (ret < 0) {
pr_err("Failed push ongoing transfer to channel %d\n", channel->index);
return ret;
}
new_num_avail = (u16)((last_desc + 1) % desc_list->desc_count);
channel->state.num_avail = new_num_avail;
hailo_vdma_set_num_avail(channel->host_regs, new_num_avail);
return (int)total_descs;
}
static void hailo_vdma_push_timestamp(struct hailo_vdma_channel *channel)
{
struct hailo_channel_interrupt_timestamp_list *timestamp_list = &channel->timestamp_list;
const u16 num_proc = hailo_vdma_get_num_proc(channel->host_regs);
if (TIMESTAMPS_CIRC_SPACE(*timestamp_list) != 0) {
timestamp_list->timestamps[timestamp_list->head].timestamp_ns = ktime_get_ns();
timestamp_list->timestamps[timestamp_list->head].desc_num_processed = num_proc;
timestamp_list->head = (timestamp_list->head + 1) & CHANNEL_IRQ_TIMESTAMPS_SIZE_MASK;
}
}
// Returns false if there are no items
static bool hailo_vdma_pop_timestamp(struct hailo_channel_interrupt_timestamp_list *timestamp_list,
struct hailo_channel_interrupt_timestamp *out_timestamp)
{
if (0 == TIMESTAMPS_CIRC_CNT(*timestamp_list)) {
return false;
}
*out_timestamp = timestamp_list->timestamps[timestamp_list->tail];
timestamp_list->tail = (timestamp_list->tail+1) & CHANNEL_IRQ_TIMESTAMPS_SIZE_MASK;
return true;
}
static void hailo_vdma_pop_timestamps_to_response(struct hailo_vdma_channel *channel,
struct hailo_vdma_interrupts_read_timestamp_params *result)
{
const u32 max_timestamps = ARRAY_SIZE(result->timestamps);
u32 i = 0;
while (hailo_vdma_pop_timestamp(&channel->timestamp_list, &result->timestamps[i]) &&
(i < max_timestamps)) {
// Although the hw_num_processed should be a number between 0 and
// desc_count-1, if desc_count < 0x10000 (the maximum desc size),
// the actual hw_num_processed is a number between 1 and desc_count.
// Therefore the value can be desc_count, in this case we change it to
// zero.
result->timestamps[i].desc_num_processed = result->timestamps[i].desc_num_processed &
channel->state.desc_count_mask;
i++;
}
result->timestamps_count = i;
}
static void channel_state_init(struct hailo_vdma_channel_state *state)
{
state->num_avail = state->num_proc = 0;
// Special value used when the channel is not activate.
state->desc_count_mask = U32_MAX;
}
static u8 __iomem *get_channel_regs(u8 __iomem *regs_base, u8 channel_index, bool is_host_side, u32 src_channels_bitmask)
{
// Check if getting host side regs or device side
u8 __iomem *channel_regs_base = regs_base + CHANNEL_BASE_OFFSET(channel_index);
if (is_host_side) {
return hailo_test_bit(channel_index, &src_channels_bitmask) ? channel_regs_base :
(channel_regs_base + CHANNEL_DEST_REGS_OFFSET);
} else {
return hailo_test_bit(channel_index, &src_channels_bitmask) ? (channel_regs_base + CHANNEL_DEST_REGS_OFFSET) :
channel_regs_base;
}
}
void hailo_vdma_engine_init(struct hailo_vdma_engine *engine, u8 engine_index,
const struct hailo_resource *channel_registers, u32 src_channels_bitmask)
{
u8 channel_index = 0;
struct hailo_vdma_channel *channel;
engine->index = engine_index;
engine->enabled_channels = 0x0;
engine->interrupted_channels = 0x0;
for_each_vdma_channel(engine, channel, channel_index) {
u8 __iomem *regs_base = (u8 __iomem *)channel_registers->address;
channel->host_regs = get_channel_regs(regs_base, channel_index, true, src_channels_bitmask);
channel->device_regs = get_channel_regs(regs_base, channel_index, false, src_channels_bitmask);
channel->index = channel_index;
channel->timestamp_measure_enabled = false;
channel_state_init(&channel->state);
channel->last_desc_list = NULL;
channel->ongoing_transfers.head = 0;
channel->ongoing_transfers.tail = 0;
}
}
/**
* Enables the given channels bitmap in the given engine. Allows launching transfer
* and reading interrupts from the channels.
*
* @param engine - dma engine.
* @param bitmap - channels bitmap to enable.
* @param measure_timestamp - if set, allow interrupts timestamp measure.
*/
void hailo_vdma_engine_enable_channels(struct hailo_vdma_engine *engine, u32 bitmap,
bool measure_timestamp)
{
struct hailo_vdma_channel *channel = NULL;
u8 channel_index = 0;
for_each_vdma_channel(engine, channel, channel_index) {
if (hailo_test_bit(channel_index, &bitmap)) {
channel->timestamp_measure_enabled = measure_timestamp;
channel->timestamp_list.head = channel->timestamp_list.tail = 0;
}
}
engine->enabled_channels |= bitmap;
}
/**
* Disables the given channels bitmap in the given engine.
*
* @param engine - dma engine.
* @param bitmap - channels bitmap to enable.
* @param measure_timestamp - if set, allow interrupts timestamp measure.
*/
void hailo_vdma_engine_disable_channels(struct hailo_vdma_engine *engine, u32 bitmap)
{
struct hailo_vdma_channel *channel = NULL;
u8 channel_index = 0;
engine->enabled_channels &= ~bitmap;
for_each_vdma_channel(engine, channel, channel_index) {
if (hailo_test_bit(channel_index, &bitmap)) {
channel_state_init(&channel->state);
while (ONGOING_TRANSFERS_CIRC_CNT(channel->ongoing_transfers) > 0) {
struct hailo_ongoing_transfer transfer;
ongoing_transfer_pop(channel, &transfer);
if (channel->last_desc_list == NULL) {
pr_err("Channel %d has ongoing transfers but no desc list\n", channel->index);
continue;
}
clear_dirty_descs(channel, &transfer);
}
channel->last_desc_list = NULL;
}
}
}
void hailo_vdma_engine_push_timestamps(struct hailo_vdma_engine *engine, u32 bitmap)
{
struct hailo_vdma_channel *channel = NULL;
u8 channel_index = 0;
for_each_vdma_channel(engine, channel, channel_index) {
if (unlikely(hailo_test_bit(channel_index, &bitmap) &&
channel->timestamp_measure_enabled)) {
hailo_vdma_push_timestamp(channel);
}
}
}
int hailo_vdma_engine_read_timestamps(struct hailo_vdma_engine *engine,
struct hailo_vdma_interrupts_read_timestamp_params *params)
{
struct hailo_vdma_channel *channel = NULL;
if (params->channel_index >= MAX_VDMA_CHANNELS_PER_ENGINE) {
return -EINVAL;
}
channel = &engine->channels[params->channel_index];
hailo_vdma_pop_timestamps_to_response(channel, params);
return 0;
}
void hailo_vdma_engine_clear_channel_interrupts(struct hailo_vdma_engine *engine, u32 bitmap)
{
engine->interrupted_channels &= ~bitmap;
}
void hailo_vdma_engine_set_channel_interrupts(struct hailo_vdma_engine *engine, u32 bitmap)
{
engine->interrupted_channels |= bitmap;
}
static void fill_channel_irq_data(struct hailo_vdma_interrupts_channel_data *irq_data,
struct hailo_vdma_engine *engine, struct hailo_vdma_channel *channel, u8 transfers_completed,
bool validation_success)
{
u8 host_control = READ_BITS_AT_OFFSET(BYTE_SIZE * BITS_IN_BYTE, CHANNEL_CONTROL_OFFSET * BITS_IN_BYTE, ioread32(channel->host_regs));
u8 device_control = READ_BITS_AT_OFFSET(BYTE_SIZE * BITS_IN_BYTE, CHANNEL_CONTROL_OFFSET * BITS_IN_BYTE, ioread32(channel->device_regs));
irq_data->engine_index = engine->index;
irq_data->channel_index = channel->index;
irq_data->is_active = channel_control_reg_is_active(host_control) &&
channel_control_reg_is_active(device_control);
irq_data->transfers_completed = transfers_completed;
irq_data->host_error = READ_BITS_AT_OFFSET(BYTE_SIZE * BITS_IN_BYTE, 0, ioread32(channel->host_regs + CHANNEL_ERROR_OFFSET));
irq_data->device_error = READ_BITS_AT_OFFSET(BYTE_SIZE * BITS_IN_BYTE, 0, ioread32(channel->device_regs + CHANNEL_ERROR_OFFSET));
irq_data->validation_success = validation_success;
}
static bool is_desc_between(u16 begin, u16 end, u16 desc)
{
if (begin == end) {
// There is nothing between
return false;
}
if (begin < end) {
// desc needs to be in [begin, end)
return (begin <= desc) && (desc < end);
}
else {
// desc needs to be in [0, end) or [begin, m_descs.size()-1]
return (desc < end) || (begin <= desc);
}
}
static bool is_transfer_complete(struct hailo_vdma_channel *channel,
struct hailo_ongoing_transfer *transfer, u16 hw_num_proc)
{
if (channel->state.num_avail == hw_num_proc) {
return true;
}
return is_desc_between(channel->state.num_proc, hw_num_proc, transfer->last_desc);
}
int hailo_vdma_engine_fill_irq_data(struct hailo_vdma_interrupts_wait_params *irq_data,
struct hailo_vdma_engine *engine, u32 irq_channels_bitmap,
transfer_done_cb_t transfer_done, void *transfer_done_opaque)
{
struct hailo_vdma_channel *channel = NULL;
u8 channel_index = 0;
bool validation_success = true;
for_each_vdma_channel(engine, channel, channel_index) {
u8 transfers_completed = 0;
u16 hw_num_proc = U16_MAX;
BUILD_BUG_ON_MSG(HAILO_VDMA_MAX_ONGOING_TRANSFERS >= U8_MAX,
"HAILO_VDMA_MAX_ONGOING_TRANSFERS must be less than U8_MAX to use transfers_completed as u8");
if (!hailo_test_bit(channel->index, &irq_channels_bitmap)) {
continue;
}
if (channel->last_desc_list == NULL) {
// Channel not active or no transfer, skipping.
continue;
}
if (irq_data->channels_count >= ARRAY_SIZE(irq_data->irq_data)) {
return -EINVAL;
}
// Although the hw_num_processed should be a number between 0 and
// desc_count-1, if desc_count < 0x10000 (the maximum desc size),
// the actual hw_num_processed is a number between 1 and desc_count.
// Therefore the value can be desc_count, in this case we change it to
// zero.
hw_num_proc = hailo_vdma_get_num_proc(channel->host_regs) & channel->state.desc_count_mask;
while (ONGOING_TRANSFERS_CIRC_CNT(channel->ongoing_transfers) > 0) {
struct hailo_ongoing_transfer *cur_transfer =
&channel->ongoing_transfers.transfers[channel->ongoing_transfers.tail];
if (!is_transfer_complete(channel, cur_transfer, hw_num_proc)) {
break;
}
if (cur_transfer->is_debug &&
!validate_last_desc_status(channel, cur_transfer)) {
validation_success = false;
}
clear_dirty_descs(channel, cur_transfer);
transfer_done(cur_transfer, transfer_done_opaque);
channel->state.num_proc = (u16)((cur_transfer->last_desc + 1) & channel->state.desc_count_mask);
ongoing_transfer_pop(channel, NULL);
transfers_completed++;
}
fill_channel_irq_data(&irq_data->irq_data[irq_data->channels_count],
engine, channel, transfers_completed, validation_success);
irq_data->channels_count++;
}
return 0;
}
// For all these functions - best way to optimize might be to not call the function when need to pause and then abort,
// Rather read value once and maybe save
// This function reads and writes the register - should try to make more optimized in future
static void start_vdma_control_register(u8 __iomem *host_regs)
{
u32 host_regs_value = ioread32(host_regs);
iowrite32(WRITE_BITS_AT_OFFSET(BYTE_SIZE * BITS_IN_BYTE, CHANNEL_CONTROL_OFFSET * BITS_IN_BYTE, host_regs_value,
VDMA_CHANNEL_CONTROL_START_RESUME), host_regs);
}
static void hailo_vdma_channel_pause(u8 __iomem *host_regs)
{
u32 host_regs_value = ioread32(host_regs);
iowrite32(WRITE_BITS_AT_OFFSET(BYTE_SIZE * BITS_IN_BYTE, CHANNEL_CONTROL_OFFSET * BITS_IN_BYTE, host_regs_value,
VDMA_CHANNEL_CONTROL_START_PAUSE), host_regs);
}
// This function reads and writes the register - should try to make more optimized in future
static void hailo_vdma_channel_abort(u8 __iomem *host_regs)
{
u32 host_regs_value = ioread32(host_regs);
iowrite32(WRITE_BITS_AT_OFFSET(BYTE_SIZE * BITS_IN_BYTE, CHANNEL_CONTROL_OFFSET * BITS_IN_BYTE, host_regs_value,
VDMA_CHANNEL_CONTROL_ABORT), host_regs);
}
int hailo_vdma_start_channel(u8 __iomem *regs, uint64_t desc_dma_address, uint32_t desc_count,
uint8_t data_id)
{
u16 dma_address_l = 0;
u32 dma_address_h = 0;
u32 desc_depth_data_id = 0;
u8 desc_depth = ceil_log2(desc_count);
if (((desc_dma_address & 0xFFFF) != 0) ||
(desc_depth > DESCRIPTOR_LIST_MAX_DEPTH)) {
return -EINVAL;
}
// According to spec, depth 16 is equivalent to depth 0.
if (DESCRIPTOR_LIST_MAX_DEPTH == desc_depth) {
desc_depth = 0;
}
// Stop old channel state
hailo_vdma_stop_channel(regs);
// Configure address, depth and id
dma_address_l = (uint16_t)((desc_dma_address >> 16) & 0xFFFF);
iowrite32(WRITE_BITS_AT_OFFSET(WORD_SIZE * BITS_IN_BYTE, (VDMA_CHANNEL__ADDRESS_L_OFFSET -
VDMA_CHANNEL__ALIGNED_ADDRESS_L_OFFSET) * BITS_IN_BYTE, ioread32(regs +
VDMA_CHANNEL__ALIGNED_ADDRESS_L_OFFSET), dma_address_l), regs + VDMA_CHANNEL__ALIGNED_ADDRESS_L_OFFSET);
dma_address_h = (uint32_t)(desc_dma_address >> 32);
iowrite32(dma_address_h, regs + VDMA_CHANNEL__ADDRESS_H_OFFSET);
desc_depth_data_id = (uint32_t)(desc_depth << VDMA_CHANNEL_DESC_DEPTH_SHIFT) |
(data_id << VDMA_CHANNEL_DATA_ID_SHIFT);
iowrite32(desc_depth_data_id, regs);
start_vdma_control_register(regs);
return 0;
}
static bool hailo_vdma_channel_is_idle(u8 __iomem *host_regs, size_t host_side_max_desc_count)
{
// Num processed and ongoing are next to each other in the memory.
// Reading them both in order to save BAR reads.
u32 host_side_num_processed_ongoing = ioread32(host_regs + CHANNEL_NUM_PROC_OFFSET);
u16 host_side_num_processed = (host_side_num_processed_ongoing & VDMA_CHANNEL_NUM_PROCESSED_MASK);
u16 host_side_num_ongoing = (host_side_num_processed_ongoing >> VDMA_CHANNEL_NUM_PROCESSED_WIDTH) &
VDMA_CHANNEL_NUM_ONGOING_MASK;
if ((host_side_num_processed % host_side_max_desc_count) == (host_side_num_ongoing % host_side_max_desc_count)) {
return true;
}
return false;
}
static int hailo_vdma_wait_until_channel_idle(u8 __iomem *host_regs)
{
bool is_idle = false;
uint32_t check_counter = 0;
u8 depth = (uint8_t)(READ_BITS_AT_OFFSET(VDMA_CHANNEL_DESC_DEPTH_WIDTH, VDMA_CHANNEL_DESC_DEPTH_SHIFT,
ioread32(host_regs)));
size_t host_side_max_desc_count = (size_t)(1 << depth);
for (check_counter = 0; check_counter < VDMA_CHANNEL__MAX_CHECKS_CHANNEL_IS_IDLE; check_counter++) {
is_idle = hailo_vdma_channel_is_idle(host_regs, host_side_max_desc_count);
if (is_idle) {
return 0;
}
}
return -ETIMEDOUT;
}
void hailo_vdma_stop_channel(u8 __iomem *regs)
{
int err = 0;
u8 host_side_channel_regs = READ_BITS_AT_OFFSET(BYTE_SIZE * BITS_IN_BYTE, CHANNEL_CONTROL_OFFSET * BITS_IN_BYTE, ioread32(regs));
if ((host_side_channel_regs & VDMA_CHANNEL_CONTROL_START_ABORT_PAUSE_RESUME_BITMASK) == VDMA_CHANNEL_CONTROL_ABORT_PAUSE) {
// The channel is aborted (we set the channel to VDMA_CHANNEL_CONTROL_ABORT_PAUSE at the end of this function)
return;
}
// Pause the channel
// The channel is paused to allow for "all transfers from fetched descriptors..." to be "...completed"
// (from PLDA PCIe refernce manual, "9.2.5 Starting a Channel and Transferring Data")
hailo_vdma_channel_pause(regs);
// Even if channel is stuck and not idle, force abort and return error in the end
err = hailo_vdma_wait_until_channel_idle(regs);
// Success oriented - if error occured print error but still abort channel
if (err < 0) {
pr_err("Timeout occured while waiting for channel to become idle\n");
}
// Abort the channel (even of hailo_vdma_wait_until_channel_idle function fails)
hailo_vdma_channel_abort(regs);
}
bool hailo_check_channel_index(u8 channel_index, u32 src_channels_bitmask, bool is_input_channel)
{
return is_input_channel ? hailo_test_bit(channel_index, &src_channels_bitmask) :
(!hailo_test_bit(channel_index, &src_channels_bitmask));
}

284
drivers/media/pci/hailo/common/vdma_common.h
View File

@@ -1,284 +0,0 @@
// SPDX-License-Identifier: MIT
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_COMMON_VDMA_COMMON_H_
#define _HAILO_COMMON_VDMA_COMMON_H_
#include "hailo_resource.h"
#include "utils.h"
#include <linux/types.h>
#include <linux/scatterlist.h>
#include <linux/io.h>
#define VDMA_DESCRIPTOR_LIST_ALIGN (1 << 16)
#define INVALID_VDMA_ADDRESS (0)
#define CHANNEL_BASE_OFFSET(channel_index) ((channel_index) << 5)
#define CHANNEL_CONTROL_OFFSET (0x0)
#define CHANNEL_DEPTH_ID_OFFSET (0x1)
#define CHANNEL_NUM_AVAIL_OFFSET (0x2)
#define CHANNEL_NUM_PROC_OFFSET (0x4)
#define CHANNEL_ERROR_OFFSET (0x8)
#define CHANNEL_DEST_REGS_OFFSET (0x10)
#ifdef __cplusplus
extern "C"
{
#endif
struct hailo_vdma_descriptor {
u32 PageSize_DescControl;
u32 AddrL_rsvd_DataID;
u32 AddrH;
u32 RemainingPageSize_Status;
};
struct hailo_vdma_descriptors_list {
struct hailo_vdma_descriptor *desc_list;
// Must be power of 2 if is_circular is set.
u32 desc_count;
// The nearest power of 2 to desc_count (including desc_count), minus 1.
// * If the list is circular, then 'index & desc_count_mask' can be used instead of modulo.
// * Otherwise, we can't wrap around the list anyway. However, for any index < desc_count, 'index & desc_count_mask'
// will return the same value.
u32 desc_count_mask;
u16 desc_page_size;
bool is_circular;
};
struct hailo_channel_interrupt_timestamp_list {
int head;
int tail;
struct hailo_channel_interrupt_timestamp timestamps[CHANNEL_IRQ_TIMESTAMPS_SIZE];
};
// For each buffers in transfer, the last descriptor will be programmed with
// the residue size. In addition, if configured, the first descriptor (in
// all transfer) may be programmed with interrupts.
#define MAX_DIRTY_DESCRIPTORS_PER_TRANSFER \
(HAILO_MAX_BUFFERS_PER_SINGLE_TRANSFER + 1)
struct hailo_vdma_mapped_transfer_buffer {
struct sg_table *sg_table;
u32 size;
u32 offset;
void *opaque; // Drivers can set any opaque data here.
};
struct hailo_ongoing_transfer {
uint16_t last_desc;
u8 buffers_count;
struct hailo_vdma_mapped_transfer_buffer buffers[HAILO_MAX_BUFFERS_PER_SINGLE_TRANSFER];
// Contains all descriptors that were programmed with non-default values
// for the transfer (by non-default we mean - different size or different
// interrupts domain).
uint8_t dirty_descs_count;
uint16_t dirty_descs[MAX_DIRTY_DESCRIPTORS_PER_TRANSFER];
// If set, validate descriptors status on transfer completion.
bool is_debug;
};
struct hailo_ongoing_transfers_list {
unsigned long head;
unsigned long tail;
struct hailo_ongoing_transfer transfers[HAILO_VDMA_MAX_ONGOING_TRANSFERS];
};
struct hailo_vdma_channel_state {
// vdma channel counters. num_avail should be synchronized with the hw
// num_avail value. num_proc is the last num proc updated when the user
// reads interrupts.
u16 num_avail;
u16 num_proc;
// Mask of the num-avail/num-proc counters.
u32 desc_count_mask;
};
struct hailo_vdma_channel {
u8 index;
u8 __iomem *host_regs;
u8 __iomem *device_regs;
// Last descriptors list attached to the channel. When it changes,
// assumes that the channel got reset.
struct hailo_vdma_descriptors_list *last_desc_list;
struct hailo_vdma_channel_state state;
struct hailo_ongoing_transfers_list ongoing_transfers;
bool timestamp_measure_enabled;
struct hailo_channel_interrupt_timestamp_list timestamp_list;
};
struct hailo_vdma_engine {
u8 index;
u32 enabled_channels;
u32 interrupted_channels;
struct hailo_vdma_channel channels[MAX_VDMA_CHANNELS_PER_ENGINE];
};
struct hailo_vdma_hw_ops {
// Accepts start, end and step of an address range (of type dma_addr_t).
// Returns the encoded base address or INVALID_VDMA_ADDRESS if the range/step is invalid.
// All addresses in the range of [returned_addr, returned_addr + step, returned_addr + 2*step, ..., dma_address_end) are valid.
u64 (*encode_desc_dma_address_range)(dma_addr_t dma_address_start, dma_addr_t dma_address_end, u32 step, u8 channel_id);
};
struct hailo_vdma_hw {
struct hailo_vdma_hw_ops hw_ops;
// The data_id code of ddr addresses.
u8 ddr_data_id;
// Bitmask needed to set on each descriptor to enable interrupts (either host/device).
unsigned long host_interrupts_bitmask;
unsigned long device_interrupts_bitmask;
// Bitmask for each vdma hw, which channels are src side by index (on pcie/dram - 0x0000FFFF, pci ep - 0xFFFF0000)
u32 src_channels_bitmask;
};
#define _for_each_element_array(array, size, element, index) \
for (index = 0, element = &array[index]; index < size; index++, element = &array[index])
#define for_each_vdma_channel(engine, channel, channel_index) \
_for_each_element_array((engine)->channels, MAX_VDMA_CHANNELS_PER_ENGINE, \
channel, channel_index)
/**
* Program the given descriptors list to map the given buffer.
*
* @param vdma_hw vdma hw object
* @param desc_list descriptors list object to program
* @param starting_desc index of the first descriptor to program. If the list
* is circular, this function may wrap around the list.
* @param buffer buffer to program to the descriptors list.
* @param should_bind If false, assumes the buffer was already bound to the
* desc list. Used for optimization.
* @param channel_index channel index of the channel attached.
* @param last_desc_interrupts - interrupts settings on last descriptor.
* @param is_debug program descriptors for debug run.
*
* @return On success - the amount of descriptors programmed, negative value on error.
*/
int hailo_vdma_program_descriptors_list(
struct hailo_vdma_hw *vdma_hw,
struct hailo_vdma_descriptors_list *desc_list,
u32 starting_desc,
struct hailo_vdma_mapped_transfer_buffer *buffer,
bool should_bind,
u8 channel_index,
enum hailo_vdma_interrupts_domain last_desc_interrupts,
bool is_debug);
int hailo_vdma_program_descriptors_in_chunk(
struct hailo_vdma_hw *vdma_hw,
dma_addr_t chunk_addr,
unsigned int chunk_size,
struct hailo_vdma_descriptors_list *desc_list,
u32 desc_index,
u32 max_desc_index,
u8 channel_index,
u8 data_id);
void hailo_vdma_set_num_avail(u8 __iomem *regs, u16 num_avail);
u16 hailo_vdma_get_num_proc(u8 __iomem *regs);
/**
* Launch a transfer on some vdma channel. Includes:
* 1. Binding the transfer buffers to the descriptors list.
* 2. Program the descriptors list.
* 3. Increase num available
*
* @param vdma_hw vdma hw object
* @param channel vdma channel object.
* @param desc_list descriptors list object to program.
* @param starting_desc index of the first descriptor to program.
* @param buffers_count amount of transfer mapped buffers to program.
* @param buffers array of buffers to program to the descriptors list.
* @param should_bind whether to bind the buffer to the descriptors list.
* @param first_interrupts_domain - interrupts settings on first descriptor.
* @param last_desc_interrupts - interrupts settings on last descriptor.
* @param is_debug program descriptors for debug run, adds some overhead (for
* example, hw will write desc complete status).
*
* @return On success - the amount of descriptors programmed, negative value on error.
*/
int hailo_vdma_launch_transfer(
struct hailo_vdma_hw *vdma_hw,
struct hailo_vdma_channel *channel,
struct hailo_vdma_descriptors_list *desc_list,
u32 starting_desc,
u8 buffers_count,
struct hailo_vdma_mapped_transfer_buffer *buffers,
bool should_bind,
enum hailo_vdma_interrupts_domain first_interrupts_domain,
enum hailo_vdma_interrupts_domain last_desc_interrupts,
bool is_debug);
void hailo_vdma_engine_init(struct hailo_vdma_engine *engine, u8 engine_index,
const struct hailo_resource *channel_registers, u32 src_channels_bitmask);
void hailo_vdma_engine_enable_channels(struct hailo_vdma_engine *engine, u32 bitmap,
bool measure_timestamp);
void hailo_vdma_engine_disable_channels(struct hailo_vdma_engine *engine, u32 bitmap);
void hailo_vdma_engine_push_timestamps(struct hailo_vdma_engine *engine, u32 bitmap);
int hailo_vdma_engine_read_timestamps(struct hailo_vdma_engine *engine,
struct hailo_vdma_interrupts_read_timestamp_params *params);
static inline bool hailo_vdma_engine_got_interrupt(struct hailo_vdma_engine *engine,
u32 channels_bitmap)
{
// Reading interrupts without lock is ok (needed only for writes)
const bool any_interrupt = (0 != (channels_bitmap & engine->interrupted_channels));
const bool any_disabled = (channels_bitmap != (channels_bitmap & engine->enabled_channels));
return (any_disabled || any_interrupt);
}
// Set/Clear/Read channels interrupts, must called under some lock (driver specific)
void hailo_vdma_engine_clear_channel_interrupts(struct hailo_vdma_engine *engine, u32 bitmap);
void hailo_vdma_engine_set_channel_interrupts(struct hailo_vdma_engine *engine, u32 bitmap);
static inline u32 hailo_vdma_engine_read_interrupts(struct hailo_vdma_engine *engine,
u32 requested_bitmap)
{
// Interrupts only for channels that are requested and enabled.
u32 irq_channels_bitmap = requested_bitmap &
engine->enabled_channels &
engine->interrupted_channels;
engine->interrupted_channels &= ~irq_channels_bitmap;
return irq_channels_bitmap;
}
typedef void(*transfer_done_cb_t)(struct hailo_ongoing_transfer *transfer, void *opaque);
// Assuming irq_data->channels_count contains the amount of channels already
// written (used for multiple engines).
int hailo_vdma_engine_fill_irq_data(struct hailo_vdma_interrupts_wait_params *irq_data,
struct hailo_vdma_engine *engine, u32 irq_channels_bitmap,
transfer_done_cb_t transfer_done, void *transfer_done_opaque);
int hailo_vdma_start_channel(u8 __iomem *regs, uint64_t desc_dma_address, uint32_t desc_count, uint8_t data_id);
void hailo_vdma_stop_channel(u8 __iomem *regs);
bool hailo_check_channel_index(u8 channel_index, u32 src_channels_bitmask, bool is_input_channel);
#ifdef __cplusplus
}
#endif
#endif /* _HAILO_COMMON_VDMA_COMMON_H_ */

14
drivers/media/pci/hailo/include/hailo_pcie_version.h
View File

@@ -1,14 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2022 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_PCIE_VERSION_H_
#define _HAILO_PCIE_VERSION_H_
#include <linux/stringify.h>
#include "../common/hailo_pcie_version.h"
#define HAILO_DRV_VER __stringify(HAILO_DRV_VER_MAJOR) "." __stringify(HAILO_DRV_VER_MINOR) "." __stringify(HAILO_DRV_VER_REVISION)
#endif /* _HAILO_PCIE_VERSION_H_ */

570
drivers/media/pci/hailo/src/fops.c
View File

@@ -1,570 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#include <linux/version.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
#include <linux/pagemap.h>
#include <linux/uaccess.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/delay.h>
#include <asm/thread_info.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
#include <linux/sched/signal.h>
#endif
#include "fops.h"
#include "vdma_common.h"
#include "utils/logs.h"
#include "vdma/memory.h"
#include "vdma/ioctl.h"
#include "utils/compact.h"
#include "nnc.h"
#include "soc.h"
#if LINUX_VERSION_CODE >= KERNEL_VERSION( 4, 13, 0 )
#define wait_queue_t wait_queue_entry_t
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION( 4, 15, 0 )
#define ACCESS_ONCE READ_ONCE
#endif
#ifndef VM_RESERVED
#define VMEM_FLAGS (VM_IO | VM_DONTEXPAND | VM_DONTDUMP)
#else
#define VMEM_FLAGS (VM_IO | VM_RESERVED)
#endif
#define IS_PO2_ALIGNED(size, alignment) (!(size & (alignment-1)))
// On pcie driver there is only one dma engine
#define DEFAULT_VDMA_ENGINE_INDEX (0)
static struct hailo_file_context *create_file_context(struct hailo_pcie_board *board, struct file *filp)
{
struct hailo_file_context *context = kzalloc(sizeof(*context), GFP_KERNEL);
if (!context) {
hailo_err(board, "Failed to alloc file context (required size %zu)\n", sizeof(*context));
return ERR_PTR(-ENOMEM);
}
context->filp = filp;
hailo_vdma_file_context_init(&context->vdma_context);
list_add(&context->open_files_list, &board->open_files_list);
context->is_valid = true;
return context;
}
static void release_file_context(struct hailo_file_context *context)
{
context->is_valid = false;
list_del(&context->open_files_list);
kfree(context);
}
static struct hailo_file_context *find_file_context(struct hailo_pcie_board *board, struct file *filp)
{
struct hailo_file_context *cur = NULL;
list_for_each_entry(cur, &board->open_files_list, open_files_list) {
if (cur->filp == filp) {
return cur;
}
}
return NULL;
}
int hailo_pcie_fops_open(struct inode *inode, struct file *filp)
{
u32 major = MAJOR(inode->i_rdev);
u32 minor = MINOR(inode->i_rdev);
struct hailo_pcie_board *pBoard;
int err = 0;
pci_power_t previous_power_state = PCI_UNKNOWN;
bool interrupts_enabled_by_filp = false;
struct hailo_file_context *context = NULL;
pr_debug(DRIVER_NAME ": (%d: %d-%d): fops_open\n", current->tgid, major, minor);
// allow multiple processes to open a device, count references in hailo_pcie_get_board_index.
if (!(pBoard = hailo_pcie_get_board_index(minor))) {
pr_err(DRIVER_NAME ": fops_open: PCIe board not found for /dev/hailo%d node.\n", minor);
err = -ENODEV;
goto l_exit;
}
filp->private_data = pBoard;
if (down_interruptible(&pBoard->mutex)) {
hailo_err(pBoard, "fops_open down_interruptible fail tgid:%d\n", current->tgid);
err = -ERESTARTSYS;
goto l_decrease_ref_count;
}
context = create_file_context(pBoard, filp);
if (IS_ERR(context)) {
err = PTR_ERR(context);
goto l_release_mutex;
}
previous_power_state = pBoard->pDev->current_state;
if (PCI_D0 != previous_power_state) {
hailo_info(pBoard, "Waking up board change state from %d to PCI_D0\n", previous_power_state);
err = pci_set_power_state(pBoard->pDev, PCI_D0);
if (err < 0) {
hailo_err(pBoard, "Failed waking up board %d", err);
goto l_free_context;
}
}
if (!hailo_pcie_is_device_connected(&pBoard->pcie_resources)) {
hailo_err(pBoard, "Device disconnected while opening device\n");
err = -ENXIO;
goto l_revert_power_state;
}
// enable interrupts
if (!pBoard->interrupts_enabled) {
err = hailo_enable_interrupts(pBoard);
if (err < 0) {
hailo_err(pBoard, "Failed Enabling interrupts %d\n", err);
goto l_revert_power_state;
}
interrupts_enabled_by_filp = true;
}
if (pBoard->pcie_resources.accelerator_type == HAILO_ACCELERATOR_TYPE_NNC) {
err = hailo_nnc_file_context_init(pBoard, context);
} else {
err = hailo_soc_file_context_init(pBoard, context);
}
if (err < 0) {
goto l_release_irq;
}
hailo_dbg(pBoard, "(%d: %d-%d): fops_open: SUCCESS on /dev/hailo%d\n", current->tgid,
major, minor, minor);
up(&pBoard->mutex);
return 0;
l_release_irq:
if (interrupts_enabled_by_filp) {
hailo_disable_interrupts(pBoard);
}
l_revert_power_state:
if (pBoard->pDev->current_state != previous_power_state) {
hailo_info(pBoard, "Power changing state from %d to %d\n", previous_power_state, pBoard->pDev->current_state);
if (pci_set_power_state(pBoard->pDev, previous_power_state) < 0) {
hailo_err(pBoard, "Failed setting power state back to %d\n", (int)previous_power_state);
}
}
l_free_context:
release_file_context(context);
l_release_mutex:
up(&pBoard->mutex);
l_decrease_ref_count:
atomic_dec(&pBoard->ref_count);
l_exit:
return err;
}
int hailo_pcie_fops_release(struct inode *inode, struct file *filp)
{
struct hailo_pcie_board *board = (struct hailo_pcie_board *)filp->private_data;
struct hailo_file_context *context = NULL;
u32 major = MAJOR(inode->i_rdev);
u32 minor = MINOR(inode->i_rdev);
if (board) {
hailo_info(board, "(%d: %d-%d): fops_release\n", current->tgid, major, minor);
down(&board->mutex);
context = find_file_context(board, filp);
if (NULL == context) {
hailo_err(board, "Invalid driver state, file context does not exist\n");
up(&board->mutex);
return -EINVAL;
}
if (false == context->is_valid) {
// File context is invalid, but open. It's OK to continue finalize and release it.
hailo_err(board, "Invalid file context\n");
}
if (board->pcie_resources.accelerator_type == HAILO_ACCELERATOR_TYPE_NNC) {
hailo_nnc_file_context_finalize(board, context);
} else {
hailo_soc_file_context_finalize(board, context);
}
hailo_vdma_file_context_finalize(&context->vdma_context, &board->vdma, filp);
release_file_context(context);
if (atomic_dec_and_test(&board->ref_count)) {
// Disable interrupts
hailo_disable_interrupts(board);
if (power_mode_enabled()) {
hailo_info(board, "Power change state to PCI_D3hot\n");
if (board->pDev && pci_set_power_state(board->pDev, PCI_D3hot) < 0) {
hailo_err(board, "Failed setting power state to D3hot");
}
}
// deallocate board if already removed
if (!board->pDev) {
hailo_dbg(board, "fops_release, freed board\n");
up(&board->mutex);
kfree(board);
board = NULL;
} else {
hailo_dbg(board, "fops_release, released resources for board\n");
up(&board->mutex);
}
} else {
up(&board->mutex);
}
hailo_dbg(board, "(%d: %d-%d): fops_release: SUCCESS on /dev/hailo%d\n", current->tgid,
major, minor, minor);
}
return 0;
}
static long hailo_memory_transfer_ioctl(struct hailo_pcie_board *board, unsigned long arg)
{
long err = 0;
struct hailo_memory_transfer_params* transfer = &board->memory_transfer_params;
hailo_dbg(board, "Start memory transfer ioctl\n");
if (copy_from_user(transfer, (void __user*)arg, sizeof(*transfer))) {
hailo_err(board, "copy_from_user fail\n");
return -ENOMEM;
}
err = hailo_pcie_memory_transfer(&board->pcie_resources, transfer);
if (err < 0) {
hailo_err(board, "memory transfer failed %ld", err);
}
if (copy_to_user((void __user*)arg, transfer, sizeof(*transfer))) {
hailo_err(board, "copy_to_user fail\n");
return -ENOMEM;
}
return err;
}
static void firmware_notification_irq_handler(struct hailo_pcie_board *board)
{
struct hailo_notification_wait *notif_wait_cursor = NULL;
int err = 0;
unsigned long irq_saved_flags = 0;
spin_lock_irqsave(&board->nnc.notification_read_spinlock, irq_saved_flags);
err = hailo_pcie_read_firmware_notification(&board->pcie_resources.fw_access, &board->nnc.notification_cache);
spin_unlock_irqrestore(&board->nnc.notification_read_spinlock, irq_saved_flags);
if (err < 0) {
hailo_err(board, "Failed reading firmware notification");
}
else {
// TODO: HRT-14502 move interrupt handling to nnc
rcu_read_lock();
list_for_each_entry_rcu(notif_wait_cursor, &board->nnc.notification_wait_list, notification_wait_list)
{
complete(&notif_wait_cursor->notification_completion);
}
rcu_read_unlock();
}
}
static void boot_irq_handler(struct hailo_pcie_board *board, struct hailo_pcie_interrupt_source *irq_source)
{
if (irq_source->sw_interrupts & HAILO_PCIE_BOOT_SOFT_RESET_IRQ) {
hailo_dbg(board, "soft reset trigger IRQ\n");
complete(&board->soft_reset.reset_completed);
}
if (irq_source->sw_interrupts & HAILO_PCIE_BOOT_IRQ) {
hailo_dbg(board, "boot trigger IRQ\n");
complete_all(&board->fw_boot.fw_loaded_completion);
} else {
board->fw_boot.boot_used_channel_bitmap &= ~irq_source->vdma_channels_bitmap;
hailo_dbg(board, "boot vDMA data IRQ - channel_bitmap = 0x%x\n", irq_source->vdma_channels_bitmap);
if (0 == board->fw_boot.boot_used_channel_bitmap) {
complete_all(&board->fw_boot.vdma_boot_completion);
hailo_dbg(board, "boot vDMA data trigger IRQ\n");
}
}
}
static void nnc_irq_handler(struct hailo_pcie_board *board, struct hailo_pcie_interrupt_source *irq_source)
{
if (irq_source->sw_interrupts & HAILO_PCIE_NNC_FW_CONTROL_IRQ) {
complete(&board->nnc.fw_control.completion);
}
if (irq_source->sw_interrupts & HAILO_PCIE_NNC_DRIVER_DOWN_IRQ) {
complete(&board->driver_down.reset_completed);
}
if (irq_source->sw_interrupts & HAILO_PCIE_NNC_FW_NOTIFICATION_IRQ) {
firmware_notification_irq_handler(board);
}
}
static void soc_irq_handler(struct hailo_pcie_board *board, struct hailo_pcie_interrupt_source *irq_source)
{
if (irq_source->sw_interrupts & HAILO_PCIE_SOC_CONTROL_IRQ) {
complete_all(&board->soc.control_resp_ready);
}
if (irq_source->sw_interrupts & HAILO_PCIE_SOC_CLOSE_IRQ) {
hailo_info(board, "soc_irq_handler - HAILO_PCIE_SOC_CLOSE_IRQ\n");
// always use bitmap=0xFFFFFFFF - it is ok to wake all interrupts since each handler will check if the stream was aborted or not.
hailo_vdma_wakeup_interrupts(&board->vdma, &board->vdma.vdma_engines[DEFAULT_VDMA_ENGINE_INDEX],
0xFFFFFFFF);
}
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
irqreturn_t hailo_irqhandler(int irq, void *dev_id, struct pt_regs *regs)
#else
irqreturn_t hailo_irqhandler(int irq, void *dev_id)
#endif
{
irqreturn_t return_value = IRQ_NONE;
struct hailo_pcie_board *board = (struct hailo_pcie_board *)dev_id;
bool got_interrupt = false;
struct hailo_pcie_interrupt_source irq_source = {0};
hailo_dbg(board, "hailo_irqhandler\n");
while (true) {
if (!hailo_pcie_is_device_connected(&board->pcie_resources)) {
hailo_err(board, "Device disconnected while handling irq\n");
break;
}
got_interrupt = hailo_pcie_read_interrupt(&board->pcie_resources, &irq_source);
if (!got_interrupt) {
break;
}
return_value = IRQ_HANDLED;
if (board->fw_boot.is_in_boot) {
boot_irq_handler(board, &irq_source);
} else {
if (HAILO_ACCELERATOR_TYPE_NNC == board->pcie_resources.accelerator_type) {
nnc_irq_handler(board, &irq_source);
} else if (HAILO_ACCELERATOR_TYPE_SOC == board->pcie_resources.accelerator_type) {
soc_irq_handler(board, &irq_source);
} else {
hailo_err(board, "Invalid accelerator type %d\n", board->pcie_resources.accelerator_type);
}
if (0 != irq_source.vdma_channels_bitmap) {
hailo_vdma_irq_handler(&board->vdma, DEFAULT_VDMA_ENGINE_INDEX,
irq_source.vdma_channels_bitmap);
}
}
}
return return_value;
}
static long hailo_query_device_properties(struct hailo_pcie_board *board, unsigned long arg)
{
struct hailo_device_properties props = {
.desc_max_page_size = board->desc_max_page_size,
.board_type = board->pcie_resources.board_type,
.allocation_mode = board->allocation_mode,
.dma_type = HAILO_DMA_TYPE_PCIE,
.dma_engines_count = board->vdma.vdma_engines_count,
.is_fw_loaded = hailo_pcie_is_firmware_loaded(&board->pcie_resources),
};
hailo_info(board, "HAILO_QUERY_DEVICE_PROPERTIES: desc_max_page_size=%u\n", props.desc_max_page_size);
if (copy_to_user((void __user*)arg, &props, sizeof(props))) {
hailo_err(board, "HAILO_QUERY_DEVICE_PROPERTIES, copy_to_user failed\n");
return -ENOMEM;
}
return 0;
}
static long hailo_query_driver_info(struct hailo_pcie_board *board, unsigned long arg)
{
struct hailo_driver_info info = {
.major_version = HAILO_DRV_VER_MAJOR,
.minor_version = HAILO_DRV_VER_MINOR,
.revision_version = HAILO_DRV_VER_REVISION
};
hailo_info(board, "HAILO_QUERY_DRIVER_INFO: major=%u, minor=%u, revision=%u\n",
info.major_version, info.minor_version, info.revision_version);
if (copy_to_user((void __user*)arg, &info, sizeof(info))) {
hailo_err(board, "HAILO_QUERY_DRIVER_INFO, copy_to_user failed\n");
return -ENOMEM;
}
return 0;
}
static long hailo_general_ioctl(struct hailo_pcie_board *board, unsigned int cmd, unsigned long arg)
{
switch (cmd) {
case HAILO_MEMORY_TRANSFER:
return hailo_memory_transfer_ioctl(board, arg);
case HAILO_QUERY_DEVICE_PROPERTIES:
return hailo_query_device_properties(board, arg);
case HAILO_QUERY_DRIVER_INFO:
return hailo_query_driver_info(board, arg);
default:
hailo_err(board, "Invalid general ioctl code 0x%x (nr: %d)\n", cmd, _IOC_NR(cmd));
return -ENOTTY;
}
}
long hailo_pcie_fops_unlockedioctl(struct file* filp, unsigned int cmd, unsigned long arg)
{
long err = 0;
struct hailo_pcie_board* board = (struct hailo_pcie_board*) filp->private_data;
struct hailo_file_context *context = NULL;
bool should_up_board_mutex = true;
if (!board || !board->pDev) return -ENODEV;
hailo_dbg(board, "(%d): fops_unlockedioctl. cmd:%d\n", current->tgid, _IOC_NR(cmd));
if (_IOC_DIR(cmd) & _IOC_READ)
{
err = !compatible_access_ok(VERIFY_WRITE, (void __user *)arg, _IOC_SIZE(cmd));
}
else if (_IOC_DIR(cmd) & _IOC_WRITE)
{
err = !compatible_access_ok(VERIFY_READ, (void __user *)arg, _IOC_SIZE(cmd));
}
if (err) {
hailo_err(board, "Invalid ioctl parameter access 0x%x", cmd);
return -EFAULT;
}
if (down_interruptible(&board->mutex)) {
hailo_err(board, "unlockedioctl down_interruptible failed");
return -ERESTARTSYS;
}
BUG_ON(board->mutex.count != 0);
context = find_file_context(board, filp);
if (NULL == context) {
hailo_err(board, "Invalid driver state, file context does not exist\n");
up(&board->mutex);
return -EINVAL;
}
if (false == context->is_valid) {
hailo_err(board, "Invalid file context\n");
up(&board->mutex);
return -EINVAL;
}
switch (_IOC_TYPE(cmd)) {
case HAILO_GENERAL_IOCTL_MAGIC:
err = hailo_general_ioctl(board, cmd, arg);
break;
case HAILO_VDMA_IOCTL_MAGIC:
err = hailo_vdma_ioctl(&context->vdma_context, &board->vdma, cmd, arg, filp, &board->mutex,
&should_up_board_mutex);
break;
case HAILO_SOC_IOCTL_MAGIC:
if (HAILO_ACCELERATOR_TYPE_SOC != board->pcie_resources.accelerator_type) {
hailo_err(board, "Ioctl %d is not supported on this accelerator type\n", _IOC_TYPE(cmd));
err = -EINVAL;
} else {
err = hailo_soc_ioctl(board, context, &board->vdma, cmd, arg);
}
break;
case HAILO_NNC_IOCTL_MAGIC:
if (HAILO_ACCELERATOR_TYPE_NNC != board->pcie_resources.accelerator_type) {
hailo_err(board, "Ioctl %d is not supported on this accelerator type\n", _IOC_TYPE(cmd));
err = -EINVAL;
} else {
err = hailo_nnc_ioctl(board, cmd, arg, filp, &should_up_board_mutex);
}
break;
default:
hailo_err(board, "Invalid ioctl type %d\n", _IOC_TYPE(cmd));
err = -ENOTTY;
}
if (should_up_board_mutex) {
up(&board->mutex);
}
hailo_dbg(board, "(%d): fops_unlockedioct: SUCCESS\n", current->tgid);
return err;
}
int hailo_pcie_fops_mmap(struct file* filp, struct vm_area_struct *vma)
{
int err = 0;
uintptr_t vdma_handle = vma->vm_pgoff << PAGE_SHIFT;
struct hailo_pcie_board* board = (struct hailo_pcie_board*)filp->private_data;
struct hailo_file_context *context = NULL;
BUILD_BUG_ON_MSG(sizeof(vma->vm_pgoff) < sizeof(vdma_handle),
"If this expression fails to compile it means the target HW is not compatible with our approach to use "
"the page offset paramter of 'mmap' to pass the driver the 'handle' of the desired descriptor");
vma->vm_pgoff = 0; // vm_pgoff contains vdma_handle page offset, the actual offset from the phys addr is 0
hailo_info(board, "%d fops_mmap\n", current->tgid);
if (!board || !board->pDev) return -ENODEV;
if (down_interruptible(&board->mutex)) {
hailo_err(board, "hailo_pcie_fops_mmap down_interruptible fail tgid:%d\n", current->tgid);
return -ERESTARTSYS;
}
context = find_file_context(board, filp);
if (NULL == context) {
up(&board->mutex);
hailo_err(board, "Invalid driver state, file context does not exist\n");
return -EINVAL;
}
if (false == context->is_valid) {
up(&board->mutex);
hailo_err(board, "Invalid file context\n");
return -EINVAL;
}
err = hailo_vdma_mmap(&context->vdma_context, &board->vdma, vma, vdma_handle);
up(&board->mutex);
return err;
}

23
drivers/media/pci/hailo/src/fops.h
View File

@@ -1,23 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_PCI_FOPS_H_
#define _HAILO_PCI_FOPS_H_
#include "pcie.h"
int hailo_pcie_fops_open(struct inode* inode, struct file* filp);
int hailo_pcie_fops_release(struct inode* inode, struct file* filp);
long hailo_pcie_fops_unlockedioctl(struct file* filp, unsigned int cmd, unsigned long arg);
int hailo_pcie_fops_mmap(struct file* filp, struct vm_area_struct *vma);
void hailo_pcie_ep_init(struct hailo_pcie_board *board);
#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22)
irqreturn_t hailo_irqhandler(int irq, void* dev_id, struct pt_regs *regs);
#else
irqreturn_t hailo_irqhandler(int irq, void* dev_id);
#endif
#endif /* _HAILO_PCI_FOPS_H_ */

299
drivers/media/pci/hailo/src/nnc.c
View File

@@ -1,299 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
/**
* A Hailo PCIe NNC device is a device contains a NNC (neural network core) and some basic FW.
* The device supports sending controls, receiving notification and reading the FW log.
*/
#include "nnc.h"
#include "hailo_ioctl_common.h"
#include "utils/logs.h"
#include "utils/compact.h"
#include <linux/uaccess.h>
#if !defined(HAILO_EMULATOR)
#define DEFAULT_SHUTDOWN_TIMEOUT_MS (5)
#else /* !defined(HAILO_EMULATOR) */
#define DEFAULT_SHUTDOWN_TIMEOUT_MS (1000)
#endif /* !defined(HAILO_EMULATOR) */
void hailo_nnc_init(struct hailo_pcie_nnc *nnc)
{
sema_init(&nnc->fw_control.mutex, 1);
spin_lock_init(&nnc->notification_read_spinlock);
init_completion(&nnc->fw_control.completion);
INIT_LIST_HEAD(&nnc->notification_wait_list);
memset(&nnc->notification_cache, 0, sizeof(nnc->notification_cache));
}
void hailo_nnc_finalize(struct hailo_pcie_nnc *nnc)
{
struct hailo_notification_wait *cursor = NULL;
// Lock rcu_read_lock and send notification_completion to wake anyone waiting on the notification_wait_list when removed
rcu_read_lock();
list_for_each_entry_rcu(cursor, &nnc->notification_wait_list, notification_wait_list) {
cursor->is_disabled = true;
complete(&cursor->notification_completion);
}
rcu_read_unlock();
}
static int hailo_fw_control(struct hailo_pcie_board *board, unsigned long arg, bool* should_up_board_mutex)
{
struct hailo_fw_control *command = &board->nnc.fw_control.command;
long completion_result = 0;
int err = 0;
up(&board->mutex);
*should_up_board_mutex = false;
if (down_interruptible(&board->nnc.fw_control.mutex)) {
hailo_info(board, "hailo_fw_control down_interruptible fail tgid:%d (process was interrupted or killed)\n", current->tgid);
return -ERESTARTSYS;
}
if (copy_from_user(command, (void __user*)arg, sizeof(*command))) {
hailo_err(board, "hailo_fw_control, copy_from_user fail\n");
err = -ENOMEM;
goto l_exit;
}
reinit_completion(&board->nnc.fw_control.completion);
err = hailo_pcie_write_firmware_control(&board->pcie_resources, command);
if (err < 0) {
hailo_err(board, "Failed writing fw control to pcie\n");
goto l_exit;
}
// Wait for response
completion_result = wait_for_completion_interruptible_timeout(&board->nnc.fw_control.completion, msecs_to_jiffies(command->timeout_ms));
if (completion_result <= 0) {
if (0 == completion_result) {
hailo_err(board, "hailo_fw_control, timeout waiting for control (timeout_ms=%d)\n", command->timeout_ms);
err = -ETIMEDOUT;
} else {
hailo_info(board, "hailo_fw_control, wait for completion failed with err=%ld (process was interrupted or killed)\n", completion_result);
err = -EINTR;
}
goto l_exit;
}
err = hailo_pcie_read_firmware_control(&board->pcie_resources, command);
if (err < 0) {
hailo_err(board, "Failed reading fw control from pcie\n");
goto l_exit;
}
if (copy_to_user((void __user*)arg, command, sizeof(*command))) {
hailo_err(board, "hailo_fw_control, copy_to_user fail\n");
err = -ENOMEM;
goto l_exit;
}
l_exit:
up(&board->nnc.fw_control.mutex);
return err;
}
static long hailo_get_notification_wait_thread(struct hailo_pcie_board *board, struct file *filp,
struct hailo_notification_wait **current_waiting_thread)
{
struct hailo_notification_wait *cursor = NULL;
// note: safe to access without rcu because the notification_wait_list is closed only on file release
list_for_each_entry(cursor, &board->nnc.notification_wait_list, notification_wait_list)
{
if ((current->tgid == cursor->tgid) && (filp == cursor->filp)) {
*current_waiting_thread = cursor;
return 0;
}
}
return -EFAULT;
}
static long hailo_read_notification_ioctl(struct hailo_pcie_board *board, unsigned long arg, struct file *filp,
bool* should_up_board_mutex)
{
long err = 0;
struct hailo_notification_wait *current_waiting_thread = NULL;
struct hailo_d2h_notification *notification = &board->nnc.notification_to_user;
unsigned long irq_saved_flags;
err = hailo_get_notification_wait_thread(board, filp, &current_waiting_thread);
if (0 != err) {
goto l_exit;
}
up(&board->mutex);
if (0 > (err = wait_for_completion_interruptible(&current_waiting_thread->notification_completion))) {
hailo_info(board,
"HAILO_READ_NOTIFICATION - wait_for_completion_interruptible error. err=%ld. tgid=%d (process was interrupted or killed)\n",
err, current_waiting_thread->tgid);
*should_up_board_mutex = false;
goto l_exit;
}
if (down_interruptible(&board->mutex)) {
hailo_info(board, "HAILO_READ_NOTIFICATION - down_interruptible error (process was interrupted or killed)\n");
*should_up_board_mutex = false;
err = -ERESTARTSYS;
goto l_exit;
}
// Check if was disabled
if (current_waiting_thread->is_disabled) {
hailo_info(board, "HAILO_READ_NOTIFICATION - notification disabled for tgid=%d\n", current->tgid);
err = -ECANCELED;
goto l_exit;
}
reinit_completion(&current_waiting_thread->notification_completion);
spin_lock_irqsave(&board->nnc.notification_read_spinlock, irq_saved_flags);
notification->buffer_len = board->nnc.notification_cache.buffer_len;
memcpy(notification->buffer, board->nnc.notification_cache.buffer, notification->buffer_len);
spin_unlock_irqrestore(&board->nnc.notification_read_spinlock, irq_saved_flags);
if (copy_to_user((void __user*)arg, notification, sizeof(*notification))) {
hailo_err(board, "HAILO_READ_NOTIFICATION copy_to_user fail\n");
err = -ENOMEM;
goto l_exit;
}
l_exit:
return err;
}
static long hailo_disable_notification(struct hailo_pcie_board *board, struct file *filp)
{
struct hailo_notification_wait *cursor = NULL;
hailo_info(board, "HAILO_DISABLE_NOTIFICATION: disable notification");
rcu_read_lock();
list_for_each_entry_rcu(cursor, &board->nnc.notification_wait_list, notification_wait_list) {
if ((current->tgid == cursor->tgid) && (filp == cursor->filp)) {
cursor->is_disabled = true;
complete(&cursor->notification_completion);
break;
}
}
rcu_read_unlock();
return 0;
}
static long hailo_read_log_ioctl(struct hailo_pcie_board *board, unsigned long arg)
{
long err = 0;
struct hailo_read_log_params params;
if (copy_from_user(&params, (void __user*)arg, sizeof(params))) {
hailo_err(board, "HAILO_READ_LOG, copy_from_user fail\n");
return -ENOMEM;
}
if (0 > (err = hailo_pcie_read_firmware_log(&board->pcie_resources.fw_access, &params))) {
hailo_err(board, "HAILO_READ_LOG, reading from log failed with error: %ld \n", err);
return err;
}
if (copy_to_user((void*)arg, &params, sizeof(params))) {
return -ENOMEM;
}
return 0;
}
long hailo_nnc_ioctl(struct hailo_pcie_board *board, unsigned int cmd, unsigned long arg,
struct file *filp, bool *should_up_board_mutex)
{
switch (cmd) {
case HAILO_FW_CONTROL:
return hailo_fw_control(board, arg, should_up_board_mutex);
case HAILO_READ_NOTIFICATION:
return hailo_read_notification_ioctl(board, arg, filp, should_up_board_mutex);
case HAILO_DISABLE_NOTIFICATION:
return hailo_disable_notification(board, filp);
case HAILO_READ_LOG:
return hailo_read_log_ioctl(board, arg);
default:
hailo_err(board, "Invalid nnc ioctl code 0x%x (nr: %d)\n", cmd, _IOC_NR(cmd));
return -ENOTTY;
}
}
static int add_notification_wait(struct hailo_pcie_board *board, struct file *filp)
{
struct hailo_notification_wait *wait = kmalloc(sizeof(*wait), GFP_KERNEL);
if (!wait) {
hailo_err(board, "Failed to allocate notification wait structure.\n");
return -ENOMEM;
}
wait->tgid = current->tgid;
wait->filp = filp;
wait->is_disabled = false;
init_completion(&wait->notification_completion);
list_add_rcu(&wait->notification_wait_list, &board->nnc.notification_wait_list);
return 0;
}
int hailo_nnc_file_context_init(struct hailo_pcie_board *board, struct hailo_file_context *context)
{
return add_notification_wait(board, context->filp);
}
static void clear_notification_wait_list(struct hailo_pcie_board *board, struct file *filp)
{
struct hailo_notification_wait *cur = NULL, *next = NULL;
list_for_each_entry_safe(cur, next, &board->nnc.notification_wait_list, notification_wait_list) {
if (cur->filp == filp) {
list_del_rcu(&cur->notification_wait_list);
synchronize_rcu();
kfree(cur);
}
}
}
int hailo_nnc_driver_down(struct hailo_pcie_board *board)
{
long completion_result = 0;
int err = 0;
reinit_completion(&board->driver_down.reset_completed);
hailo_pcie_write_firmware_driver_shutdown(&board->pcie_resources);
// Wait for response
completion_result =
wait_for_completion_timeout(&board->driver_down.reset_completed, msecs_to_jiffies(DEFAULT_SHUTDOWN_TIMEOUT_MS));
if (completion_result <= 0) {
if (0 == completion_result) {
hailo_err(board, "hailo_nnc_driver_down, timeout waiting for shutdown response (timeout_ms=%d)\n", DEFAULT_SHUTDOWN_TIMEOUT_MS);
err = -ETIMEDOUT;
} else {
hailo_info(board, "hailo_nnc_driver_down, wait for completion failed with err=%ld (process was interrupted or killed)\n",
completion_result);
err = completion_result;
}
goto l_exit;
}
l_exit:
return err;
}
void hailo_nnc_file_context_finalize(struct hailo_pcie_board *board, struct hailo_file_context *context)
{
clear_notification_wait_list(board, context->filp);
if (context->filp == board->vdma.used_by_filp) {
hailo_nnc_driver_down(board);
}
}

22
drivers/media/pci/hailo/src/nnc.h
View File

@@ -1,22 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_PCI_NNC_H_
#define _HAILO_PCI_NNC_H_
#include "pcie.h"
void hailo_nnc_init(struct hailo_pcie_nnc *nnc);
void hailo_nnc_finalize(struct hailo_pcie_nnc *nnc);
long hailo_nnc_ioctl(struct hailo_pcie_board *board, unsigned int cmd, unsigned long arg,
struct file *filp, bool *should_up_board_mutex);
int hailo_nnc_file_context_init(struct hailo_pcie_board *board, struct hailo_file_context *context);
void hailo_nnc_file_context_finalize(struct hailo_pcie_board *board, struct hailo_file_context *context);
int hailo_nnc_driver_down(struct hailo_pcie_board *board);
#endif /* _HAILO_PCI_NNC_H_ */

1563
drivers/media/pci/hailo/src/pcie.c
View File

File diff suppressed because it is too large Load Diff

131
drivers/media/pci/hailo/src/pcie.h
View File

@@ -1,131 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_PCI_PCIE_H_
#define _HAILO_PCI_PCIE_H_
#include "vdma/vdma.h"
#include "hailo_ioctl_common.h"
#include "pcie_common.h"
#include "utils/fw_common.h"
#include <linux/pci.h>
#include <linux/fs.h>
#include <linux/interrupt.h>
#include <linux/circ_buf.h>
#include <linux/device.h>
#include <linux/ioctl.h>
#define HAILO_PCI_OVER_VDMA_NUM_CHANNELS (8)
#define HAILO_PCI_OVER_VDMA_PAGE_SIZE (512)
struct hailo_fw_control_info {
// protects that only one fw control will be send at a time
struct semaphore mutex;
// called from the interrupt handler to notify that a response is ready
struct completion completion;
// the command we are currently handling
struct hailo_fw_control command;
};
struct hailo_pcie_driver_down_info {
// called from the interrupt handler to notify that FW completed reset
struct completion reset_completed;
};
struct hailo_pcie_soft_reset {
// called from the interrupt handler to notify that FW completed reset
struct completion reset_completed;
};
struct hailo_fw_boot {
// the filp that enabled interrupts for fw boot. the interrupt is enabled if this is not null
struct file *filp;
// called from the interrupt handler to notify that an interrupt was raised
struct completion completion;
};
struct hailo_pcie_nnc {
struct hailo_fw_control_info fw_control;
spinlock_t notification_read_spinlock;
struct list_head notification_wait_list;
struct hailo_d2h_notification notification_cache;
struct hailo_d2h_notification notification_to_user;
};
struct hailo_pcie_soc {
struct completion control_resp_ready;
};
// Context for each open file handle
// TODO: store board and use as actual context
struct hailo_file_context {
struct list_head open_files_list;
struct file *filp;
struct hailo_vdma_file_context vdma_context;
bool is_valid;
u32 soc_used_channels_bitmap;
};
struct hailo_pcie_boot_dma_channel_state {
struct hailo_descriptors_list_buffer host_descriptors_buffer;
struct hailo_descriptors_list_buffer device_descriptors_buffer;
struct sg_table sg_table;
u64 buffer_size;
void *kernel_addrs;
u32 desc_program_num;
};
struct hailo_pcie_boot_dma_state {
struct hailo_pcie_boot_dma_channel_state channels[HAILO_PCI_OVER_VDMA_NUM_CHANNELS];
u8 curr_channel_index;
};
struct hailo_pcie_fw_boot {
struct hailo_pcie_boot_dma_state boot_dma_state;
// is_in_boot is set to true when the board is in boot mode
bool is_in_boot;
// boot_used_channel_bitmap is a bitmap of the channels that are used for boot
u16 boot_used_channel_bitmap;
// fw_loaded_completion is used to notify that the FW was loaded - SOC & NNC
struct completion fw_loaded_completion;
// vdma_boot_completion is used to notify that the vDMA boot data was transferred completely on all used channels for boot
struct completion vdma_boot_completion;
};
struct hailo_pcie_board {
struct list_head board_list;
struct pci_dev *pDev;
u32 board_index;
atomic_t ref_count;
struct list_head open_files_list;
struct hailo_pcie_resources pcie_resources;
struct hailo_pcie_nnc nnc;
struct hailo_pcie_soc soc;
struct hailo_pcie_driver_down_info driver_down;
struct hailo_pcie_soft_reset soft_reset;
struct semaphore mutex;
struct hailo_vdma_controller vdma;
struct hailo_pcie_fw_boot fw_boot;
struct hailo_memory_transfer_params memory_transfer_params;
u32 desc_max_page_size;
enum hailo_allocation_mode allocation_mode;
bool interrupts_enabled;
};
bool power_mode_enabled(void);
struct hailo_pcie_board* hailo_pcie_get_board_index(u32 index);
void hailo_disable_interrupts(struct hailo_pcie_board *board);
int hailo_enable_interrupts(struct hailo_pcie_board *board);
int hailo_pcie_soft_reset(struct hailo_pcie_resources *resources, struct completion *reset_completed);
#endif /* _HAILO_PCI_PCIE_H_ */

244
drivers/media/pci/hailo/src/soc.c
View File

@@ -1,244 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
/**
* A Hailo PCIe NNC device is a device contains a full SoC over PCIe. The SoC contains NNC (neural network core) and
* some application processor (pci_ep).
*/
#include "soc.h"
#include "vdma_common.h"
#include "utils/logs.h"
#include "vdma/memory.h"
#include "pcie_common.h"
#include <linux/uaccess.h>
#ifndef HAILO_EMULATOR
#define PCI_SOC_CONTROL_CONNECT_TIMEOUT_MS (1000)
#else
#define PCI_SOC_CONTROL_CONNECT_TIMEOUT_MS (1000000)
#endif /* ifndef HAILO_EMULATOR */
void hailo_soc_init(struct hailo_pcie_soc *soc)
{
init_completion(&soc->control_resp_ready);
}
long hailo_soc_ioctl(struct hailo_pcie_board *board, struct hailo_file_context *context,
struct hailo_vdma_controller *controller, unsigned int cmd, unsigned long arg)
{
switch (cmd) {
case HAILO_SOC_CONNECT:
return hailo_soc_connect_ioctl(board, context, controller, arg);
case HAILO_SOC_CLOSE:
return hailo_soc_close_ioctl(board, controller, context, arg);
default:
hailo_err(board, "Invalid pcie EP ioctl code 0x%x (nr: %d)\n", cmd, _IOC_NR(cmd));
return -ENOTTY;
}
}
static int soc_control(struct hailo_pcie_board *board,
const struct hailo_pcie_soc_request *request,
struct hailo_pcie_soc_response *response)
{
int ret = 0;
reinit_completion(&board->soc.control_resp_ready);
hailo_pcie_soc_write_request(&board->pcie_resources, request);
ret = wait_for_completion_interruptible_timeout(&board->soc.control_resp_ready,
msecs_to_jiffies(PCI_SOC_CONTROL_CONNECT_TIMEOUT_MS));
if (ret <= 0) {
if (0 == ret) {
hailo_err(board, "Timeout waiting for soc control (timeout_ms=%d)\n", PCI_SOC_CONTROL_CONNECT_TIMEOUT_MS);
return -ETIMEDOUT;
} else {
hailo_info(board, "soc control failed with err=%d (process was interrupted or killed)\n",
ret);
return ret;
}
}
hailo_pcie_soc_read_response(&board->pcie_resources, response);
if (response->status < 0) {
hailo_err(board, "soc control failed with status=%d\n", response->status);
return response->status;
}
if (response->control_code != request->control_code) {
hailo_err(board, "Invalid response control code %d (expected %d)\n",
response->control_code, request->control_code);
return -EINVAL;
}
return 0;
}
long hailo_soc_connect_ioctl(struct hailo_pcie_board *board, struct hailo_file_context *context,
struct hailo_vdma_controller *controller, unsigned long arg)
{
struct hailo_pcie_soc_request request = {0};
struct hailo_pcie_soc_response response = {0};
struct hailo_soc_connect_params params;
struct hailo_vdma_channel *input_channel = NULL;
struct hailo_vdma_channel *output_channel = NULL;
struct hailo_vdma_engine *vdma_engine = &controller->vdma_engines[PCI_VDMA_ENGINE_INDEX];
struct hailo_descriptors_list_buffer *input_descriptors_buffer = NULL;
struct hailo_descriptors_list_buffer *output_descriptors_buffer = NULL;
int err = 0;
if (copy_from_user(&params, (void *)arg, sizeof(params))) {
hailo_err(board, "copy_from_user fail\n");
return -ENOMEM;
}
request = (struct hailo_pcie_soc_request) {
.control_code = HAILO_PCIE_SOC_CONTROL_CODE_CONNECT,
.connect = {
.port = params.port_number
}
};
err = soc_control(board, &request, &response);
if (err < 0) {
return err;
}
params.input_channel_index = response.connect.input_channel_index;
params.output_channel_index = response.connect.output_channel_index;
if (!hailo_check_channel_index(params.input_channel_index, controller->hw->src_channels_bitmask, true)) {
hailo_dev_err(&board->pDev->dev, "Invalid input channel index %u\n", params.input_channel_index);
return -EINVAL;
}
if (!hailo_check_channel_index(params.output_channel_index, controller->hw->src_channels_bitmask, false)) {
hailo_dev_err(&board->pDev->dev, "Invalid output channel index %u\n", params.output_channel_index);
return -EINVAL;
}
input_channel = &vdma_engine->channels[params.input_channel_index];
output_channel = &vdma_engine->channels[params.output_channel_index];
input_descriptors_buffer = hailo_vdma_find_descriptors_buffer(&context->vdma_context, params.input_desc_handle);
output_descriptors_buffer = hailo_vdma_find_descriptors_buffer(&context->vdma_context, params.output_desc_handle);
if (NULL == input_descriptors_buffer || NULL == output_descriptors_buffer) {
hailo_dev_err(&board->pDev->dev, "input / output descriptors buffer not found \n");
return -EINVAL;
}
if (!is_powerof2((size_t)input_descriptors_buffer->desc_list.desc_count) ||
!is_powerof2((size_t)output_descriptors_buffer->desc_list.desc_count)) {
hailo_dev_err(&board->pDev->dev, "Invalid desc list size\n");
return -EINVAL;
}
// configure and start input channel
// DMA Direction is only to get channel index - so
err = hailo_vdma_start_channel(input_channel->host_regs, input_descriptors_buffer->dma_address, input_descriptors_buffer->desc_list.desc_count,
board->vdma.hw->ddr_data_id);
if (err < 0) {
hailo_dev_err(&board->pDev->dev, "Error starting vdma input channel index %u\n", params.input_channel_index);
return -EINVAL;
}
// Store the input channels state in bitmap (open)
hailo_set_bit(params.input_channel_index, &context->soc_used_channels_bitmap);
// configure and start output channel
// DMA Direction is only to get channel index - so
err = hailo_vdma_start_channel(output_channel->host_regs, output_descriptors_buffer->dma_address, output_descriptors_buffer->desc_list.desc_count,
board->vdma.hw->ddr_data_id);
if (err < 0) {
hailo_dev_err(&board->pDev->dev, "Error starting vdma output channel index %u\n", params.output_channel_index);
// Close input channel
hailo_vdma_stop_channel(input_channel->host_regs);
return -EINVAL;
}
// Store the output channels state in bitmap (open)
hailo_set_bit(params.output_channel_index, &context->soc_used_channels_bitmap);
if (copy_to_user((void *)arg, &params, sizeof(params))) {
hailo_dev_err(&board->pDev->dev, "copy_to_user fail\n");
return -ENOMEM;
}
return 0;
}
static int close_channels(struct hailo_pcie_board *board, u32 channels_bitmap)
{
struct hailo_pcie_soc_request request = {0};
struct hailo_pcie_soc_response response = {0};
struct hailo_vdma_engine *engine = &board->vdma.vdma_engines[PCI_VDMA_ENGINE_INDEX];
struct hailo_vdma_channel *channel = NULL;
u8 channel_index = 0;
hailo_info(board, "Closing channels bitmap 0x%x\n", channels_bitmap);
for_each_vdma_channel(engine, channel, channel_index) {
if (hailo_test_bit(channel_index, &channels_bitmap)) {
hailo_vdma_stop_channel(channel->host_regs);
}
}
request = (struct hailo_pcie_soc_request) {
.control_code = HAILO_PCIE_SOC_CONTROL_CODE_CLOSE,
.close = {
.channels_bitmap = channels_bitmap
}
};
return soc_control(board, &request, &response);
}
long hailo_soc_close_ioctl(struct hailo_pcie_board *board, struct hailo_vdma_controller *controller,
struct hailo_file_context *context, unsigned long arg)
{
struct hailo_soc_close_params params;
u32 channels_bitmap = 0;
int err = 0;
if (copy_from_user(&params, (void *)arg, sizeof(params))) {
hailo_dev_err(&board->pDev->dev, "copy_from_user fail\n");
return -ENOMEM;
}
// TOOD: check channels are connected
channels_bitmap = (1 << params.input_channel_index) | (1 << params.output_channel_index);
err = close_channels(board, channels_bitmap);
if (0 != err) {
hailo_dev_err(&board->pDev->dev, "Error closing channels\n");
return err;
}
// Store the channel state in bitmap (closed)
hailo_clear_bit(params.input_channel_index, &context->soc_used_channels_bitmap);
hailo_clear_bit(params.output_channel_index, &context->soc_used_channels_bitmap);
return err;
}
int hailo_soc_file_context_init(struct hailo_pcie_board *board, struct hailo_file_context *context)
{
// Nothing to init yet
return 0;
}
void hailo_soc_file_context_finalize(struct hailo_pcie_board *board, struct hailo_file_context *context)
{
// close only channels connected by this (by bitmap)
if (context->soc_used_channels_bitmap != 0) {
close_channels(board, context->soc_used_channels_bitmap);
}
}
int hailo_soc_driver_down(struct hailo_pcie_board *board)
{
return close_channels(board, 0xFFFFFFFF);
}

26
drivers/media/pci/hailo/src/soc.h
View File

@@ -1,26 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_PCI_SOC_IOCTL_H_
#define _HAILO_PCI_SOC_IOCTL_H_
#include "vdma/ioctl.h"
#include "pcie.h"
void hailo_soc_init(struct hailo_pcie_soc *soc);
long hailo_soc_ioctl(struct hailo_pcie_board *board, struct hailo_file_context *context,
struct hailo_vdma_controller *controller, unsigned int cmd, unsigned long arg);
long hailo_soc_connect_ioctl(struct hailo_pcie_board *board, struct hailo_file_context *context,
struct hailo_vdma_controller *controller, unsigned long arg);
long hailo_soc_close_ioctl(struct hailo_pcie_board *board, struct hailo_vdma_controller *controller, struct hailo_file_context *context, unsigned long arg);
int hailo_soc_file_context_init(struct hailo_pcie_board *board, struct hailo_file_context *context);
void hailo_soc_file_context_finalize(struct hailo_pcie_board *board, struct hailo_file_context *context);
int hailo_soc_driver_down(struct hailo_pcie_board *board);
#endif // _HAILO_PCI_SOC_IOCTL_H_

45
drivers/media/pci/hailo/src/sysfs.c
View File

@@ -1,45 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#include "sysfs.h"
#include "pcie.h"
#include <linux/device.h>
#include <linux/sysfs.h>
static ssize_t board_location_show(struct device *dev, struct device_attribute *_attr,
char *buf)
{
struct hailo_pcie_board *board = (struct hailo_pcie_board *)dev_get_drvdata(dev);
const char *dev_info = pci_name(board->pDev);
return sprintf(buf, "%s", dev_info);
}
static DEVICE_ATTR_RO(board_location);
static ssize_t device_id_show(struct device *dev, struct device_attribute *_attr,
char *buf)
{
struct hailo_pcie_board *board = (struct hailo_pcie_board *)dev_get_drvdata(dev);
return sprintf(buf, "%x:%x", board->pDev->vendor, board->pDev->device);
}
static DEVICE_ATTR_RO(device_id);
static ssize_t accelerator_type_show(struct device *dev, struct device_attribute *_attr,
char *buf)
{
struct hailo_pcie_board *board = (struct hailo_pcie_board *)dev_get_drvdata(dev);
return sprintf(buf, "%d", board->pcie_resources.accelerator_type);
}
static DEVICE_ATTR_RO(accelerator_type);
static struct attribute *hailo_dev_attrs[] = {
&dev_attr_board_location.attr,
&dev_attr_device_id.attr,
&dev_attr_accelerator_type.attr,
NULL
};
ATTRIBUTE_GROUPS(hailo_dev);
const struct attribute_group **g_hailo_dev_groups = hailo_dev_groups;

13
drivers/media/pci/hailo/src/sysfs.h
View File

@@ -1,13 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_PCI_SYSFS_H_
#define _HAILO_PCI_SYSFS_H_
#include <linux/sysfs.h>
extern const struct attribute_group **g_hailo_dev_groups;
#endif /* _HAILO_PCI_SYSFS_H_ */

21
drivers/media/pci/hailo/src/utils.h
View File

@@ -1,21 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2022 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_PCI_UTILS_H_
#define _HAILO_PCI_UTILS_H_
#include <linux/version.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
#include <linux/pagemap.h>
#include "pcie.h"
void hailo_pcie_clear_notification_wait_list(struct hailo_pcie_board *pBoard, struct file *filp);
#endif /* _HAILO_PCI_UTILS_H_ */

161
drivers/media/pci/hailo/utils/compact.h
View File

@@ -1,161 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_PCI_COMPACT_H_
#define _HAILO_PCI_COMPACT_H_
#include <linux/version.h>
#include <linux/scatterlist.h>
#include <linux/vmalloc.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)
#define class_create_compat class_create
#else
#define class_create_compat(name) class_create(THIS_MODULE, name)
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0)
#define pci_printk(level, pdev, fmt, arg...) \
dev_printk(level, &(pdev)->dev, fmt, ##arg)
#define pci_emerg(pdev, fmt, arg...) dev_emerg(&(pdev)->dev, fmt, ##arg)
#define pci_alert(pdev, fmt, arg...) dev_alert(&(pdev)->dev, fmt, ##arg)
#define pci_crit(pdev, fmt, arg...) dev_crit(&(pdev)->dev, fmt, ##arg)
#define pci_err(pdev, fmt, arg...) dev_err(&(pdev)->dev, fmt, ##arg)
#define pci_warn(pdev, fmt, arg...) dev_warn(&(pdev)->dev, fmt, ##arg)
#define pci_notice(pdev, fmt, arg...) dev_notice(&(pdev)->dev, fmt, ##arg)
#define pci_info(pdev, fmt, arg...) dev_info(&(pdev)->dev, fmt, ##arg)
#define pci_dbg(pdev, fmt, arg...) dev_dbg(&(pdev)->dev, fmt, ##arg)
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0)
#define get_user_pages_compact get_user_pages
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)
#define get_user_pages_compact(start, nr_pages, gup_flags, pages) \
get_user_pages(start, nr_pages, gup_flags, pages, NULL)
#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 168)) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0))
#define get_user_pages_compact(start, nr_pages, gup_flags, pages) \
get_user_pages(current, current->mm, start, nr_pages, gup_flags, pages, NULL)
#else
static inline long get_user_pages_compact(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages)
{
int write = !!((gup_flags & FOLL_WRITE) == FOLL_WRITE);
int force = !!((gup_flags & FOLL_FORCE) == FOLL_FORCE);
return get_user_pages(current, current->mm, start, nr_pages, write, force,
pages, NULL);
}
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0)
static inline void dma_sync_sgtable_for_device(struct device *dev,
struct sg_table *sgt, enum dma_data_direction dir)
{
dma_sync_sg_for_device(dev, sgt->sgl, sgt->orig_nents, dir);
}
#endif
#ifndef _LINUX_MMAP_LOCK_H
static inline void mmap_read_lock(struct mm_struct *mm)
{
down_read(&mm->mmap_sem);
}
static inline void mmap_read_unlock(struct mm_struct *mm)
{
up_read(&mm->mmap_sem);
}
#endif /* _LINUX_MMAP_LOCK_H */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(5, 15, 0)
#define sg_alloc_table_from_pages_segment_compat __sg_alloc_table_from_pages
#else
static inline struct scatterlist *sg_alloc_table_from_pages_segment_compat(struct sg_table *sgt,
struct page **pages, unsigned int n_pages, unsigned int offset,
unsigned long size, unsigned int max_segment,
struct scatterlist *prv, unsigned int left_pages,
gfp_t gfp_mask)
{
int res = 0;
if (NULL != prv) {
// prv not suported
return ERR_PTR(-EINVAL);
}
if (0 != left_pages) {
// Left pages not supported
return ERR_PTR(-EINVAL);
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)
res = sg_alloc_table_from_pages_segment(sgt, pages, n_pages, offset, size, max_segment, gfp_mask);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0)
res = __sg_alloc_table_from_pages(sgt, pages, n_pages, offset, size, max_segment, gfp_mask);
#else
res = sg_alloc_table_from_pages(sgt, pages, n_pages, offset, size, gfp_mask);
#endif
if (res < 0) {
return ERR_PTR(res);
}
return sgt->sgl;
}
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION( 5, 0, 0 )
#define compatible_access_ok(a,b,c) access_ok(b, c)
#else
#define compatible_access_ok(a,b,c) access_ok(a, b, c)
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)
#define PCI_DEVICE_DATA(vend, dev, data) \
.vendor = PCI_VENDOR_ID_##vend, .device = PCI_DEVICE_ID_##vend##_##dev, \
.subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, 0, 0, \
.driver_data = (kernel_ulong_t)(data)
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)
// On kernels < 4.1.12, kvmalloc, kvfree is not implemented. For simplicity, instead of implement our own
// kvmalloc/kvfree, just using vmalloc and vfree (It may reduce allocate/access performance, but it worth it).
static inline void *kvmalloc_array(size_t n, size_t size, gfp_t flags)
{
(void)flags; //ignore
return vmalloc(n * size);
}
#define kvfree vfree
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)
static inline bool is_dma_capable(struct device *dev, dma_addr_t dma_addr, size_t size)
{
// Case for Rasberry Pie kernel versions 5.4.83 <=> 5.5.0 - already changed bus_dma_mask -> bus_dma_limit
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0)) || (defined(HAILO_RASBERRY_PIE) && LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 83))
const u64 bus_dma_limit = dev->bus_dma_limit;
#else
const u64 bus_dma_limit = dev->bus_dma_mask;
#endif
return (dma_addr <= min_not_zero(*dev->dma_mask, bus_dma_limit));
}
#else
static inline bool is_dma_capable(struct device *dev, dma_addr_t dma_addr, size_t size)
{
// Implementation of dma_capable from linux kernel
const u64 bus_dma_limit = (*dev->dma_mask + 1) & ~(*dev->dma_mask);
if (bus_dma_limit && size > bus_dma_limit) {
return false;
}
if ((dma_addr | (dma_addr + size - 1)) & ~(*dev->dma_mask)) {
return false;
}
return true;
}
#endif // LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)
#endif /* _HAILO_PCI_COMPACT_H_ */

19
drivers/media/pci/hailo/utils/fw_common.h
View File

@@ -1,19 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_LINUX_COMMON_H_
#define _HAILO_LINUX_COMMON_H_
#include "hailo_ioctl_common.h"
struct hailo_notification_wait {
struct list_head notification_wait_list;
int tgid;
struct file* filp;
struct completion notification_completion;
bool is_disabled;
};
#endif /* _HAILO_LINUX_COMMON_H_ */

109
drivers/media/pci/hailo/utils/integrated_nnc_utils.c
View File

@@ -1,109 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#include "integrated_nnc_utils.h"
#include "utils/logs.h"
#include <linux/uaccess.h>
#include <asm/io.h>
#include <linux/of_address.h>
#include <linux/cdev.h>
int hailo_ioremap_resource(struct platform_device *pdev, struct hailo_resource *resource,
const char *name)
{
void __iomem *address;
struct resource *platform_resource = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
if (NULL == platform_resource) {
return -ENOENT;
}
address = devm_ioremap_resource(&pdev->dev, platform_resource);
if (IS_ERR(address)) {
return PTR_ERR(address);
}
resource->address = (uintptr_t)address;
resource->size = resource_size(platform_resource);
hailo_dev_dbg(&pdev->dev, "resource[%s]: remap %pr of %zx bytes to virtual start address %lx\n",
platform_resource->name, platform_resource, resource->size, (uintptr_t)address);
return 0;
}
// TODO: HRT-8475 - change to name instead of index
int hailo_ioremap_shmem(struct platform_device *pdev, int index, struct hailo_resource *resource)
{
int ret;
struct resource res;
struct device_node *shmem;
void __iomem * remap_ptr;
shmem = of_parse_phandle(pdev->dev.of_node, "shmem", index);
if (!shmem) {
hailo_dev_err(&pdev->dev, "Failed to find shmem node index: %d in device tree\n", index);
return -ENODEV;
}
ret = of_address_to_resource(shmem, 0, &res);
if (ret) {
hailo_dev_err(&pdev->dev, "hailo_ioremap_shmem, failed to get memory (index: %d)\n", index);
of_node_put(shmem);
return ret;
}
// Decrement the refcount of the node
of_node_put(shmem);
remap_ptr = devm_ioremap(&pdev->dev, res.start, resource_size(&res));
if (!remap_ptr) {
hailo_dev_err(&pdev->dev, "hailo_ioremap_shmem, failed to ioremap shmem (index: %d)\n", index);
return -EADDRNOTAVAIL;
}
resource->address = (uintptr_t)remap_ptr;
resource->size = resource_size(&res);
return 0;
}
int direct_memory_transfer(struct platform_device *pdev, struct hailo_memory_transfer_params *params)
{
int err = -EINVAL;
void __iomem *mem = ioremap(params->address, params->count);
if (NULL == mem) {
hailo_dev_err(&pdev->dev, "Failed ioremap %llu %zu\n", params->address, params->count);
return -ENOMEM;
}
switch (params->transfer_direction) {
case TRANSFER_READ:
memcpy_fromio(params->buffer, mem, params->count);
err = 0;
break;
case TRANSFER_WRITE:
memcpy_toio(mem, params->buffer, params->count);
err = 0;
break;
default:
hailo_dev_err(&pdev->dev, "Invalid transfer direction %d\n", (int)params->transfer_direction);
err = -EINVAL;
}
iounmap(mem);
return err;
}
int hailo_get_resource_physical_addr(struct platform_device *pdev, const char *name, u64 *address)
{
struct resource *platform_resource = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
if (NULL == platform_resource) {
return -ENOENT;
}
*address = (u64)(platform_resource->start);
return 0;
}

30
drivers/media/pci/hailo/utils/integrated_nnc_utils.h
View File

@@ -1,30 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _INTEGRATED_NNC_UTILS_H_
#define _INTEGRATED_NNC_UTILS_H_
#include <linux/platform_device.h>
#include "hailo_resource.h"
#define HAILO15_CORE_CONTROL_MAILBOX_INDEX (0)
#define HAILO15_CORE_NOTIFICATION_MAILBOX_INDEX (1)
#define HAILO15_CORE_DRIVER_DOWN_MAILBOX_INDEX (2)
#define HAILO15_CORE_CONTROL_MAILBOX_TX_SHMEM_INDEX (0)
#define HAILO15_CORE_CONTROL_MAILBOX_RX_SHMEM_INDEX (1)
#define HAILO15_CORE_NOTIFICATION_MAILBOX_RX_SHMEM_INDEX (2)
int hailo_ioremap_resource(struct platform_device *pdev, struct hailo_resource *resource,
const char *name);
// TODO: HRT-8475 - change to name instead of index
int hailo_ioremap_shmem(struct platform_device *pdev, int index, struct hailo_resource *resource);
int direct_memory_transfer(struct platform_device *pDev, struct hailo_memory_transfer_params *params);
int hailo_get_resource_physical_addr(struct platform_device *pdev, const char *name, u64 *address);
#endif /* _INTEGRATED_NNC_UTILS_H_ */

8
drivers/media/pci/hailo/utils/logs.c
View File

@@ -1,8 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#include "logs.h"
int o_dbg = LOGLEVEL_NOTICE;

45
drivers/media/pci/hailo/utils/logs.h
View File

@@ -1,45 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _COMMON_LOGS_H_
#define _COMMON_LOGS_H_
#include <linux/kern_levels.h>
// Should be used only by "module_param".
// Specify the current debug level for the logs
extern int o_dbg;
// Logging, same interface as dev_*, uses o_dbg to filter
// log messages
#define hailo_printk(level, dev, fmt, ...) \
do { \
int __level = (level[1] - '0'); \
if (__level <= o_dbg) { \
dev_printk((level), dev, fmt, ##__VA_ARGS__); \
} \
} while (0)
#define hailo_emerg(board, fmt, ...) hailo_printk(KERN_EMERG, &(board)->pDev->dev, fmt, ##__VA_ARGS__)
#define hailo_alert(board, fmt, ...) hailo_printk(KERN_ALERT, &(board)->pDev->dev, fmt, ##__VA_ARGS__)
#define hailo_crit(board, fmt, ...) hailo_printk(KERN_CRIT, &(board)->pDev->dev, fmt, ##__VA_ARGS__)
#define hailo_err(board, fmt, ...) hailo_printk(KERN_ERR, &(board)->pDev->dev, fmt, ##__VA_ARGS__)
#define hailo_warn(board, fmt, ...) hailo_printk(KERN_WARNING, &(board)->pDev->dev, fmt, ##__VA_ARGS__)
#define hailo_notice(board, fmt, ...) hailo_printk(KERN_NOTICE, &(board)->pDev->dev, fmt, ##__VA_ARGS__)
#define hailo_info(board, fmt, ...) hailo_printk(KERN_INFO, &(board)->pDev->dev, fmt, ##__VA_ARGS__)
#define hailo_dbg(board, fmt, ...) hailo_printk(KERN_DEBUG, &(board)->pDev->dev, fmt, ##__VA_ARGS__)
#define hailo_dev_emerg(dev, fmt, ...) hailo_printk(KERN_EMERG, dev, fmt, ##__VA_ARGS__)
#define hailo_dev_alert(dev, fmt, ...) hailo_printk(KERN_ALERT, dev, fmt, ##__VA_ARGS__)
#define hailo_dev_crit(dev, fmt, ...) hailo_printk(KERN_CRIT, dev, fmt, ##__VA_ARGS__)
#define hailo_dev_err(dev, fmt, ...) hailo_printk(KERN_ERR, dev, fmt, ##__VA_ARGS__)
#define hailo_dev_warn(dev, fmt, ...) hailo_printk(KERN_WARNING, dev, fmt, ##__VA_ARGS__)
#define hailo_dev_notice(dev, fmt, ...) hailo_printk(KERN_NOTICE, dev, fmt, ##__VA_ARGS__)
#define hailo_dev_info(dev, fmt, ...) hailo_printk(KERN_INFO, dev, fmt, ##__VA_ARGS__)
#define hailo_dev_dbg(dev, fmt, ...) hailo_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__)
#endif //_COMMON_LOGS_H_

721
drivers/media/pci/hailo/vdma/ioctl.c
View File

@@ -1,721 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#include "ioctl.h"
#include "memory.h"
#include "utils/logs.h"
#include "utils.h"
#include <linux/slab.h>
#include <linux/uaccess.h>
long hailo_vdma_enable_channels_ioctl(struct hailo_vdma_controller *controller, unsigned long arg, struct hailo_vdma_file_context *context)
{
struct hailo_vdma_enable_channels_params input;
struct hailo_vdma_engine *engine = NULL;
u8 engine_index = 0;
u32 channels_bitmap = 0;
if (copy_from_user(&input, (void *)arg, sizeof(input))) {
hailo_dev_err(controller->dev, "copy_from_user fail\n");
return -ENOMEM;
}
// Validate params (ignoring engine_index >= controller->vdma_engines_count).
for_each_vdma_engine(controller, engine, engine_index) {
channels_bitmap = input.channels_bitmap_per_engine[engine_index];
if (0 != (channels_bitmap & engine->enabled_channels)) {
hailo_dev_err(controller->dev, "Trying to enable channels that are already enabled\n");
return -EINVAL;
}
}
for_each_vdma_engine(controller, engine, engine_index) {
channels_bitmap = input.channels_bitmap_per_engine[engine_index];
hailo_vdma_engine_enable_channels(engine, channels_bitmap,
input.enable_timestamps_measure);
hailo_vdma_update_interrupts_mask(controller, engine_index);
hailo_dev_info(controller->dev, "Enabled interrupts for engine %u, channels bitmap 0x%x\n",
engine_index, channels_bitmap);
// Update the context with the enabled channels bitmap
context->enabled_channels_bitmap[engine_index] |= channels_bitmap;
}
return 0;
}
long hailo_vdma_disable_channels_ioctl(struct hailo_vdma_controller *controller, unsigned long arg, struct hailo_vdma_file_context *context)
{
struct hailo_vdma_disable_channels_params input;
struct hailo_vdma_engine *engine = NULL;
u8 engine_index = 0;
u32 channels_bitmap = 0;
unsigned long irq_saved_flags = 0;
if (copy_from_user(&input, (void*)arg, sizeof(input))) {
hailo_dev_err(controller->dev, "copy_from_user fail\n");
return -ENOMEM;
}
// Validate params (ignoring engine_index >= controller->vdma_engines_count).
for_each_vdma_engine(controller, engine, engine_index) {
channels_bitmap = input.channels_bitmap_per_engine[engine_index];
if (channels_bitmap != (channels_bitmap & engine->enabled_channels)) {
hailo_dev_warn(controller->dev, "Trying to disable channels that were not enabled\n");
}
}
for_each_vdma_engine(controller, engine, engine_index) {
channels_bitmap = input.channels_bitmap_per_engine[engine_index];
hailo_vdma_engine_disable_channels(engine, channels_bitmap);
hailo_vdma_update_interrupts_mask(controller, engine_index);
spin_lock_irqsave(&controller->interrupts_lock, irq_saved_flags);
hailo_vdma_engine_clear_channel_interrupts(engine, channels_bitmap);
spin_unlock_irqrestore(&controller->interrupts_lock, irq_saved_flags);
hailo_dev_info(controller->dev, "Disabled channels for engine %u, bitmap 0x%x\n",
engine_index, channels_bitmap);
// Update the context with the disabled channels bitmap
context->enabled_channels_bitmap[engine_index] &= ~channels_bitmap;
}
// Wake up threads waiting
wake_up_interruptible_all(&controller->interrupts_wq);
return 0;
}
static bool got_interrupt(struct hailo_vdma_controller *controller,
u32 channels_bitmap_per_engine[MAX_VDMA_ENGINES])
{
struct hailo_vdma_engine *engine = NULL;
u8 engine_index = 0;
for_each_vdma_engine(controller, engine, engine_index) {
if (hailo_vdma_engine_got_interrupt(engine,
channels_bitmap_per_engine[engine_index])) {
return true;
}
}
return false;
}
static void transfer_done(struct hailo_ongoing_transfer *transfer, void *opaque)
{
u8 i = 0;
struct hailo_vdma_controller *controller = (struct hailo_vdma_controller *)opaque;
for (i = 0; i < transfer->buffers_count; i++) {
struct hailo_vdma_buffer *mapped_buffer = (struct hailo_vdma_buffer *)transfer->buffers[i].opaque;
hailo_vdma_buffer_sync_cyclic(controller, mapped_buffer, HAILO_SYNC_FOR_CPU,
transfer->buffers[i].offset, transfer->buffers[i].size);
}
}
long hailo_vdma_interrupts_wait_ioctl(struct hailo_vdma_controller *controller, unsigned long arg,
struct semaphore *mutex, bool *should_up_board_mutex)
{
long err = 0;
struct hailo_vdma_interrupts_wait_params params = {0};
struct hailo_vdma_engine *engine = NULL;
bool bitmap_not_empty = false;
u8 engine_index = 0;
u32 irq_bitmap = 0;
unsigned long irq_saved_flags = 0;
if (copy_from_user(&params, (void*)arg, sizeof(params))) {
hailo_dev_err(controller->dev, "HAILO_VDMA_INTERRUPTS_WAIT, copy_from_user fail\n");
return -ENOMEM;
}
// We don't need to validate that channels_bitmap_per_engine are enabled -
// If the channel is not enabled we just return an empty interrupts list.
// Validate params (ignoring engine_index >= controller->vdma_engines_count).
// It us ok to wait on a disabled channel - the wait will just exit.
for_each_vdma_engine(controller, engine, engine_index) {
if (0 != params.channels_bitmap_per_engine[engine_index]) {
bitmap_not_empty = true;
}
}
if (!bitmap_not_empty) {
hailo_dev_err(controller->dev, "Got an empty bitmap for wait interrupts\n");
return -EINVAL;
}
up(mutex);
err = wait_event_interruptible(controller->interrupts_wq,
got_interrupt(controller, params.channels_bitmap_per_engine));
if (err < 0) {
hailo_dev_info(controller->dev,
"wait channel interrupts failed with err=%ld (process was interrupted or killed)\n", err);
*should_up_board_mutex = false;
return err;
}
if (down_interruptible(mutex)) {
hailo_dev_info(controller->dev, "down_interruptible error (process was interrupted or killed)\n");
*should_up_board_mutex = false;
return -ERESTARTSYS;
}
params.channels_count = 0;
for_each_vdma_engine(controller, engine, engine_index) {
spin_lock_irqsave(&controller->interrupts_lock, irq_saved_flags);
irq_bitmap = hailo_vdma_engine_read_interrupts(engine,
params.channels_bitmap_per_engine[engine->index]);
spin_unlock_irqrestore(&controller->interrupts_lock, irq_saved_flags);
err = hailo_vdma_engine_fill_irq_data(&params, engine, irq_bitmap,
transfer_done, controller);
if (err < 0) {
hailo_dev_err(controller->dev, "Failed fill irq data %ld", err);
return err;
}
}
if (copy_to_user((void __user*)arg, &params, sizeof(params))) {
hailo_dev_err(controller->dev, "copy_to_user fail\n");
return -ENOMEM;
}
return 0;
}
static uintptr_t hailo_get_next_vdma_handle(struct hailo_vdma_file_context *context)
{
// Note: The kernel code left-shifts the 'offset' param from the user-space call to mmap by PAGE_SHIFT bits and
// stores the result in 'vm_area_struct.vm_pgoff'. We pass the desc_handle to mmap in the offset param. To
// counter this, we right-shift the desc_handle. See also 'mmap function'.
uintptr_t next_handle = 0;
next_handle = atomic_inc_return(&context->last_vdma_handle);
return (next_handle << PAGE_SHIFT);
}
long hailo_vdma_buffer_map_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
unsigned long arg)
{
struct hailo_vdma_buffer_map_params buf_info;
struct hailo_vdma_buffer *mapped_buffer = NULL;
enum dma_data_direction direction = DMA_NONE;
struct hailo_vdma_low_memory_buffer *low_memory_buffer = NULL;
if (copy_from_user(&buf_info, (void __user*)arg, sizeof(buf_info))) {
hailo_dev_err(controller->dev, "copy from user fail\n");
return -EFAULT;
}
hailo_dev_dbg(controller->dev, "address %lx tgid %d size: %zu\n",
buf_info.user_address, current->tgid, buf_info.size);
direction = get_dma_direction(buf_info.data_direction);
if (DMA_NONE == direction) {
hailo_dev_err(controller->dev, "invalid data direction %d\n", buf_info.data_direction);
return -EINVAL;
}
low_memory_buffer = hailo_vdma_find_low_memory_buffer(context, buf_info.allocated_buffer_handle);
mapped_buffer = hailo_vdma_buffer_map(controller->dev,
buf_info.user_address, buf_info.size, direction, buf_info.buffer_type, low_memory_buffer);
if (IS_ERR(mapped_buffer)) {
hailo_dev_err(controller->dev, "failed map buffer %lx\n", buf_info.user_address);
return PTR_ERR(mapped_buffer);
}
mapped_buffer->handle = atomic_inc_return(&context->last_vdma_user_buffer_handle);
buf_info.mapped_handle = mapped_buffer->handle;
if (copy_to_user((void __user*)arg, &buf_info, sizeof(buf_info))) {
hailo_dev_err(controller->dev, "copy_to_user fail\n");
hailo_vdma_buffer_put(mapped_buffer);
return -EFAULT;
}
list_add(&mapped_buffer->mapped_user_buffer_list, &context->mapped_user_buffer_list);
hailo_dev_dbg(controller->dev, "buffer %lx (handle %zu) is mapped\n",
buf_info.user_address, buf_info.mapped_handle);
return 0;
}
long hailo_vdma_buffer_unmap_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
unsigned long arg)
{
struct hailo_vdma_buffer *mapped_buffer = NULL;
struct hailo_vdma_buffer_unmap_params buffer_unmap_params;
if (copy_from_user(&buffer_unmap_params, (void __user*)arg, sizeof(buffer_unmap_params))) {
hailo_dev_err(controller->dev, "copy from user fail\n");
return -EFAULT;
}
hailo_dev_dbg(controller->dev, "unmap user buffer handle %zu\n", buffer_unmap_params.mapped_handle);
mapped_buffer = hailo_vdma_find_mapped_user_buffer(context, buffer_unmap_params.mapped_handle);
if (mapped_buffer == NULL) {
hailo_dev_warn(controller->dev, "buffer handle %zu not found\n", buffer_unmap_params.mapped_handle);
return -EINVAL;
}
list_del(&mapped_buffer->mapped_user_buffer_list);
hailo_vdma_buffer_put(mapped_buffer);
return 0;
}
long hailo_vdma_buffer_sync_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg)
{
struct hailo_vdma_buffer_sync_params sync_info = {};
struct hailo_vdma_buffer *mapped_buffer = NULL;
if (copy_from_user(&sync_info, (void __user*)arg, sizeof(sync_info))) {
hailo_dev_err(controller->dev, "copy_from_user fail\n");
return -EFAULT;
}
if (!(mapped_buffer = hailo_vdma_find_mapped_user_buffer(context, sync_info.handle))) {
hailo_dev_err(controller->dev, "buffer handle %zu doesn't exist\n", sync_info.handle);
return -EINVAL;
}
if ((sync_info.sync_type != HAILO_SYNC_FOR_CPU) && (sync_info.sync_type != HAILO_SYNC_FOR_DEVICE)) {
hailo_dev_err(controller->dev, "Invalid sync_type given for vdma buffer sync.\n");
return -EINVAL;
}
if (sync_info.offset + sync_info.count > mapped_buffer->size) {
hailo_dev_err(controller->dev, "Invalid offset/count given for vdma buffer sync. offset %zu count %zu buffer size %u\n",
sync_info.offset, sync_info.count, mapped_buffer->size);
return -EINVAL;
}
hailo_vdma_buffer_sync(controller, mapped_buffer, sync_info.sync_type,
sync_info.offset, sync_info.count);
return 0;
}
long hailo_desc_list_create_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
unsigned long arg)
{
struct hailo_desc_list_create_params params;
struct hailo_descriptors_list_buffer *descriptors_buffer = NULL;
uintptr_t next_handle = 0;
long err = -EINVAL;
if (copy_from_user(&params, (void __user*)arg, sizeof(params))) {
hailo_dev_err(controller->dev, "copy_from_user fail\n");
return -EFAULT;
}
if (params.is_circular && !is_powerof2(params.desc_count)) {
hailo_dev_err(controller->dev, "Invalid desc count given : %zu , circular descriptors count must be power of 2\n",
params.desc_count);
return -EINVAL;
}
if (!is_powerof2(params.desc_page_size)) {
hailo_dev_err(controller->dev, "Invalid desc page size given : %u\n",
params.desc_page_size);
return -EINVAL;
}
hailo_dev_info(controller->dev,
"Create desc list desc_count: %zu desc_page_size: %u\n",
params.desc_count, params.desc_page_size);
descriptors_buffer = kzalloc(sizeof(*descriptors_buffer), GFP_KERNEL);
if (NULL == descriptors_buffer) {
hailo_dev_err(controller->dev, "Failed to allocate buffer for descriptors list struct\n");
return -ENOMEM;
}
next_handle = hailo_get_next_vdma_handle(context);
err = hailo_desc_list_create(controller->dev, params.desc_count,
params.desc_page_size, next_handle, params.is_circular,
descriptors_buffer);
if (err < 0) {
hailo_dev_err(controller->dev, "failed to allocate descriptors buffer\n");
kfree(descriptors_buffer);
return err;
}
list_add(&descriptors_buffer->descriptors_buffer_list, &context->descriptors_buffer_list);
// Note: The physical address is required for CONTEXT_SWITCH firmware controls
BUILD_BUG_ON(sizeof(params.dma_address) < sizeof(descriptors_buffer->dma_address));
params.dma_address = descriptors_buffer->dma_address;
params.desc_handle = descriptors_buffer->handle;
if(copy_to_user((void __user*)arg, &params, sizeof(params))){
hailo_dev_err(controller->dev, "copy_to_user fail\n");
list_del(&descriptors_buffer->descriptors_buffer_list);
hailo_desc_list_release(controller->dev, descriptors_buffer);
kfree(descriptors_buffer);
return -EFAULT;
}
hailo_dev_info(controller->dev, "Created desc list, handle 0x%llu\n",
(u64)params.desc_handle);
return 0;
}
long hailo_desc_list_release_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
unsigned long arg)
{
struct hailo_desc_list_release_params params;
struct hailo_descriptors_list_buffer *descriptors_buffer = NULL;
if (copy_from_user(&params, (void __user*)arg, sizeof(params))) {
hailo_dev_err(controller->dev, "copy_from_user fail\n");
return -EFAULT;
}
descriptors_buffer = hailo_vdma_find_descriptors_buffer(context, params.desc_handle);
if (descriptors_buffer == NULL) {
hailo_dev_warn(controller->dev, "not found desc handle %llu\n", (unsigned long long)params.desc_handle);
return -EINVAL;
}
list_del(&descriptors_buffer->descriptors_buffer_list);
hailo_desc_list_release(controller->dev, descriptors_buffer);
kfree(descriptors_buffer);
return 0;
}
long hailo_desc_list_program_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
unsigned long arg)
{
struct hailo_desc_list_program_params configure_info;
struct hailo_vdma_buffer *mapped_buffer = NULL;
struct hailo_descriptors_list_buffer *descriptors_buffer = NULL;
struct hailo_vdma_mapped_transfer_buffer transfer_buffer = {0};
if (copy_from_user(&configure_info, (void __user*)arg, sizeof(configure_info))) {
hailo_dev_err(controller->dev, "copy from user fail\n");
return -EFAULT;
}
hailo_dev_info(controller->dev, "config buffer_handle=%zu desc_handle=%llu starting_desc=%u\n",
configure_info.buffer_handle, (u64)configure_info.desc_handle, configure_info.starting_desc);
mapped_buffer = hailo_vdma_find_mapped_user_buffer(context, configure_info.buffer_handle);
descriptors_buffer = hailo_vdma_find_descriptors_buffer(context, configure_info.desc_handle);
if (mapped_buffer == NULL || descriptors_buffer == NULL) {
hailo_dev_err(controller->dev, "invalid user/descriptors buffer\n");
return -EFAULT;
}
if (configure_info.buffer_size > mapped_buffer->size) {
hailo_dev_err(controller->dev, "invalid buffer size. \n");
return -EFAULT;
}
transfer_buffer.sg_table = &mapped_buffer->sg_table;
transfer_buffer.size = configure_info.buffer_size;
transfer_buffer.offset = configure_info.buffer_offset;
return hailo_vdma_program_descriptors_list(
controller->hw,
&descriptors_buffer->desc_list,
configure_info.starting_desc,
&transfer_buffer,
configure_info.should_bind,
configure_info.channel_index,
configure_info.last_interrupts_domain,
configure_info.is_debug
);
}
long hailo_vdma_low_memory_buffer_alloc_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
unsigned long arg)
{
struct hailo_allocate_low_memory_buffer_params buf_info = {0};
struct hailo_vdma_low_memory_buffer *low_memory_buffer = NULL;
long err = -EINVAL;
if (copy_from_user(&buf_info, (void __user*)arg, sizeof(buf_info))) {
hailo_dev_err(controller->dev, "copy from user fail\n");
return -EFAULT;
}
low_memory_buffer = kzalloc(sizeof(*low_memory_buffer), GFP_KERNEL);
if (NULL == low_memory_buffer) {
hailo_dev_err(controller->dev, "memory alloc failed\n");
return -ENOMEM;
}
err = hailo_vdma_low_memory_buffer_alloc(buf_info.buffer_size, low_memory_buffer);
if (err < 0) {
kfree(low_memory_buffer);
hailo_dev_err(controller->dev, "failed allocating buffer from driver\n");
return err;
}
// Get handle for allocated buffer
low_memory_buffer->handle = hailo_get_next_vdma_handle(context);
list_add(&low_memory_buffer->vdma_low_memory_buffer_list, &context->vdma_low_memory_buffer_list);
buf_info.buffer_handle = low_memory_buffer->handle;
if (copy_to_user((void __user*)arg, &buf_info, sizeof(buf_info))) {
hailo_dev_err(controller->dev, "copy_to_user fail\n");
list_del(&low_memory_buffer->vdma_low_memory_buffer_list);
hailo_vdma_low_memory_buffer_free(low_memory_buffer);
kfree(low_memory_buffer);
return -EFAULT;
}
return 0;
}
long hailo_vdma_low_memory_buffer_free_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
unsigned long arg)
{
struct hailo_vdma_low_memory_buffer *low_memory_buffer = NULL;
struct hailo_free_low_memory_buffer_params params = {0};
if (copy_from_user(&params, (void __user*)arg, sizeof(params))) {
hailo_dev_err(controller->dev, "copy from user fail\n");
return -EFAULT;
}
low_memory_buffer = hailo_vdma_find_low_memory_buffer(context, params.buffer_handle);
if (NULL == low_memory_buffer) {
hailo_dev_warn(controller->dev, "vdma buffer handle %lx not found\n", params.buffer_handle);
return -EINVAL;
}
list_del(&low_memory_buffer->vdma_low_memory_buffer_list);
hailo_vdma_low_memory_buffer_free(low_memory_buffer);
kfree(low_memory_buffer);
return 0;
}
long hailo_mark_as_in_use(struct hailo_vdma_controller *controller, unsigned long arg, struct file *filp)
{
struct hailo_mark_as_in_use_params params = {0};
// If device is used by this FD, return false to indicate its free for usage
if (filp == controller->used_by_filp) {
params.in_use = false;
} else if (NULL != controller->used_by_filp) {
params.in_use = true;
} else {
controller->used_by_filp = filp;
params.in_use = false;
}
if (copy_to_user((void __user*)arg, &params, sizeof(params))) {
hailo_dev_err(controller->dev, "copy_to_user fail\n");
return -EFAULT;
}
return 0;
}
long hailo_vdma_continuous_buffer_alloc_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg)
{
struct hailo_allocate_continuous_buffer_params buf_info = {0};
struct hailo_vdma_continuous_buffer *continuous_buffer = NULL;
long err = -EINVAL;
size_t aligned_buffer_size = 0;
if (copy_from_user(&buf_info, (void __user*)arg, sizeof(buf_info))) {
hailo_dev_err(controller->dev, "copy from user fail\n");
return -EFAULT;
}
continuous_buffer = kzalloc(sizeof(*continuous_buffer), GFP_KERNEL);
if (NULL == continuous_buffer) {
hailo_dev_err(controller->dev, "memory alloc failed\n");
return -ENOMEM;
}
// We use PAGE_ALIGN to support mmap
aligned_buffer_size = PAGE_ALIGN(buf_info.buffer_size);
err = hailo_vdma_continuous_buffer_alloc(controller->dev, aligned_buffer_size, continuous_buffer);
if (err < 0) {
kfree(continuous_buffer);
return err;
}
continuous_buffer->handle = hailo_get_next_vdma_handle(context);
list_add(&continuous_buffer->continuous_buffer_list, &context->continuous_buffer_list);
buf_info.buffer_handle = continuous_buffer->handle;
buf_info.dma_address = continuous_buffer->dma_address;
if (copy_to_user((void __user*)arg, &buf_info, sizeof(buf_info))) {
hailo_dev_err(controller->dev, "copy_to_user fail\n");
list_del(&continuous_buffer->continuous_buffer_list);
hailo_vdma_continuous_buffer_free(controller->dev, continuous_buffer);
kfree(continuous_buffer);
return -EFAULT;
}
return 0;
}
long hailo_vdma_continuous_buffer_free_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg)
{
struct hailo_free_continuous_buffer_params params;
struct hailo_vdma_continuous_buffer *continuous_buffer = NULL;
if (copy_from_user(&params, (void __user*)arg, sizeof(params))) {
hailo_dev_err(controller->dev, "copy from user fail\n");
return -EFAULT;
}
continuous_buffer = hailo_vdma_find_continuous_buffer(context, params.buffer_handle);
if (NULL == continuous_buffer) {
hailo_dev_warn(controller->dev, "vdma buffer handle %lx not found\n", params.buffer_handle);
return -EINVAL;
}
list_del(&continuous_buffer->continuous_buffer_list);
hailo_vdma_continuous_buffer_free(controller->dev, continuous_buffer);
kfree(continuous_buffer);
return 0;
}
long hailo_vdma_interrupts_read_timestamps_ioctl(struct hailo_vdma_controller *controller, unsigned long arg)
{
struct hailo_vdma_interrupts_read_timestamp_params *params = &controller->read_interrupt_timestamps_params;
struct hailo_vdma_engine *engine = NULL;
int err = -EINVAL;
hailo_dev_dbg(controller->dev, "Start read interrupt timestamps ioctl\n");
if (copy_from_user(params, (void __user*)arg, sizeof(*params))) {
hailo_dev_err(controller->dev, "copy_from_user fail\n");
return -ENOMEM;
}
if (params->engine_index >= controller->vdma_engines_count) {
hailo_dev_err(controller->dev, "Invalid engine %u", params->engine_index);
return -EINVAL;
}
engine = &controller->vdma_engines[params->engine_index];
err = hailo_vdma_engine_read_timestamps(engine, params);
if (err < 0) {
hailo_dev_err(controller->dev, "Failed read engine interrupts for %u:%u",
params->engine_index, params->channel_index);
return err;
}
if (copy_to_user((void __user*)arg, params, sizeof(*params))) {
hailo_dev_err(controller->dev, "copy_to_user fail\n");
return -ENOMEM;
}
return 0;
}
long hailo_vdma_launch_transfer_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
unsigned long arg)
{
struct hailo_vdma_launch_transfer_params params;
struct hailo_vdma_engine *engine = NULL;
struct hailo_vdma_channel *channel = NULL;
struct hailo_descriptors_list_buffer *descriptors_buffer = NULL;
struct hailo_vdma_mapped_transfer_buffer mapped_transfer_buffers[ARRAY_SIZE(params.buffers)] = {0};
int ret = -EINVAL;
u8 i = 0;
if (copy_from_user(&params, (void __user*)arg, sizeof(params))) {
hailo_dev_err(controller->dev, "copy from user fail\n");
return -EFAULT;
}
if (params.engine_index >= controller->vdma_engines_count) {
hailo_dev_err(controller->dev, "Invalid engine %u", params.engine_index);
return -EINVAL;
}
engine = &controller->vdma_engines[params.engine_index];
if (params.channel_index >= ARRAY_SIZE(engine->channels)) {
hailo_dev_err(controller->dev, "Invalid channel %u", params.channel_index);
return -EINVAL;
}
channel = &engine->channels[params.channel_index];
if (params.buffers_count > ARRAY_SIZE(params.buffers)) {
hailo_dev_err(controller->dev, "too many buffers %u\n", params.buffers_count);
return -EINVAL;
}
descriptors_buffer = hailo_vdma_find_descriptors_buffer(context, params.desc_handle);
if (descriptors_buffer == NULL) {
hailo_dev_err(controller->dev, "invalid descriptors list handle\n");
return -EFAULT;
}
for (i = 0; i < params.buffers_count; i++) {
struct hailo_vdma_buffer *mapped_buffer =
hailo_vdma_find_mapped_user_buffer(context, params.buffers[i].mapped_buffer_handle);
if (mapped_buffer == NULL) {
hailo_dev_err(controller->dev, "invalid user buffer\n");
return -EFAULT;
}
if (params.buffers[i].size > mapped_buffer->size) {
hailo_dev_err(controller->dev, "Syncing size %u while buffer size is %u\n",
params.buffers[i].size, mapped_buffer->size);
return -EINVAL;
}
if (params.buffers[i].offset > mapped_buffer->size) {
hailo_dev_err(controller->dev, "Syncing offset %u while buffer size is %u\n",
params.buffers[i].offset, mapped_buffer->size);
return -EINVAL;
}
// Syncing the buffer to device change its ownership from host to the device.
// We sync on D2H as well if the user owns the buffer since the buffer might have been changed by
// the host between the time it was mapped and the current async transfer.
hailo_vdma_buffer_sync_cyclic(controller, mapped_buffer, HAILO_SYNC_FOR_DEVICE,
params.buffers[i].offset, params.buffers[i].size);
mapped_transfer_buffers[i].sg_table = &mapped_buffer->sg_table;
mapped_transfer_buffers[i].size = params.buffers[i].size;
mapped_transfer_buffers[i].offset = params.buffers[i].offset;
mapped_transfer_buffers[i].opaque = mapped_buffer;
}
ret = hailo_vdma_launch_transfer(
controller->hw,
channel,
&descriptors_buffer->desc_list,
params.starting_desc,
params.buffers_count,
mapped_transfer_buffers,
params.should_bind,
params.first_interrupts_domain,
params.last_interrupts_domain,
params.is_debug
);
if (ret < 0) {
params.launch_transfer_status = ret;
if (-ECONNRESET != ret) {
hailo_dev_err(controller->dev, "Failed launch transfer %d\n", ret);
}
// Still need to copy fail status back to userspace - success oriented
if (copy_to_user((void __user*)arg, &params, sizeof(params))) {
hailo_dev_err(controller->dev, "copy_to_user fail\n");
}
return ret;
}
params.descs_programed = ret;
params.launch_transfer_status = 0;
if (copy_to_user((void __user*)arg, &params, sizeof(params))) {
hailo_dev_err(controller->dev, "copy_to_user fail\n");
return -EFAULT;
}
return 0;
}

37
drivers/media/pci/hailo/vdma/ioctl.h
View File

@@ -1,37 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#ifndef _HAILO_VDMA_IOCTL_H_
#define _HAILO_VDMA_IOCTL_H_
#include "vdma/vdma.h"
long hailo_vdma_enable_channels_ioctl(struct hailo_vdma_controller *controller, unsigned long arg, struct hailo_vdma_file_context *context);
long hailo_vdma_disable_channels_ioctl(struct hailo_vdma_controller *controller, unsigned long arg, struct hailo_vdma_file_context *context);
long hailo_vdma_interrupts_wait_ioctl(struct hailo_vdma_controller *controller, unsigned long arg,
struct semaphore *mutex, bool *should_up_board_mutex);
long hailo_vdma_buffer_map_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg);
long hailo_vdma_buffer_unmap_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long handle);
long hailo_vdma_buffer_sync_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg);
long hailo_desc_list_create_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg);
long hailo_desc_list_release_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg);
long hailo_desc_list_program_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg);
long hailo_vdma_low_memory_buffer_alloc_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg);
long hailo_vdma_low_memory_buffer_free_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg);
long hailo_mark_as_in_use(struct hailo_vdma_controller *controller, unsigned long arg, struct file *filp);
long hailo_vdma_continuous_buffer_alloc_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg);
long hailo_vdma_continuous_buffer_free_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller, unsigned long arg);
long hailo_vdma_interrupts_read_timestamps_ioctl(struct hailo_vdma_controller *controller, unsigned long arg);
long hailo_vdma_launch_transfer_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
unsigned long arg);
#endif /* _HAILO_VDMA_IOCTL_H_ */

767
drivers/media/pci/hailo/vdma/memory.c
View File

@@ -1,767 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#define pr_fmt(fmt) "hailo: " fmt
#include "memory.h"
#include "utils.h"
#include "utils/compact.h"
#include <linux/highmem-internal.h>
#include <linux/slab.h>
#include <linux/scatterlist.h>
#include <linux/sched.h>
#include <linux/module.h>
#define SGL_MAX_SEGMENT_SIZE (0x10000)
// See linux/mm.h
#define MMIO_AND_NO_PAGES_VMA_MASK (VM_IO | VM_PFNMAP)
// The linux kernel names the dmabuf's vma vm_file field "dmabuf"
#define VMA_VM_FILE_DMABUF_NAME ("dmabuf")
static int map_mmio_address(uintptr_t user_address, u32 size, struct vm_area_struct *vma,
struct sg_table *sgt);
static int prepare_sg_table(struct sg_table *sg_table, uintptr_t user_address, u32 size,
struct hailo_vdma_low_memory_buffer *low_mem_driver_allocated_buffer);
static void clear_sg_table(struct sg_table *sgt);
#if LINUX_VERSION_CODE >= KERNEL_VERSION( 3, 3, 0 )
#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 13, 0)
#define DMA_NS_NAME DMA_BUF
#else
#define DMA_NS_NAME "DMA_BUF"
#endif // LINUX_VERSION_CODE < KERNEL_VERSION(6, 13, 0)
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)
// Import DMA_BUF namespace for needed kernels
MODULE_IMPORT_NS(DMA_NS_NAME);
#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0) */
static int hailo_map_dmabuf(struct device *dev, int dmabuf_fd, enum dma_data_direction direction, struct sg_table *sgt,
struct hailo_dmabuf_info *dmabuf_info)
{
int ret = -EINVAL;
struct dma_buf *dmabuf = NULL;
struct dma_buf_attachment *dmabuf_attachment = NULL;
struct sg_table *res_sgt = NULL;
dmabuf = dma_buf_get(dmabuf_fd);
if (IS_ERR(dmabuf)) {
dev_err(dev, "dma_buf_get failed, err=%ld\n", PTR_ERR(dmabuf));
ret = -EINVAL;
goto cleanup;
}
dmabuf_attachment = dma_buf_attach(dmabuf, dev);
if (IS_ERR(dmabuf_attachment)) {
dev_err(dev, "dma_buf_attach failed, err=%ld\n", PTR_ERR(dmabuf_attachment));
ret = -EINVAL;
goto l_buf_get;
}
res_sgt = dma_buf_map_attachment(dmabuf_attachment, direction);
if (IS_ERR(res_sgt)) {
dev_err(dev, "dma_buf_map_attachment failed, err=%ld\n", PTR_ERR(res_sgt));
goto l_buf_attach;
}
*sgt = *res_sgt;
dmabuf_info->dmabuf = dmabuf;
dmabuf_info->dmabuf_attachment = dmabuf_attachment;
dmabuf_info->dmabuf_sg_table = res_sgt;
return 0;
l_buf_attach:
dma_buf_detach(dmabuf, dmabuf_attachment);
l_buf_get:
dma_buf_put(dmabuf);
cleanup:
return ret;
}
static void hailo_unmap_dmabuf(struct hailo_vdma_buffer *vdma_buffer)
{
dma_buf_unmap_attachment(vdma_buffer->dmabuf_info.dmabuf_attachment, vdma_buffer->dmabuf_info.dmabuf_sg_table, vdma_buffer->data_direction);
dma_buf_detach(vdma_buffer->dmabuf_info.dmabuf, vdma_buffer->dmabuf_info.dmabuf_attachment);
dma_buf_put(vdma_buffer->dmabuf_info.dmabuf);
}
#else /* LINUX_VERSION_CODE >= KERNEL_VERSION( 3, 3, 0 ) */
static int hailo_map_dmabuf(struct device *dev, int dmabuf_fd, enum dma_data_direction direction, struct sg_table *sgt,
struct hailo_dmabuf_info *dmabuf_info)
{
(void) dmabuf_fd;
(void) direction;
(void) sgt;
(void) mapped_buffer;
dev_err(dev, "dmabuf not supported in kernel versions lower than 3.3.0\n");
return -EINVAL;
}
static void hailo_unmap_dmabuf(struct hailo_vdma_buffer *vdma_buffer)
{
dev_err(vdma_buffer->device, "dmabuf not supported in kernel versions lower than 3.3.0\n");
return -EINVAL;
}
#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION( 3, 3, 0 ) */
// Function that checks if the vma is backed by a mapped dmabuf
static bool is_dmabuf_vma(struct vm_area_struct *vma)
{
return (vma && vma->vm_file && (0 == strcmp(vma->vm_file->f_path.dentry->d_name.name, VMA_VM_FILE_DMABUF_NAME)));
}
static int create_fd_from_vma(struct device *dev, struct vm_area_struct *vma) {
struct file *file = NULL;
int fd = 0;
if (!vma || !vma->vm_file) {
dev_err(dev, "Invalid VMA or no associated file.\n");
return -EINVAL;
}
file = vma->vm_file;
// This functions increments the ref count of the file
get_file(file);
// 0 for default flags
fd = get_unused_fd_flags(0);
if (fd < 0) {
dev_err(dev, "Failed to get unused file descriptor.\n");
fput(file);
return fd;
}
// Install the file into the file descriptor table
fd_install(fd, file);
return fd;
}
struct hailo_vdma_buffer *hailo_vdma_buffer_map(struct device *dev,
uintptr_t user_address, size_t size, enum dma_data_direction direction,
enum hailo_dma_buffer_type buffer_type, struct hailo_vdma_low_memory_buffer *low_mem_driver_allocated_buffer)
{
int ret = -EINVAL;
struct hailo_vdma_buffer *mapped_buffer = NULL;
struct sg_table sgt = {0};
struct vm_area_struct *vma = NULL;
bool is_mmio = false;
struct hailo_dmabuf_info dmabuf_info = {0};
bool created_dmabuf_fd_from_vma = false;
mapped_buffer = kzalloc(sizeof(*mapped_buffer), GFP_KERNEL);
if (NULL == mapped_buffer) {
dev_err(dev, "memory alloc failed\n");
ret = -ENOMEM;
goto cleanup;
}
if (HAILO_DMA_DMABUF_BUFFER != buffer_type) {
mmap_read_lock(current->mm);
vma = find_vma(current->mm, user_address);
mmap_read_unlock(current->mm);
if (IS_ENABLED(HAILO_SUPPORT_MMIO_DMA_MAPPING)) {
if (NULL == vma) {
dev_err(dev, "no vma for virt_addr/size = 0x%08lx/0x%08zx\n", user_address, size);
ret = -EFAULT;
goto cleanup;
}
}
if (is_dmabuf_vma(vma)) {
dev_dbg(dev, "Given vma is backed by dmabuf - creating fd and mapping as dmabuf\n");
buffer_type = HAILO_DMA_DMABUF_BUFFER;
ret = create_fd_from_vma(dev, vma);
if (ret < 0) {
dev_err(dev, "Failed creating fd from vma in given dmabuf\n");
goto cleanup;
}
// Override user address with fd to the dmabuf - like normal dmabuf flow
user_address = ret;
created_dmabuf_fd_from_vma = true;
}
}
// TODO: is MMIO DMA MAPPINGS STILL needed after dmabuf
if (IS_ENABLED(HAILO_SUPPORT_MMIO_DMA_MAPPING) &&
(MMIO_AND_NO_PAGES_VMA_MASK == (vma->vm_flags & MMIO_AND_NO_PAGES_VMA_MASK)) &&
(HAILO_DMA_DMABUF_BUFFER != buffer_type)) {
// user_address represents memory mapped I/O and isn't backed by 'struct page' (only by pure pfn)
if (NULL != low_mem_driver_allocated_buffer) {
// low_mem_driver_allocated_buffer are backed by regular 'struct page' addresses, just in low memory
dev_err(dev, "low_mem_driver_allocated_buffer shouldn't be provided with an mmio address\n");
ret = -EINVAL;
goto free_buffer_struct;
}
ret = map_mmio_address(user_address, size, vma, &sgt);
if (ret < 0) {
dev_err(dev, "failed to map mmio address %d\n", ret);
goto free_buffer_struct;
}
is_mmio = true;
} else if (HAILO_DMA_DMABUF_BUFFER == buffer_type) {
// Content user_address in case of dmabuf is fd - for now
ret = hailo_map_dmabuf(dev, user_address, direction, &sgt, &dmabuf_info);
if (ret < 0) {
dev_err(dev, "Failed mapping dmabuf\n");
goto cleanup;
}
// If created dmabuf fd from vma need to decrement refcount and release fd
if (created_dmabuf_fd_from_vma) {
fput(vma->vm_file);
put_unused_fd(user_address);
}
} else {
// user_address is a standard 'struct page' backed memory address
ret = prepare_sg_table(&sgt, user_address, size, low_mem_driver_allocated_buffer);
if (ret < 0) {
dev_err(dev, "failed to set sg list for user buffer %d\n", ret);
goto free_buffer_struct;
}
sgt.nents = dma_map_sg(dev, sgt.sgl, sgt.orig_nents, direction);
if (0 == sgt.nents) {
dev_err(dev, "failed to map sg list for user buffer\n");
ret = -ENXIO;
goto clear_sg_table;
}
}
kref_init(&mapped_buffer->kref);
mapped_buffer->device = dev;
mapped_buffer->user_address = user_address;
mapped_buffer->size = size;
mapped_buffer->data_direction = direction;
mapped_buffer->sg_table = sgt;
mapped_buffer->is_mmio = is_mmio;
mapped_buffer->dmabuf_info = dmabuf_info;
return mapped_buffer;
clear_sg_table:
clear_sg_table(&sgt);
free_buffer_struct:
kfree(mapped_buffer);
cleanup:
return ERR_PTR(ret);
}
static void unmap_buffer(struct kref *kref)
{
struct hailo_vdma_buffer *buf = container_of(kref, struct hailo_vdma_buffer, kref);
// If dmabuf - unmap and detatch dmabuf
if (NULL != buf->dmabuf_info.dmabuf) {
hailo_unmap_dmabuf(buf);
} else {
if (!buf->is_mmio) {
dma_unmap_sg(buf->device, buf->sg_table.sgl, buf->sg_table.orig_nents, buf->data_direction);
}
clear_sg_table(&buf->sg_table);
}
kfree(buf);
}
void hailo_vdma_buffer_get(struct hailo_vdma_buffer *buf)
{
kref_get(&buf->kref);
}
void hailo_vdma_buffer_put(struct hailo_vdma_buffer *buf)
{
kref_put(&buf->kref, unmap_buffer);
}
static void vdma_sync_entire_buffer(struct hailo_vdma_controller *controller,
struct hailo_vdma_buffer *mapped_buffer, enum hailo_vdma_buffer_sync_type sync_type)
{
if (sync_type == HAILO_SYNC_FOR_CPU) {
dma_sync_sg_for_cpu(controller->dev, mapped_buffer->sg_table.sgl, mapped_buffer->sg_table.nents,
mapped_buffer->data_direction);
} else {
dma_sync_sg_for_device(controller->dev, mapped_buffer->sg_table.sgl, mapped_buffer->sg_table.nents,
mapped_buffer->data_direction);
}
}
typedef void (*dma_sync_single_callback)(struct device *, dma_addr_t, size_t, enum dma_data_direction);
// Map sync_info->count bytes starting at sync_info->offset
static void vdma_sync_buffer_interval(struct hailo_vdma_controller *controller,
struct hailo_vdma_buffer *mapped_buffer,
size_t offset, size_t size, enum hailo_vdma_buffer_sync_type sync_type)
{
size_t sync_start_offset = offset;
size_t sync_end_offset = offset + size;
dma_sync_single_callback dma_sync_single = (sync_type == HAILO_SYNC_FOR_CPU) ?
dma_sync_single_for_cpu :
dma_sync_single_for_device;
struct scatterlist* sg_entry = NULL;
size_t current_iter_offset = 0;
int i = 0;
for_each_sg(mapped_buffer->sg_table.sgl, sg_entry, mapped_buffer->sg_table.nents, i) {
// Check if the intervals: [current_iter_offset, sg_dma_len(sg_entry)] and [sync_start_offset, sync_end_offset]
// have any intersection. If offset isn't at the start of a sg_entry, we still want to sync it.
if (max(sync_start_offset, current_iter_offset) <= min(sync_end_offset, current_iter_offset + sg_dma_len(sg_entry))) {
dma_sync_single(controller->dev, sg_dma_address(sg_entry), sg_dma_len(sg_entry),
mapped_buffer->data_direction);
}
current_iter_offset += sg_dma_len(sg_entry);
}
}
void hailo_vdma_buffer_sync(struct hailo_vdma_controller *controller,
struct hailo_vdma_buffer *mapped_buffer, enum hailo_vdma_buffer_sync_type sync_type,
size_t offset, size_t size)
{
if ((IS_ENABLED(HAILO_SUPPORT_MMIO_DMA_MAPPING) && mapped_buffer->is_mmio) ||
(NULL != mapped_buffer->dmabuf_info.dmabuf)) {
// MMIO buffers and dmabufs don't need to be sync'd
return;
}
if ((offset == 0) && (size == mapped_buffer->size)) {
vdma_sync_entire_buffer(controller, mapped_buffer, sync_type);
} else {
vdma_sync_buffer_interval(controller, mapped_buffer, offset, size, sync_type);
}
}
// Similar to vdma_buffer_sync, allow circular sync of the buffer.
void hailo_vdma_buffer_sync_cyclic(struct hailo_vdma_controller *controller,
struct hailo_vdma_buffer *mapped_buffer, enum hailo_vdma_buffer_sync_type sync_type,
size_t offset, size_t size)
{
size_t size_to_end = min(size, mapped_buffer->size - offset);
hailo_vdma_buffer_sync(controller, mapped_buffer, sync_type, offset, size_to_end);
if (size_to_end < size) {
hailo_vdma_buffer_sync(controller, mapped_buffer, sync_type, 0, size - size_to_end);
}
}
struct hailo_vdma_buffer* hailo_vdma_find_mapped_user_buffer(struct hailo_vdma_file_context *context,
size_t buffer_handle)
{
struct hailo_vdma_buffer *cur = NULL;
list_for_each_entry(cur, &context->mapped_user_buffer_list, mapped_user_buffer_list) {
if (cur->handle == buffer_handle) {
return cur;
}
}
return NULL;
}
void hailo_vdma_clear_mapped_user_buffer_list(struct hailo_vdma_file_context *context,
struct hailo_vdma_controller *controller)
{
struct hailo_vdma_buffer *cur = NULL, *next = NULL;
list_for_each_entry_safe(cur, next, &context->mapped_user_buffer_list, mapped_user_buffer_list) {
list_del(&cur->mapped_user_buffer_list);
hailo_vdma_buffer_put(cur);
}
}
int hailo_desc_list_create(struct device *dev, u32 descriptors_count, u16 desc_page_size,
uintptr_t desc_handle, bool is_circular, struct hailo_descriptors_list_buffer *descriptors)
{
size_t buffer_size = 0;
const u64 align = VDMA_DESCRIPTOR_LIST_ALIGN; //First addr must be aligned on 64 KB (from the VDMA registers documentation)
if (MAX_POWER_OF_2_VALUE < descriptors_count) {
dev_err(dev, "Invalid descriptors count %u\n", descriptors_count);
return -EINVAL;
}
buffer_size = descriptors_count * sizeof(struct hailo_vdma_descriptor);
buffer_size = ALIGN(buffer_size, align);
descriptors->kernel_address = dma_alloc_coherent(dev, buffer_size,
&descriptors->dma_address, GFP_KERNEL | __GFP_ZERO);
if (descriptors->kernel_address == NULL) {
dev_err(dev, "Failed to allocate descriptors list, desc_count 0x%x, buffer_size 0x%zx, This failure means there is not a sufficient amount of CMA memory "
"(contiguous physical memory), This usually is caused by lack of general system memory. Please check you have sufficient memory.\n",
descriptors_count, buffer_size);
return -ENOBUFS;
}
descriptors->buffer_size = buffer_size;
descriptors->handle = desc_handle;
descriptors->desc_list.desc_list = descriptors->kernel_address;
descriptors->desc_list.desc_count = descriptors_count;
// No need to check the return value of get_nearest_powerof_2 because we already checked the input
descriptors->desc_list.desc_count_mask = is_circular ? (descriptors_count - 1) : (get_nearest_powerof_2(descriptors_count) - 1);
descriptors->desc_list.desc_page_size = desc_page_size;
descriptors->desc_list.is_circular = is_circular;
return 0;
}
void hailo_desc_list_release(struct device *dev, struct hailo_descriptors_list_buffer *descriptors)
{
dma_free_coherent(dev, descriptors->buffer_size, descriptors->kernel_address, descriptors->dma_address);
}
struct hailo_descriptors_list_buffer* hailo_vdma_find_descriptors_buffer(struct hailo_vdma_file_context *context,
uintptr_t desc_handle)
{
struct hailo_descriptors_list_buffer *cur = NULL;
list_for_each_entry(cur, &context->descriptors_buffer_list, descriptors_buffer_list) {
if (cur->handle == desc_handle) {
return cur;
}
}
return NULL;
}
void hailo_vdma_clear_descriptors_buffer_list(struct hailo_vdma_file_context *context,
struct hailo_vdma_controller *controller)
{
struct hailo_descriptors_list_buffer *cur = NULL, *next = NULL;
list_for_each_entry_safe(cur, next, &context->descriptors_buffer_list, descriptors_buffer_list) {
list_del(&cur->descriptors_buffer_list);
hailo_desc_list_release(controller->dev, cur);
kfree(cur);
}
}
int hailo_vdma_low_memory_buffer_alloc(size_t size, struct hailo_vdma_low_memory_buffer *low_memory_buffer)
{
int ret = -EINVAL;
void *kernel_address = NULL;
size_t pages_count = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
size_t num_allocated = 0, i = 0;
void **pages = NULL;
pages = kcalloc(pages_count, sizeof(*pages), GFP_KERNEL);
if (NULL == pages) {
pr_err("Failed to allocate pages for buffer (size %zu)\n", size);
ret = -ENOMEM;
goto cleanup;
}
for (num_allocated = 0; num_allocated < pages_count; num_allocated++) {
// __GFP_DMA32 flag is used to limit system memory allocations to the lowest 4 GB of physical memory in order to guarantee DMA
// Operations will not have to use bounce buffers on certain architectures (e.g 32-bit DMA enabled architectures)
kernel_address = (void*)__get_free_page(__GFP_DMA32);
if (NULL == kernel_address) {
pr_err("Failed to allocate %zu coherent bytes\n", (size_t)PAGE_SIZE);
ret = -ENOMEM;
goto cleanup;
}
pages[num_allocated] = kernel_address;
}
low_memory_buffer->pages_count = pages_count;
low_memory_buffer->pages_address = pages;
return 0;
cleanup:
if (NULL != pages) {
for (i = 0; i < num_allocated; i++) {
free_page((long unsigned)pages[i]);
}
kfree(pages);
}
return ret;
}
void hailo_vdma_low_memory_buffer_free(struct hailo_vdma_low_memory_buffer *low_memory_buffer)
{
size_t i = 0;
if (NULL == low_memory_buffer) {
return;
}
for (i = 0; i < low_memory_buffer->pages_count; i++) {
free_page((long unsigned)low_memory_buffer->pages_address[i]);
}
kfree(low_memory_buffer->pages_address);
}
struct hailo_vdma_low_memory_buffer* hailo_vdma_find_low_memory_buffer(struct hailo_vdma_file_context *context,
uintptr_t buf_handle)
{
struct hailo_vdma_low_memory_buffer *cur = NULL;
list_for_each_entry(cur, &context->vdma_low_memory_buffer_list, vdma_low_memory_buffer_list) {
if (cur->handle == buf_handle) {
return cur;
}
}
return NULL;
}
void hailo_vdma_clear_low_memory_buffer_list(struct hailo_vdma_file_context *context)
{
struct hailo_vdma_low_memory_buffer *cur = NULL, *next = NULL;
list_for_each_entry_safe(cur, next, &context->vdma_low_memory_buffer_list, vdma_low_memory_buffer_list) {
list_del(&cur->vdma_low_memory_buffer_list);
hailo_vdma_low_memory_buffer_free(cur);
kfree(cur);
}
}
int hailo_vdma_continuous_buffer_alloc(struct device *dev, size_t size,
struct hailo_vdma_continuous_buffer *continuous_buffer)
{
dma_addr_t dma_address = 0;
void *kernel_address = NULL;
kernel_address = dma_alloc_coherent(dev, size, &dma_address, GFP_KERNEL);
if (NULL == kernel_address) {
dev_warn(dev, "Failed to allocate continuous buffer, size 0x%zx. This failure means there is not a sufficient amount of CMA memory "
"(contiguous physical memory), This usually is caused by lack of general system memory. Please check you have sufficent memory.\n", size);
return -ENOBUFS;
}
continuous_buffer->kernel_address = kernel_address;
continuous_buffer->dma_address = dma_address;
continuous_buffer->size = size;
return 0;
}
void hailo_vdma_continuous_buffer_free(struct device *dev,
struct hailo_vdma_continuous_buffer *continuous_buffer)
{
dma_free_coherent(dev, continuous_buffer->size, continuous_buffer->kernel_address,
continuous_buffer->dma_address);
}
struct hailo_vdma_continuous_buffer* hailo_vdma_find_continuous_buffer(struct hailo_vdma_file_context *context,
uintptr_t buf_handle)
{
struct hailo_vdma_continuous_buffer *cur = NULL;
list_for_each_entry(cur, &context->continuous_buffer_list, continuous_buffer_list) {
if (cur->handle == buf_handle) {
return cur;
}
}
return NULL;
}
void hailo_vdma_clear_continuous_buffer_list(struct hailo_vdma_file_context *context,
struct hailo_vdma_controller *controller)
{
struct hailo_vdma_continuous_buffer *cur = NULL, *next = NULL;
list_for_each_entry_safe(cur, next, &context->continuous_buffer_list, continuous_buffer_list) {
list_del(&cur->continuous_buffer_list);
hailo_vdma_continuous_buffer_free(controller->dev, cur);
kfree(cur);
}
}
/**
* follow_pfn - look up PFN at a user virtual address
* @vma: memory mapping
* @address: user virtual address
* @pfn: location to store found PFN
*
* Only IO mappings and raw PFN mappings are allowed.
*
* This function does not allow the caller to read the permissions
* of the PTE. Do not use it.
*
* Return: zero and the pfn at @pfn on success, -ve otherwise.
*/
#if defined(HAILO_SUPPORT_MMIO_DMA_MAPPING)
static int follow_pfn(struct vm_area_struct *vma, unsigned long address,
unsigned long *pfn)
{
int ret = -EINVAL;
spinlock_t *ptl;
pte_t *ptep;
if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
return ret;
ret = follow_pte(vma, address, &ptep, &ptl);
if (ret)
return ret;
*pfn = pte_pfn(ptep_get(ptep));
pte_unmap_unlock(ptep, ptl);
return 0;
}
#endif
// Assumes the provided user_address belongs to the vma and that MMIO_AND_NO_PAGES_VMA_MASK bits are set under
// vma->vm_flags. This is validated in hailo_vdma_buffer_map, and won't be checked here
#if defined(HAILO_SUPPORT_MMIO_DMA_MAPPING)
static int map_mmio_address(uintptr_t user_address, u32 size, struct vm_area_struct *vma,
struct sg_table *sgt)
{
int ret = -EINVAL;
unsigned long i = 0;
unsigned long pfn = 0;
unsigned long next_pfn = 0;
phys_addr_t phys_addr = 0;
dma_addr_t mmio_dma_address = 0;
const uintptr_t virt_addr = user_address;
const u32 vma_size = vma->vm_end - vma->vm_start + 1;
const uintptr_t num_pages = PFN_UP(virt_addr + size) - PFN_DOWN(virt_addr);
// Check that the vma that was marked as MMIO_AND_NO_PAGES_VMA_MASK is big enough
if (vma_size < size) {
pr_err("vma (%u bytes) smaller than provided buffer (%u bytes)\n", vma_size, size);
return -EINVAL;
}
// Get the physical address of user_address
ret = follow_pfn(vma, virt_addr, &pfn);
if (ret) {
pr_err("follow_pfn failed with %d\n", ret);
return ret;
}
phys_addr = __pfn_to_phys(pfn) + offset_in_page(virt_addr);
// Make sure the physical memory is contiguous
for (i = 1; i < num_pages; ++i) {
ret = follow_pfn(vma, virt_addr + (i << PAGE_SHIFT), &next_pfn);
if (ret < 0) {
pr_err("follow_pfn failed with %d\n", ret);
return ret;
}
if (next_pfn != pfn + 1) {
pr_err("non-contiguous physical memory\n");
return -EFAULT;
}
pfn = next_pfn;
}
// phys_addr to dma
// TODO: need dma_map_resource here? doesn't work currently (we get dma_mapping_error on the returned dma addr)
// (HRT-12521)
mmio_dma_address = (dma_addr_t)phys_addr;
// Create a page-less scatterlist.
ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
if (ret < 0) {
return ret;
}
sg_assign_page(sgt->sgl, NULL);
sg_dma_address(sgt->sgl) = mmio_dma_address;
sg_dma_len(sgt->sgl) = size;
return 0;
}
#else /* defined(HAILO_SUPPORT_MMIO_DMA_MAPPING) */
static int map_mmio_address(uintptr_t user_address, u32 size, struct vm_area_struct *vma,
struct sg_table *sgt)
{
(void) user_address;
(void) size;
(void) vma;
(void) sgt;
pr_err("MMIO DMA MAPPINGS are not supported in this kernel version\n");
return -EINVAL;
}
#endif /* defined(HAILO_SUPPORT_MMIO_DMA_MAPPING) */
static int prepare_sg_table(struct sg_table *sg_table, uintptr_t user_address, u32 size,
struct hailo_vdma_low_memory_buffer *low_mem_driver_allocated_buffer)
{
int ret = -EINVAL;
int pinned_pages = 0;
size_t npages = 0;
struct page **pages = NULL;
int i = 0;
struct scatterlist *sg_alloc_res = NULL;
npages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
pages = kvmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
if (!pages) {
return -ENOMEM;
}
// Check whether mapping user allocated buffer or driver allocated low memory buffer
if (NULL == low_mem_driver_allocated_buffer) {
mmap_read_lock(current->mm);
pinned_pages = get_user_pages_compact(user_address, npages, FOLL_WRITE | FOLL_FORCE, pages);
mmap_read_unlock(current->mm);
if (pinned_pages < 0) {
pr_err("get_user_pages failed with %d\n", pinned_pages);
ret = pinned_pages;
goto exit;
} else if (pinned_pages != npages) {
pr_err("Pinned %d out of %zu\n", pinned_pages, npages);
ret = -EINVAL;
goto release_pages;
}
} else {
// Check to make sure in case user provides wrong buffer
if (npages != low_mem_driver_allocated_buffer->pages_count) {
pr_err("Received wrong amount of pages %zu to map expected %zu\n",
npages, low_mem_driver_allocated_buffer->pages_count);
ret = -EINVAL;
goto exit;
}
for (i = 0; i < npages; i++) {
pages[i] = virt_to_page(low_mem_driver_allocated_buffer->pages_address[i]);
get_page(pages[i]);
}
}
sg_alloc_res = sg_alloc_table_from_pages_segment_compat(sg_table, pages, npages,
0, size, SGL_MAX_SEGMENT_SIZE, NULL, 0, GFP_KERNEL);
if (IS_ERR(sg_alloc_res)) {
ret = PTR_ERR(sg_alloc_res);
pr_err("sg table alloc failed (err %d)..\n", ret);
goto release_pages;
}
ret = 0;
goto exit;
release_pages:
for (i = 0; i < pinned_pages; i++) {
if (!PageReserved(pages[i])) {
SetPageDirty(pages[i]);
}
put_page(pages[i]);
}
exit:
kvfree(pages);
return ret;
}
static void clear_sg_table(struct sg_table *sgt)
{
struct sg_page_iter iter;
struct page *page = NULL;
for_each_sg_page(sgt->sgl, &iter, sgt->orig_nents, 0) {
page = sg_page_iter_page(&iter);
if (page) {
if (!PageReserved(page)) {
SetPageDirty(page);
}
put_page(page);
}
}
sg_free_table(sgt);
}

56
drivers/media/pci/hailo/vdma/memory.h
View File

@@ -1,56 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
/**
* vDMA memory utility (including allocation and mappings)
*/
#ifndef _HAILO_VDMA_MEMORY_H_
#define _HAILO_VDMA_MEMORY_H_
#include "vdma/vdma.h"
#define SGL_MAX_SEGMENT_SIZE (0x10000)
struct hailo_vdma_buffer *hailo_vdma_buffer_map(struct device *dev, uintptr_t user_address, size_t size,
enum dma_data_direction direction, enum hailo_dma_buffer_type buffer_type,
struct hailo_vdma_low_memory_buffer *low_mem_driver_allocated_buffer);
void hailo_vdma_buffer_get(struct hailo_vdma_buffer *buf);
void hailo_vdma_buffer_put(struct hailo_vdma_buffer *buf);
void hailo_vdma_buffer_sync(struct hailo_vdma_controller *controller,
struct hailo_vdma_buffer *mapped_buffer, enum hailo_vdma_buffer_sync_type sync_type,
size_t offset, size_t size);
void hailo_vdma_buffer_sync_cyclic(struct hailo_vdma_controller *controller,
struct hailo_vdma_buffer *mapped_buffer, enum hailo_vdma_buffer_sync_type sync_type,
size_t offset, size_t size);
struct hailo_vdma_buffer* hailo_vdma_find_mapped_user_buffer(struct hailo_vdma_file_context *context,
size_t buffer_handle);
void hailo_vdma_clear_mapped_user_buffer_list(struct hailo_vdma_file_context *context,
struct hailo_vdma_controller *controller);
int hailo_desc_list_create(struct device *dev, u32 descriptors_count, u16 desc_page_size,
uintptr_t desc_handle, bool is_circular, struct hailo_descriptors_list_buffer *descriptors);
void hailo_desc_list_release(struct device *dev, struct hailo_descriptors_list_buffer *descriptors);
struct hailo_descriptors_list_buffer* hailo_vdma_find_descriptors_buffer(struct hailo_vdma_file_context *context,
uintptr_t desc_handle);
void hailo_vdma_clear_descriptors_buffer_list(struct hailo_vdma_file_context *context,
struct hailo_vdma_controller *controller);
int hailo_vdma_low_memory_buffer_alloc(size_t size, struct hailo_vdma_low_memory_buffer *low_memory_buffer);
void hailo_vdma_low_memory_buffer_free(struct hailo_vdma_low_memory_buffer *low_memory_buffer);
struct hailo_vdma_low_memory_buffer* hailo_vdma_find_low_memory_buffer(struct hailo_vdma_file_context *context,
uintptr_t buf_handle);
void hailo_vdma_clear_low_memory_buffer_list(struct hailo_vdma_file_context *context);
int hailo_vdma_continuous_buffer_alloc(struct device *dev, size_t size,
struct hailo_vdma_continuous_buffer *continuous_buffer);
void hailo_vdma_continuous_buffer_free(struct device *dev,
struct hailo_vdma_continuous_buffer *continuous_buffer);
struct hailo_vdma_continuous_buffer* hailo_vdma_find_continuous_buffer(struct hailo_vdma_file_context *context,
uintptr_t buf_handle);
void hailo_vdma_clear_continuous_buffer_list(struct hailo_vdma_file_context *context,
struct hailo_vdma_controller *controller);
#endif /* _HAILO_VDMA_MEMORY_H_ */

313
drivers/media/pci/hailo/vdma/vdma.c
View File

@@ -1,313 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
#define pr_fmt(fmt) "hailo: " fmt
#include "vdma.h"
#include "memory.h"
#include "ioctl.h"
#include "utils/logs.h"
#include <linux/sched.h>
#include <linux/version.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
#include <linux/dma-map-ops.h>
#else
#include <linux/dma-mapping.h>
#endif
static struct hailo_vdma_engine* init_vdma_engines(struct device *dev,
struct hailo_resource *channel_registers_per_engine, size_t engines_count, u32 src_channels_bitmask)
{
struct hailo_vdma_engine *engines = NULL;
u8 i = 0;
engines = devm_kmalloc_array(dev, engines_count, sizeof(*engines), GFP_KERNEL);
if (NULL == engines) {
dev_err(dev, "Failed allocating vdma engines\n");
return ERR_PTR(-ENOMEM);
}
for (i = 0; i < engines_count; i++) {
hailo_vdma_engine_init(&engines[i], i, &channel_registers_per_engine[i], src_channels_bitmask);
}
return engines;
}
static int hailo_set_dma_mask(struct device *dev)
{
int err = -EINVAL;
/* Check and configure DMA length */
if (!(err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))) {
dev_notice(dev, "Probing: Enabled 64 bit dma\n");
} else if (!(err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)))) {
dev_notice(dev, "Probing: Enabled 48 bit dma\n");
} else if (!(err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)))) {
dev_notice(dev, "Probing: Enabled 40 bit dma\n");
} else if (!(err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)))) {
dev_notice(dev, "Probing: Enabled 36 bit dma\n");
} else if (!(err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)))) {
dev_notice(dev, "Probing: Enabled 32 bit dma\n");
} else {
dev_err(dev, "Probing: Error enabling dma %d\n", err);
return err;
}
return 0;
}
int hailo_vdma_controller_init(struct hailo_vdma_controller *controller,
struct device *dev, struct hailo_vdma_hw *vdma_hw,
struct hailo_vdma_controller_ops *ops,
struct hailo_resource *channel_registers_per_engine, size_t engines_count)
{
int err = 0;
controller->hw = vdma_hw;
controller->ops = ops;
controller->dev = dev;
controller->vdma_engines_count = engines_count;
controller->vdma_engines = init_vdma_engines(dev, channel_registers_per_engine, engines_count,
vdma_hw->src_channels_bitmask);
if (IS_ERR(controller->vdma_engines)) {
dev_err(dev, "Failed initialized vdma engines\n");
return PTR_ERR(controller->vdma_engines);
}
controller->used_by_filp = NULL;
spin_lock_init(&controller->interrupts_lock);
init_waitqueue_head(&controller->interrupts_wq);
/* Check and configure DMA length */
err = hailo_set_dma_mask(dev);
if (0 > err) {
return err;
}
if (get_dma_ops(controller->dev)) {
hailo_dev_notice(controller->dev, "Probing: Using specialized dma_ops=%ps", get_dma_ops(controller->dev));
}
return 0;
}
void hailo_vdma_file_context_init(struct hailo_vdma_file_context *context)
{
atomic_set(&context->last_vdma_user_buffer_handle, 0);
INIT_LIST_HEAD(&context->mapped_user_buffer_list);
atomic_set(&context->last_vdma_handle, 0);
INIT_LIST_HEAD(&context->descriptors_buffer_list);
INIT_LIST_HEAD(&context->vdma_low_memory_buffer_list);
INIT_LIST_HEAD(&context->continuous_buffer_list);
BUILD_BUG_ON_MSG(MAX_VDMA_CHANNELS_PER_ENGINE > sizeof(context->enabled_channels_bitmap[0]) * BITS_IN_BYTE,
"Unexpected amount of VDMA channels per engine");
}
void hailo_vdma_update_interrupts_mask(struct hailo_vdma_controller *controller,
size_t engine_index)
{
struct hailo_vdma_engine *engine = &controller->vdma_engines[engine_index];
controller->ops->update_channel_interrupts(controller, engine_index, engine->enabled_channels);
}
void hailo_vdma_file_context_finalize(struct hailo_vdma_file_context *context,
struct hailo_vdma_controller *controller, struct file *filp)
{
size_t engine_index = 0;
struct hailo_vdma_engine *engine = NULL;
unsigned long irq_saved_flags = 0;
// In case of FLR, the vdma registers will be NULL
const bool is_device_up = (NULL != controller->dev);
for_each_vdma_engine(controller, engine, engine_index) {
if (context->enabled_channels_bitmap[engine_index]) {
hailo_dev_info(controller->dev, "Disabling channels for engine %zu, channels bitmap 0x%x\n", engine_index,
context->enabled_channels_bitmap[engine_index]);
hailo_vdma_engine_disable_channels(engine, context->enabled_channels_bitmap[engine_index]);
if (is_device_up) {
hailo_vdma_update_interrupts_mask(controller, engine_index);
}
spin_lock_irqsave(&controller->interrupts_lock, irq_saved_flags);
hailo_vdma_engine_clear_channel_interrupts(engine, context->enabled_channels_bitmap[engine_index]);
spin_unlock_irqrestore(&controller->interrupts_lock, irq_saved_flags);
}
}
hailo_vdma_clear_mapped_user_buffer_list(context, controller);
hailo_vdma_clear_descriptors_buffer_list(context, controller);
hailo_vdma_clear_low_memory_buffer_list(context);
hailo_vdma_clear_continuous_buffer_list(context, controller);
if (filp == controller->used_by_filp) {
controller->used_by_filp = NULL;
}
}
void hailo_vdma_wakeup_interrupts(struct hailo_vdma_controller *controller, struct hailo_vdma_engine *engine,
u32 channels_bitmap)
{
unsigned long irq_saved_flags = 0;
spin_lock_irqsave(&controller->interrupts_lock, irq_saved_flags);
hailo_vdma_engine_set_channel_interrupts(engine, channels_bitmap);
spin_unlock_irqrestore(&controller->interrupts_lock, irq_saved_flags);
wake_up_interruptible_all(&controller->interrupts_wq);
}
void hailo_vdma_irq_handler(struct hailo_vdma_controller *controller,
size_t engine_index, u32 channels_bitmap)
{
struct hailo_vdma_engine *engine = NULL;
BUG_ON(engine_index >= controller->vdma_engines_count);
engine = &controller->vdma_engines[engine_index];
hailo_vdma_engine_push_timestamps(engine, channels_bitmap);
hailo_vdma_wakeup_interrupts(controller, engine, channels_bitmap);
}
long hailo_vdma_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
unsigned int cmd, unsigned long arg, struct file *filp, struct semaphore *mutex, bool *should_up_board_mutex)
{
switch (cmd) {
case HAILO_VDMA_ENABLE_CHANNELS:
return hailo_vdma_enable_channels_ioctl(controller, arg, context);
case HAILO_VDMA_DISABLE_CHANNELS:
return hailo_vdma_disable_channels_ioctl(controller, arg, context);
case HAILO_VDMA_INTERRUPTS_WAIT:
return hailo_vdma_interrupts_wait_ioctl(controller, arg, mutex, should_up_board_mutex);
case HAILO_VDMA_INTERRUPTS_READ_TIMESTAMPS:
return hailo_vdma_interrupts_read_timestamps_ioctl(controller, arg);
case HAILO_VDMA_BUFFER_MAP:
return hailo_vdma_buffer_map_ioctl(context, controller, arg);
case HAILO_VDMA_BUFFER_UNMAP:
return hailo_vdma_buffer_unmap_ioctl(context, controller, arg);
case HAILO_VDMA_BUFFER_SYNC:
return hailo_vdma_buffer_sync_ioctl(context, controller, arg);
case HAILO_DESC_LIST_CREATE:
return hailo_desc_list_create_ioctl(context, controller, arg);
case HAILO_DESC_LIST_RELEASE:
return hailo_desc_list_release_ioctl(context, controller, arg);
case HAILO_DESC_LIST_PROGRAM:
return hailo_desc_list_program_ioctl(context, controller, arg);
case HAILO_VDMA_LOW_MEMORY_BUFFER_ALLOC:
return hailo_vdma_low_memory_buffer_alloc_ioctl(context, controller, arg);
case HAILO_VDMA_LOW_MEMORY_BUFFER_FREE:
return hailo_vdma_low_memory_buffer_free_ioctl(context, controller, arg);
case HAILO_MARK_AS_IN_USE:
return hailo_mark_as_in_use(controller, arg, filp);
case HAILO_VDMA_CONTINUOUS_BUFFER_ALLOC:
return hailo_vdma_continuous_buffer_alloc_ioctl(context, controller, arg);
case HAILO_VDMA_CONTINUOUS_BUFFER_FREE:
return hailo_vdma_continuous_buffer_free_ioctl(context, controller, arg);
case HAILO_VDMA_LAUNCH_TRANSFER:
return hailo_vdma_launch_transfer_ioctl(context, controller, arg);
default:
hailo_dev_err(controller->dev, "Invalid vDMA ioctl code 0x%x (nr: %d)\n", cmd, _IOC_NR(cmd));
return -ENOTTY;
}
}
static int low_memory_buffer_mmap(struct hailo_vdma_controller *controller,
struct hailo_vdma_low_memory_buffer *vdma_buffer, struct vm_area_struct *vma)
{
int err = 0;
size_t i = 0;
unsigned long vsize = vma->vm_end - vma->vm_start;
unsigned long orig_vm_start = vma->vm_start;
unsigned long orig_vm_end = vma->vm_end;
unsigned long page_fn = 0;
if (vsize != vdma_buffer->pages_count * PAGE_SIZE) {
hailo_dev_err(controller->dev, "mmap size should be %lu (given %lu)\n",
vdma_buffer->pages_count * PAGE_SIZE, vsize);
return -EINVAL;
}
for (i = 0 ; i < vdma_buffer->pages_count ; i++) {
if (i > 0) {
vma->vm_start = vma->vm_end;
}
vma->vm_end = vma->vm_start + PAGE_SIZE;
page_fn = virt_to_phys(vdma_buffer->pages_address[i]) >> PAGE_SHIFT ;
err = remap_pfn_range(vma, vma->vm_start, page_fn, PAGE_SIZE, vma->vm_page_prot);
if (err != 0) {
hailo_dev_err(controller->dev, " fops_mmap failed mapping kernel page %d\n", err);
return err;
}
}
vma->vm_start = orig_vm_start;
vma->vm_end = orig_vm_end;
return 0;
}
static int continuous_buffer_mmap(struct hailo_vdma_controller *controller,
struct hailo_vdma_continuous_buffer *buffer, struct vm_area_struct *vma)
{
int err = 0;
const unsigned long vsize = vma->vm_end - vma->vm_start;
if (vsize > buffer->size) {
hailo_dev_err(controller->dev, "mmap size should be less than %zu (given %lu)\n",
buffer->size, vsize);
return -EINVAL;
}
err = dma_mmap_coherent(controller->dev, vma, buffer->kernel_address,
buffer->dma_address, vsize);
if (err < 0) {
hailo_dev_err(controller->dev, " vdma_mmap failed dma_mmap_coherent %d\n", err);
return err;
}
return 0;
}
int hailo_vdma_mmap(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
struct vm_area_struct *vma, uintptr_t vdma_handle)
{
struct hailo_vdma_low_memory_buffer *low_memory_buffer = NULL;
struct hailo_vdma_continuous_buffer *continuous_buffer = NULL;
hailo_dev_info(controller->dev, "Map vdma_handle %llu\n", (u64)vdma_handle);
if (NULL != (low_memory_buffer = hailo_vdma_find_low_memory_buffer(context, vdma_handle))) {
return low_memory_buffer_mmap(controller, low_memory_buffer, vma);
}
else if (NULL != (continuous_buffer = hailo_vdma_find_continuous_buffer(context, vdma_handle))) {
return continuous_buffer_mmap(controller, continuous_buffer, vma);
}
else {
hailo_dev_err(controller->dev, "Can't mmap vdma handle: %llu (not existing)\n", (u64)vdma_handle);
return -EINVAL;
}
}
enum dma_data_direction get_dma_direction(enum hailo_dma_data_direction hailo_direction)
{
switch (hailo_direction) {
case HAILO_DMA_BIDIRECTIONAL:
return DMA_BIDIRECTIONAL;
case HAILO_DMA_TO_DEVICE:
return DMA_TO_DEVICE;
case HAILO_DMA_FROM_DEVICE:
return DMA_FROM_DEVICE;
default:
pr_err("Invalid hailo direction %d\n", hailo_direction);
return DMA_NONE;
}
}

164
drivers/media/pci/hailo/vdma/vdma.h
View File

@@ -1,164 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/**
* Copyright (c) 2019-2024 Hailo Technologies Ltd. All rights reserved.
**/
/**
* Hailo vdma engine definitions
*/
#ifndef _HAILO_VDMA_VDMA_H_
#define _HAILO_VDMA_VDMA_H_
#include "hailo_ioctl_common.h"
#include "hailo_resource.h"
#include "vdma_common.h"
#include <linux/dma-mapping.h>
#include <linux/types.h>
#include <linux/semaphore.h>
#include <linux/dma-buf.h>
#include <linux/version.h>
#define VDMA_CHANNEL_CONTROL_REG_OFFSET(channel_index, direction) (((direction) == DMA_TO_DEVICE) ? \
(((channel_index) << 5) + 0x0) : (((channel_index) << 5) + 0x10))
#define VDMA_CHANNEL_CONTROL_REG_ADDRESS(vdma_registers, channel_index, direction) \
((u8*)((vdma_registers)->address) + VDMA_CHANNEL_CONTROL_REG_OFFSET(channel_index, direction))
#define VDMA_CHANNEL_NUM_PROC_OFFSET(channel_index, direction) (((direction) == DMA_TO_DEVICE) ? \
(((channel_index) << 5) + 0x4) : (((channel_index) << 5) + 0x14))
#define VDMA_CHANNEL_NUM_PROC_ADDRESS(vdma_registers, channel_index, direction) \
((u8*)((vdma_registers)->address) + VDMA_CHANNEL_NUM_PROC_OFFSET(channel_index, direction))
// dmabuf is supported from linux kernel version 3.3
#if LINUX_VERSION_CODE < KERNEL_VERSION( 3, 3, 0 )
// Make dummy struct with one byte (C standards does not allow empty struct) - in order to not have to ifdef everywhere
struct hailo_dmabuf_info {
uint8_t dummy;
};
#else
// dmabuf_sg_table is needed because in dma_buf_unmap_attachment() the sg_table's address has to match the
// The one returned from dma_buf_map_attachment() - otherwise we would need to malloc each time
struct hailo_dmabuf_info {
struct dma_buf *dmabuf;
struct dma_buf_attachment *dmabuf_attachment;
struct sg_table *dmabuf_sg_table;
};
#endif // LINUX_VERSION_CODE < KERNEL_VERSION( 3, 3, 0 )
struct hailo_vdma_buffer {
struct list_head mapped_user_buffer_list;
size_t handle;
struct kref kref;
struct device *device;
uintptr_t user_address;
u32 size;
enum dma_data_direction data_direction;
struct sg_table sg_table;
// If this flag is set, the buffer pointed by sg_table is not backed by
// 'struct page' (only by pure pfn). On this case, accessing to the page,
// or calling APIs that access the page (e.g. dma_sync_sg_for_cpu) is not
// allowed.
bool is_mmio;
// Relevant paramaters that need to be saved in case of dmabuf - otherwise struct pointers will be NULL
struct hailo_dmabuf_info dmabuf_info;
};
// Continuous buffer that holds a descriptor list.
struct hailo_descriptors_list_buffer {
struct list_head descriptors_buffer_list;
uintptr_t handle;
void *kernel_address;
dma_addr_t dma_address;
u32 buffer_size;
struct hailo_vdma_descriptors_list desc_list;
};
struct hailo_vdma_low_memory_buffer {
struct list_head vdma_low_memory_buffer_list;
uintptr_t handle;
size_t pages_count;
void **pages_address;
};
struct hailo_vdma_continuous_buffer {
struct list_head continuous_buffer_list;
uintptr_t handle;
void *kernel_address;
dma_addr_t dma_address;
size_t size;
};
struct hailo_vdma_controller;
struct hailo_vdma_controller_ops {
void (*update_channel_interrupts)(struct hailo_vdma_controller *controller, size_t engine_index,
u32 channels_bitmap);
};
struct hailo_vdma_controller {
struct hailo_vdma_hw *hw;
struct hailo_vdma_controller_ops *ops;
struct device *dev;
size_t vdma_engines_count;
struct hailo_vdma_engine *vdma_engines;
spinlock_t interrupts_lock;
wait_queue_head_t interrupts_wq;
struct file *used_by_filp;
// Putting big IOCTL structures here to avoid stack allocation.
struct hailo_vdma_interrupts_read_timestamp_params read_interrupt_timestamps_params;
};
#define for_each_vdma_engine(controller, engine, engine_index) \
_for_each_element_array(controller->vdma_engines, controller->vdma_engines_count, \
engine, engine_index)
struct hailo_vdma_file_context {
atomic_t last_vdma_user_buffer_handle;
struct list_head mapped_user_buffer_list;
// Last_vdma_handle works as a handle for vdma decriptor list and for the vdma buffer -
// there will be no collisions between the two
atomic_t last_vdma_handle;
struct list_head descriptors_buffer_list;
struct list_head vdma_low_memory_buffer_list;
struct list_head continuous_buffer_list;
u32 enabled_channels_bitmap[MAX_VDMA_ENGINES];
};
int hailo_vdma_controller_init(struct hailo_vdma_controller *controller,
struct device *dev, struct hailo_vdma_hw *vdma_hw,
struct hailo_vdma_controller_ops *ops,
struct hailo_resource *channel_registers_per_engine, size_t engines_count);
void hailo_vdma_update_interrupts_mask(struct hailo_vdma_controller *controller,
size_t engine_index);
void hailo_vdma_file_context_init(struct hailo_vdma_file_context *context);
void hailo_vdma_file_context_finalize(struct hailo_vdma_file_context *context,
struct hailo_vdma_controller *controller, struct file *filp);
void hailo_vdma_wakeup_interrupts(struct hailo_vdma_controller *controller, struct hailo_vdma_engine *engine,
u32 channels_bitmap);
void hailo_vdma_irq_handler(struct hailo_vdma_controller *controller, size_t engine_index,
u32 channels_bitmap);
// TODO: reduce params count
long hailo_vdma_ioctl(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
unsigned int cmd, unsigned long arg, struct file *filp, struct semaphore *mutex, bool *should_up_board_mutex);
int hailo_vdma_mmap(struct hailo_vdma_file_context *context, struct hailo_vdma_controller *controller,
struct vm_area_struct *vma, uintptr_t vdma_handle);
enum dma_data_direction get_dma_direction(enum hailo_dma_data_direction hailo_direction);
void hailo_vdma_disable_vdma_channels(struct hailo_vdma_controller *controller, const bool should_close_channels);
#endif /* _HAILO_VDMA_VDMA_H_ */