2743 lines
83 KiB
C
2743 lines
83 KiB
C
/* Advanced Host Controller Interface (AHCI) driver, by D.C. van Moolenbroek
|
|
* - Multithreading support by Arne Welzel
|
|
* - Native Command Queuing support by Raja Appuswamy
|
|
*/
|
|
/*
|
|
* This driver is based on the following specifications:
|
|
* - Serial ATA Advanced Host Controller Interface (AHCI) 1.3
|
|
* - Serial ATA Revision 2.6
|
|
* - AT Attachment with Packet Interface 7 (ATA/ATAPI-7)
|
|
* - ATAPI Removable Rewritable Media Devices 1.3 (SFF-8070)
|
|
*
|
|
* The driver supports device hot-plug, active device status tracking,
|
|
* nonremovable ATA and removable ATAPI devices, custom logical sector sizes,
|
|
* sector-unaligned reads, native command queuing and parallel requests to
|
|
* different devices.
|
|
*
|
|
* It does not implement transparent failure recovery, power management, or
|
|
* port multiplier support.
|
|
*/
|
|
/*
|
|
* An AHCI controller exposes a number of ports (up to 32), each of which may
|
|
* or may not have one device attached (port multipliers are not supported).
|
|
* Each port is maintained independently.
|
|
*
|
|
* The following figure depicts the possible transitions between port states.
|
|
* The NO_PORT state is not included; no transitions can be made from or to it.
|
|
*
|
|
* +----------+ +----------+
|
|
* | SPIN_UP | ------+ +-----> | BAD_DEV | ------------------+
|
|
* +----------+ | | +----------+ |
|
|
* | | | ^ |
|
|
* v v | | |
|
|
* +----------+ +----------+ +----------+ +----------+ |
|
|
* | NO_DEV | --> | WAIT_DEV | --> | WAIT_ID | --> | GOOD_DEV | |
|
|
* +----------+ +----------+ +----------+ +----------+ |
|
|
* ^ | | | |
|
|
* +----------------+----------------+----------------+--------+
|
|
*
|
|
* At driver startup, all physically present ports are put in SPIN_UP state.
|
|
* This state differs from NO_DEV in that BDEV_OPEN calls will be deferred
|
|
* until either the spin-up timer expires, or a device has been identified on
|
|
* that port. This prevents early BDEV_OPEN calls from failing erroneously at
|
|
* startup time if the device has not yet been able to announce its presence.
|
|
*
|
|
* If a device is detected, either at startup time or after hot-plug, its
|
|
* signature is checked and it is identified, after which it may be determined
|
|
* to be a usable ("good") device, which means that the device is considered to
|
|
* be in a working state. If these steps fail, the device is marked as unusable
|
|
* ("bad"). At any point in time, the device may be disconnected; the port is
|
|
* then put back into NO_DEV state.
|
|
*
|
|
* A device in working state (GOOD_DEV) may or may not have a medium. All ATA
|
|
* devices are assumed to be fixed; all ATAPI devices are assumed to have
|
|
* removable media. To prevent erroneous access to switched devices and media,
|
|
* the driver makes devices inaccessible until they are fully closed (the open
|
|
* count is zero) when a device (hot-plug) or medium change is detected.
|
|
* For hot-plug changes, access is prevented by setting the BARRIER flag until
|
|
* the device is fully closed and then reopened. For medium changes, access is
|
|
* prevented by not acknowledging the medium change until the device is fully
|
|
* closed and reopened. Removable media are not locked in the drive while
|
|
* opened, because the driver author is uncomfortable with that concept.
|
|
*
|
|
* Ports may leave the group of states where a device is connected (that is,
|
|
* WAIT_ID, GOOD_DEV, and BAD_DEV) in two ways: either due to a hot-unplug
|
|
* event, or due to a hard reset after a serious failure. For simplicity, we
|
|
* we perform a hard reset after a hot-unplug event as well, so that the link
|
|
* to the device is broken. Thus, in both cases, a transition to NO_DEV is
|
|
* made, after which the link to the device may or may not be reestablished.
|
|
* In both cases, ongoing requests are cancelled and the BARRIER flag is set.
|
|
*
|
|
* The following table lists for each state, whether the port is started
|
|
* (PxCMD.ST is set), whether a timer is running, what the PxIE mask is to be
|
|
* set to, and what BDEV_OPEN calls on this port should return.
|
|
*
|
|
* State Started Timer PxIE BDEV_OPEN
|
|
* --------- --------- --------- --------- ---------
|
|
* NO_PORT no no (none) ENXIO
|
|
* SPIN_UP no yes PCE (wait)
|
|
* NO_DEV no no PCE ENXIO
|
|
* WAIT_DEV no yes PCE (wait)
|
|
* BAD_DEV no no PRCE ENXIO
|
|
* WAIT_ID yes yes PRCE+ (wait)
|
|
* GOOD_DEV yes per-command PRCE+ OK
|
|
*
|
|
* In order to continue deferred BDEV_OPEN calls, the BUSY flag must be unset
|
|
* when changing from SPIN_UP to any state but WAIT_DEV, and when changing from
|
|
* WAIT_DEV to any state but WAIT_ID, and when changing from WAIT_ID to any
|
|
* other state.
|
|
*/
|
|
/*
|
|
* The maximum byte size of a single transfer (MAX_TRANSFER) is currently set
|
|
* to 4MB. This limit has been chosen for a number of reasons:
|
|
* - The size that can be specified in a Physical Region Descriptor (PRD) is
|
|
* limited to 4MB for AHCI. Limiting the total transfer size to at most this
|
|
* size implies that no I/O vector element needs to be split up across PRDs.
|
|
* This means that the maximum number of needed PRDs can be predetermined.
|
|
* - The limit is below what can be transferred in a single ATA request, namely
|
|
* 64k sectors (i.e., at least 32MB). This means that transfer requests need
|
|
* never be split up into smaller chunks, reducing implementation complexity.
|
|
* - A single, static timeout can be used for transfers. Very large transfers
|
|
* can legitimately take up to several minutes -- well beyond the appropriate
|
|
* timeout range for small transfers. The limit obviates the need for a
|
|
* timeout scheme that takes into account the transfer size.
|
|
* - Similarly, the transfer limit reduces the opportunity for buggy/malicious
|
|
* clients to keep the driver busy for a long time with a single request.
|
|
* - The limit is high enough for all practical purposes. The transfer setup
|
|
* overhead is already relatively negligible at this size, and even larger
|
|
* requests will not help maximize throughput. As NR_IOREQS is currently set
|
|
* to 64, the limit still allows file systems to perform I/O requests with
|
|
* vectors completely filled with 64KB-blocks.
|
|
*/
|
|
#include <minix/drivers.h>
|
|
#include <minix/blockdriver_mt.h>
|
|
#include <minix/drvlib.h>
|
|
#include <machine/pci.h>
|
|
#include <sys/ioc_disk.h>
|
|
#include <sys/mman.h>
|
|
#include <assert.h>
|
|
|
|
#include "ahci.h"
|
|
|
|
/* Host Bus Adapter (HBA) state. */
|
|
static struct {
|
|
volatile u32_t *base; /* base address of memory-mapped registers */
|
|
size_t size; /* size of memory-mapped register area */
|
|
|
|
int nr_ports; /* addressable number of ports (1..NR_PORTS) */
|
|
int nr_cmds; /* maximum number of commands per port */
|
|
int has_ncq; /* NCQ support flag */
|
|
int has_clo; /* CLO support flag */
|
|
|
|
int irq; /* IRQ number */
|
|
int hook_id; /* IRQ hook ID */
|
|
} hba_state;
|
|
|
|
#define hba_read(r) (hba_state.base[r])
|
|
#define hba_write(r, v) (hba_state.base[r] = (v))
|
|
|
|
/* Port state. */
|
|
static struct port_state {
|
|
int state; /* port state */
|
|
unsigned int flags; /* port flags */
|
|
|
|
volatile u32_t *reg; /* memory-mapped port registers */
|
|
|
|
u8_t *mem_base; /* primary memory buffer virtual address */
|
|
phys_bytes mem_phys; /* primary memory buffer physical address */
|
|
vir_bytes mem_size; /* primary memory buffer size */
|
|
|
|
/* the FIS, CL, CT[0] and TMP buffers are all in the primary buffer */
|
|
u32_t *fis_base; /* FIS receive buffer virtual address */
|
|
phys_bytes fis_phys; /* FIS receive buffer physical address */
|
|
u32_t *cl_base; /* command list buffer virtual address */
|
|
phys_bytes cl_phys; /* command list buffer physical address */
|
|
u8_t *ct_base[NR_CMDS]; /* command table virtual address */
|
|
phys_bytes ct_phys[NR_CMDS]; /* command table physical address */
|
|
u8_t *tmp_base; /* temporary storage buffer virtual address */
|
|
phys_bytes tmp_phys; /* temporary storage buffer physical address */
|
|
|
|
u8_t *pad_base; /* sector padding buffer virtual address */
|
|
phys_bytes pad_phys; /* sector padding buffer physical address */
|
|
vir_bytes pad_size; /* sector padding buffer size */
|
|
|
|
u64_t lba_count; /* number of valid Logical Block Addresses */
|
|
u32_t sector_size; /* medium sector size in bytes */
|
|
|
|
int open_count; /* number of times this port is opened */
|
|
|
|
int device; /* associated device number, or NO_DEVICE */
|
|
struct device part[DEV_PER_DRIVE]; /* partition bases and sizes */
|
|
struct device subpart[SUB_PER_DRIVE]; /* same for subpartitions */
|
|
|
|
timer_t timer; /* port-specific timeout timer */
|
|
int left; /* number of tries left before giving up */
|
|
/* (only used for signature probing) */
|
|
|
|
int queue_depth; /* NCQ queue depth */
|
|
u32_t pend_mask; /* commands not yet complete */
|
|
struct {
|
|
thread_id_t tid;/* ID of the worker thread */
|
|
timer_t timer; /* timer associated with each request */
|
|
int result; /* success/failure result of the commands */
|
|
} cmd_info[NR_CMDS];
|
|
} port_state[NR_PORTS];
|
|
|
|
#define port_read(ps, r) ((ps)->reg[r])
|
|
#define port_write(ps, r, v) ((ps)->reg[r] = (v))
|
|
|
|
static int ahci_instance; /* driver instance number */
|
|
|
|
static int ahci_verbose; /* verbosity level (0..4) */
|
|
|
|
/* Timeout-related values. */
|
|
static clock_t ahci_spinup_timeout;
|
|
static clock_t ahci_device_timeout;
|
|
static clock_t ahci_device_delay;
|
|
static unsigned int ahci_device_checks;
|
|
static clock_t ahci_command_timeout;
|
|
static clock_t ahci_transfer_timeout;
|
|
static clock_t ahci_flush_timeout;
|
|
|
|
/* Timeout environment variable names and default values. */
|
|
static struct {
|
|
char *name; /* environment variable name */
|
|
u32_t default_ms; /* default in milliseconds */
|
|
clock_t *ptr; /* clock ticks value pointer */
|
|
} ahci_timevar[] = {
|
|
{ "ahci_init_timeout", SPINUP_TIMEOUT, &ahci_spinup_timeout },
|
|
{ "ahci_device_timeout", DEVICE_TIMEOUT, &ahci_device_timeout },
|
|
{ "ahci_cmd_timeout", COMMAND_TIMEOUT, &ahci_command_timeout },
|
|
{ "ahci_io_timeout", TRANSFER_TIMEOUT, &ahci_transfer_timeout },
|
|
{ "ahci_flush_timeout", FLUSH_TIMEOUT, &ahci_flush_timeout }
|
|
};
|
|
|
|
static int ahci_map[MAX_DRIVES]; /* device-to-port mapping */
|
|
|
|
static int ahci_exiting = FALSE; /* exit after last close? */
|
|
|
|
#define BUILD_ARG(port, tag) (((port) << 8) | (tag))
|
|
#define GET_PORT(arg) ((arg) >> 8)
|
|
#define GET_TAG(arg) ((arg) & 0xFF)
|
|
|
|
#define dprintf(v,s) do { \
|
|
if (ahci_verbose >= (v)) \
|
|
printf s; \
|
|
} while (0)
|
|
|
|
/* Convert milliseconds to clock ticks. Round up. */
|
|
#define millis_to_hz(ms) (((ms) * sys_hz() + 999) / 1000)
|
|
|
|
static void port_set_cmd(struct port_state *ps, int cmd, cmd_fis_t *fis,
|
|
u8_t packet[ATAPI_PACKET_SIZE], prd_t *prdt, int nr_prds, int write);
|
|
static void port_issue(struct port_state *ps, int cmd, clock_t timeout);
|
|
static int port_exec(struct port_state *ps, int cmd, clock_t timeout);
|
|
static void port_timeout(struct timer *tp);
|
|
static void port_disconnect(struct port_state *ps);
|
|
|
|
static char *ahci_portname(struct port_state *ps);
|
|
static int ahci_open(dev_t minor, int access);
|
|
static int ahci_close(dev_t minor);
|
|
static ssize_t ahci_transfer(dev_t minor, int do_write, u64_t position,
|
|
endpoint_t endpt, iovec_t *iovec, unsigned int count,
|
|
int flags);
|
|
static struct device *ahci_part(dev_t minor);
|
|
static void ahci_alarm(clock_t stamp);
|
|
static int ahci_ioctl(dev_t minor, unsigned int request, endpoint_t endpt,
|
|
cp_grant_id_t grant);
|
|
static void ahci_intr(unsigned int mask);
|
|
static int ahci_device(dev_t minor, device_id_t *id);
|
|
static struct port_state *ahci_get_port(dev_t minor);
|
|
|
|
/* AHCI driver table. */
|
|
static struct blockdriver ahci_dtab = {
|
|
BLOCKDRIVER_TYPE_DISK,
|
|
ahci_open,
|
|
ahci_close,
|
|
ahci_transfer,
|
|
ahci_ioctl,
|
|
NULL, /* bdr_cleanup */
|
|
ahci_part,
|
|
NULL, /* bdr_geometry */
|
|
ahci_intr,
|
|
ahci_alarm,
|
|
NULL, /* bdr_other */
|
|
ahci_device
|
|
};
|
|
|
|
/*===========================================================================*
|
|
* atapi_exec *
|
|
*===========================================================================*/
|
|
static int atapi_exec(struct port_state *ps, int cmd,
|
|
u8_t packet[ATAPI_PACKET_SIZE], size_t size, int write)
|
|
{
|
|
/* Execute an ATAPI command. Return OK or error.
|
|
*/
|
|
cmd_fis_t fis;
|
|
prd_t prd[1];
|
|
int nr_prds = 0;
|
|
|
|
assert(size <= AHCI_TMP_SIZE);
|
|
|
|
/* Fill in the command table with a FIS, a packet, and if a data
|
|
* transfer is requested, also a PRD.
|
|
*/
|
|
memset(&fis, 0, sizeof(fis));
|
|
fis.cf_cmd = ATA_CMD_PACKET;
|
|
|
|
if (size > 0) {
|
|
fis.cf_feat = ATA_FEAT_PACKET_DMA;
|
|
if (!write && (ps->flags & FLAG_USE_DMADIR))
|
|
fis.cf_feat |= ATA_FEAT_PACKET_DMADIR;
|
|
|
|
prd[0].vp_addr = ps->tmp_phys;
|
|
prd[0].vp_size = size;
|
|
nr_prds++;
|
|
}
|
|
|
|
/* Start the command, and wait for it to complete or fail. */
|
|
port_set_cmd(ps, cmd, &fis, packet, prd, nr_prds, write);
|
|
|
|
return port_exec(ps, cmd, ahci_command_timeout);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* atapi_test_unit *
|
|
*===========================================================================*/
|
|
static int atapi_test_unit(struct port_state *ps, int cmd)
|
|
{
|
|
/* Test whether the ATAPI device and medium are ready.
|
|
*/
|
|
u8_t packet[ATAPI_PACKET_SIZE];
|
|
|
|
memset(packet, 0, sizeof(packet));
|
|
packet[0] = ATAPI_CMD_TEST_UNIT;
|
|
|
|
return atapi_exec(ps, cmd, packet, 0, FALSE);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* atapi_request_sense *
|
|
*===========================================================================*/
|
|
static int atapi_request_sense(struct port_state *ps, int cmd, int *sense)
|
|
{
|
|
/* Request error (sense) information from an ATAPI device, and return
|
|
* the sense key. The additional sense codes are not used at this time.
|
|
*/
|
|
u8_t packet[ATAPI_PACKET_SIZE];
|
|
int r;
|
|
|
|
memset(packet, 0, sizeof(packet));
|
|
packet[0] = ATAPI_CMD_REQUEST_SENSE;
|
|
packet[4] = ATAPI_REQUEST_SENSE_LEN;
|
|
|
|
r = atapi_exec(ps, cmd, packet, ATAPI_REQUEST_SENSE_LEN, FALSE);
|
|
|
|
if (r != OK)
|
|
return r;
|
|
|
|
dprintf(V_REQ, ("%s: ATAPI SENSE: sense %x ASC %x ASCQ %x\n",
|
|
ahci_portname(ps), ps->tmp_base[2] & 0xF, ps->tmp_base[12],
|
|
ps->tmp_base[13]));
|
|
|
|
*sense = ps->tmp_base[2] & 0xF;
|
|
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* atapi_load_eject *
|
|
*===========================================================================*/
|
|
static int atapi_load_eject(struct port_state *ps, int cmd, int load)
|
|
{
|
|
/* Load or eject a medium in an ATAPI device.
|
|
*/
|
|
u8_t packet[ATAPI_PACKET_SIZE];
|
|
|
|
memset(packet, 0, sizeof(packet));
|
|
packet[0] = ATAPI_CMD_START_STOP;
|
|
packet[4] = load ? ATAPI_START_STOP_LOAD : ATAPI_START_STOP_EJECT;
|
|
|
|
return atapi_exec(ps, cmd, packet, 0, FALSE);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* atapi_read_capacity *
|
|
*===========================================================================*/
|
|
static int atapi_read_capacity(struct port_state *ps, int cmd)
|
|
{
|
|
/* Retrieve the LBA count and sector size of an ATAPI medium.
|
|
*/
|
|
u8_t packet[ATAPI_PACKET_SIZE], *buf;
|
|
int r;
|
|
|
|
memset(packet, 0, sizeof(packet));
|
|
packet[0] = ATAPI_CMD_READ_CAPACITY;
|
|
|
|
r = atapi_exec(ps, cmd, packet, ATAPI_READ_CAPACITY_LEN, FALSE);
|
|
if (r != OK)
|
|
return r;
|
|
|
|
/* Store the number of LBA blocks and sector size. */
|
|
buf = ps->tmp_base;
|
|
ps->lba_count = add64u(cvu64((buf[0] << 24) | (buf[1] << 16) |
|
|
(buf[2] << 8) | buf[3]), 1);
|
|
ps->sector_size =
|
|
(buf[4] << 24) | (buf[5] << 16) | (buf[6] << 8) | buf[7];
|
|
|
|
if (ps->sector_size == 0 || (ps->sector_size & 1)) {
|
|
dprintf(V_ERR, ("%s: invalid medium sector size %u\n",
|
|
ahci_portname(ps), ps->sector_size));
|
|
|
|
return EINVAL;
|
|
}
|
|
|
|
dprintf(V_INFO,
|
|
("%s: medium detected (%u byte sectors, %lu MB size)\n",
|
|
ahci_portname(ps), ps->sector_size,
|
|
div64u(mul64(ps->lba_count, cvu64(ps->sector_size)),
|
|
1024*1024)));
|
|
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* atapi_check_medium *
|
|
*===========================================================================*/
|
|
static int atapi_check_medium(struct port_state *ps, int cmd)
|
|
{
|
|
/* Check whether a medium is present in a removable-media ATAPI device.
|
|
* If a new medium is detected, get its total and sector size. Return
|
|
* OK only if a usable medium is present, and an error otherwise.
|
|
*/
|
|
int sense;
|
|
|
|
/* Perform a readiness check. */
|
|
if (atapi_test_unit(ps, cmd) != OK) {
|
|
ps->flags &= ~FLAG_HAS_MEDIUM;
|
|
|
|
/* If the check failed due to a unit attention condition, retry
|
|
* reading the medium capacity. Otherwise, assume that there is
|
|
* no medium available.
|
|
*/
|
|
if (atapi_request_sense(ps, cmd, &sense) != OK ||
|
|
sense != ATAPI_SENSE_UNIT_ATT)
|
|
return ENXIO;
|
|
}
|
|
|
|
/* If a medium is newly detected, try reading its capacity now. */
|
|
if (!(ps->flags & FLAG_HAS_MEDIUM)) {
|
|
if (atapi_read_capacity(ps, cmd) != OK)
|
|
return EIO;
|
|
|
|
ps->flags |= FLAG_HAS_MEDIUM;
|
|
}
|
|
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* atapi_id_check *
|
|
*===========================================================================*/
|
|
static int atapi_id_check(struct port_state *ps, u16_t *buf)
|
|
{
|
|
/* Determine whether we support this ATAPI device based on the
|
|
* identification data it returned, and store some of its properties.
|
|
*/
|
|
|
|
/* The device must be an ATAPI device; it must have removable media;
|
|
* it must support DMA without DMADIR, or DMADIR for DMA.
|
|
*/
|
|
if ((buf[ATA_ID_GCAP] & (ATA_ID_GCAP_ATAPI_MASK |
|
|
ATA_ID_GCAP_REMOVABLE | ATA_ID_GCAP_INCOMPLETE)) !=
|
|
(ATA_ID_GCAP_ATAPI | ATA_ID_GCAP_REMOVABLE) ||
|
|
((buf[ATA_ID_CAP] & ATA_ID_CAP_DMA) != ATA_ID_CAP_DMA &&
|
|
(buf[ATA_ID_DMADIR] & (ATA_ID_DMADIR_DMADIR |
|
|
ATA_ID_DMADIR_DMA)) != (ATA_ID_DMADIR_DMADIR |
|
|
ATA_ID_DMADIR_DMA))) {
|
|
|
|
dprintf(V_ERR, ("%s: unsupported ATAPI device\n",
|
|
ahci_portname(ps)));
|
|
|
|
dprintf(V_DEV, ("%s: GCAP %04x CAP %04x DMADIR %04x\n",
|
|
ahci_portname(ps), buf[ATA_ID_GCAP], buf[ATA_ID_CAP],
|
|
buf[ATA_ID_DMADIR]));
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
/* Remember whether to use the DMADIR flag when appropriate. */
|
|
if (buf[ATA_ID_DMADIR] & ATA_ID_DMADIR_DMADIR)
|
|
ps->flags |= FLAG_USE_DMADIR;
|
|
|
|
/* ATAPI CD-ROM devices are considered read-only. */
|
|
if (((buf[ATA_ID_GCAP] & ATA_ID_GCAP_TYPE_MASK) >>
|
|
ATA_ID_GCAP_TYPE_SHIFT) == ATAPI_TYPE_CDROM)
|
|
ps->flags |= FLAG_READONLY;
|
|
|
|
if ((buf[ATA_ID_SUP1] & ATA_ID_SUP1_VALID_MASK) == ATA_ID_SUP1_VALID &&
|
|
!(ps->flags & FLAG_READONLY)) {
|
|
/* Save write cache related capabilities of the device. It is
|
|
* possible, although unlikely, that a device has support for
|
|
* either of these but not both.
|
|
*/
|
|
if (buf[ATA_ID_SUP0] & ATA_ID_SUP0_WCACHE)
|
|
ps->flags |= FLAG_HAS_WCACHE;
|
|
|
|
if (buf[ATA_ID_SUP1] & ATA_ID_SUP1_FLUSH)
|
|
ps->flags |= FLAG_HAS_FLUSH;
|
|
}
|
|
|
|
return TRUE;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* atapi_transfer *
|
|
*===========================================================================*/
|
|
static int atapi_transfer(struct port_state *ps, int cmd, u64_t start_lba,
|
|
unsigned int count, int write, prd_t *prdt, int nr_prds)
|
|
{
|
|
/* Perform data transfer from or to an ATAPI device.
|
|
*/
|
|
cmd_fis_t fis;
|
|
u8_t packet[ATAPI_PACKET_SIZE];
|
|
|
|
/* Fill in a Register Host to Device FIS. */
|
|
memset(&fis, 0, sizeof(fis));
|
|
fis.cf_cmd = ATA_CMD_PACKET;
|
|
fis.cf_feat = ATA_FEAT_PACKET_DMA;
|
|
if (!write && (ps->flags & FLAG_USE_DMADIR))
|
|
fis.cf_feat |= ATA_FEAT_PACKET_DMADIR;
|
|
|
|
/* Fill in a packet. */
|
|
memset(packet, 0, sizeof(packet));
|
|
packet[0] = write ? ATAPI_CMD_WRITE : ATAPI_CMD_READ;
|
|
packet[2] = (ex64lo(start_lba) >> 24) & 0xFF;
|
|
packet[3] = (ex64lo(start_lba) >> 16) & 0xFF;
|
|
packet[4] = (ex64lo(start_lba) >> 8) & 0xFF;
|
|
packet[5] = ex64lo(start_lba) & 0xFF;
|
|
packet[6] = (count >> 24) & 0xFF;
|
|
packet[7] = (count >> 16) & 0xFF;
|
|
packet[8] = (count >> 8) & 0xFF;
|
|
packet[9] = count & 0xFF;
|
|
|
|
/* Start the command, and wait for it to complete or fail. */
|
|
port_set_cmd(ps, cmd, &fis, packet, prdt, nr_prds, write);
|
|
|
|
return port_exec(ps, cmd, ahci_transfer_timeout);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ata_id_check *
|
|
*===========================================================================*/
|
|
static int ata_id_check(struct port_state *ps, u16_t *buf)
|
|
{
|
|
/* Determine whether we support this ATA device based on the
|
|
* identification data it returned, and store some of its properties.
|
|
*/
|
|
|
|
/* This must be an ATA device; it must not have removable media;
|
|
* it must support LBA and DMA; it must support the FLUSH CACHE
|
|
* command; it must support 48-bit addressing.
|
|
*/
|
|
if ((buf[ATA_ID_GCAP] & (ATA_ID_GCAP_ATA_MASK | ATA_ID_GCAP_REMOVABLE |
|
|
ATA_ID_GCAP_INCOMPLETE)) != ATA_ID_GCAP_ATA ||
|
|
(buf[ATA_ID_CAP] & (ATA_ID_CAP_LBA | ATA_ID_CAP_DMA)) !=
|
|
(ATA_ID_CAP_LBA | ATA_ID_CAP_DMA) ||
|
|
(buf[ATA_ID_SUP1] & (ATA_ID_SUP1_VALID_MASK |
|
|
ATA_ID_SUP1_FLUSH | ATA_ID_SUP1_LBA48)) !=
|
|
(ATA_ID_SUP1_VALID | ATA_ID_SUP1_FLUSH | ATA_ID_SUP1_LBA48)) {
|
|
|
|
dprintf(V_ERR, ("%s: unsupported ATA device\n",
|
|
ahci_portname(ps)));
|
|
|
|
dprintf(V_DEV, ("%s: GCAP %04x CAP %04x SUP1 %04x\n",
|
|
ahci_portname(ps), buf[ATA_ID_GCAP], buf[ATA_ID_CAP],
|
|
buf[ATA_ID_SUP1]));
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
/* Get number of LBA blocks, and sector size. */
|
|
ps->lba_count = make64((buf[ATA_ID_LBA1] << 16) | buf[ATA_ID_LBA0],
|
|
(buf[ATA_ID_LBA3] << 16) | buf[ATA_ID_LBA2]);
|
|
|
|
/* Determine the queue depth of the device. */
|
|
if (hba_state.has_ncq &&
|
|
(buf[ATA_ID_SATA_CAP] & ATA_ID_SATA_CAP_NCQ)) {
|
|
ps->flags |= FLAG_HAS_NCQ;
|
|
ps->queue_depth =
|
|
(buf[ATA_ID_QDEPTH] & ATA_ID_QDEPTH_MASK) + 1;
|
|
if (ps->queue_depth > hba_state.nr_cmds)
|
|
ps->queue_depth = hba_state.nr_cmds;
|
|
}
|
|
|
|
/* For now, we only support long logical sectors. Long physical sector
|
|
* support may be added later. Note that the given value is in words.
|
|
*/
|
|
if ((buf[ATA_ID_PLSS] & (ATA_ID_PLSS_VALID_MASK | ATA_ID_PLSS_LLS)) ==
|
|
(ATA_ID_PLSS_VALID | ATA_ID_PLSS_LLS))
|
|
ps->sector_size =
|
|
((buf[ATA_ID_LSS1] << 16) | buf[ATA_ID_LSS0]) << 1;
|
|
else
|
|
ps->sector_size = ATA_SECTOR_SIZE;
|
|
|
|
if (ps->sector_size < ATA_SECTOR_SIZE) {
|
|
dprintf(V_ERR, ("%s: invalid sector size %u\n",
|
|
ahci_portname(ps), ps->sector_size));
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
ps->flags |= FLAG_HAS_MEDIUM | FLAG_HAS_FLUSH;
|
|
|
|
/* FLUSH CACHE is mandatory for ATA devices; write caches are not. */
|
|
if (buf[ATA_ID_SUP0] & ATA_ID_SUP0_WCACHE)
|
|
ps->flags |= FLAG_HAS_WCACHE;
|
|
|
|
/* Check Force Unit Access capability of the device. */
|
|
if ((buf[ATA_ID_ENA2] & (ATA_ID_ENA2_VALID_MASK | ATA_ID_ENA2_FUA)) ==
|
|
(ATA_ID_ENA2_VALID | ATA_ID_ENA2_FUA))
|
|
ps->flags |= FLAG_HAS_FUA;
|
|
|
|
return TRUE;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ata_transfer *
|
|
*===========================================================================*/
|
|
static int ata_transfer(struct port_state *ps, int cmd, u64_t start_lba,
|
|
unsigned int count, int write, int force, prd_t *prdt, int nr_prds)
|
|
{
|
|
/* Perform data transfer from or to an ATA device.
|
|
*/
|
|
cmd_fis_t fis;
|
|
|
|
assert(count <= ATA_MAX_SECTORS);
|
|
|
|
/* Special case for sector counts: 65536 is specified as 0. */
|
|
if (count == ATA_MAX_SECTORS)
|
|
count = 0;
|
|
|
|
memset(&fis, 0, sizeof(fis));
|
|
fis.cf_dev = ATA_DEV_LBA;
|
|
if (ps->flags & FLAG_HAS_NCQ) {
|
|
if (write) {
|
|
if (force && (ps->flags & FLAG_HAS_FUA))
|
|
fis.cf_dev |= ATA_DEV_FUA;
|
|
|
|
fis.cf_cmd = ATA_CMD_WRITE_FPDMA_QUEUED;
|
|
} else {
|
|
fis.cf_cmd = ATA_CMD_READ_FPDMA_QUEUED;
|
|
}
|
|
}
|
|
else {
|
|
if (write) {
|
|
if (force && (ps->flags & FLAG_HAS_FUA))
|
|
fis.cf_cmd = ATA_CMD_WRITE_DMA_FUA_EXT;
|
|
else
|
|
fis.cf_cmd = ATA_CMD_WRITE_DMA_EXT;
|
|
}
|
|
else {
|
|
fis.cf_cmd = ATA_CMD_READ_DMA_EXT;
|
|
}
|
|
}
|
|
fis.cf_lba = ex64lo(start_lba) & 0x00FFFFFFL;
|
|
fis.cf_lba_exp = ex64lo(rshift64(start_lba, 24)) & 0x00FFFFFFL;
|
|
fis.cf_sec = count & 0xFF;
|
|
fis.cf_sec_exp = (count >> 8) & 0xFF;
|
|
|
|
/* Start the command, and wait for it to complete or fail. */
|
|
port_set_cmd(ps, cmd, &fis, NULL /*packet*/, prdt, nr_prds, write);
|
|
|
|
return port_exec(ps, cmd, ahci_transfer_timeout);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* gen_identify *
|
|
*===========================================================================*/
|
|
static int gen_identify(struct port_state *ps, int blocking)
|
|
{
|
|
/* Identify an ATA or ATAPI device. If the blocking flag is set, block
|
|
* until the command has completed; otherwise return immediately.
|
|
*/
|
|
cmd_fis_t fis;
|
|
prd_t prd;
|
|
|
|
/* Set up a command, and a single PRD for the result. */
|
|
memset(&fis, 0, sizeof(fis));
|
|
|
|
if (ps->flags & FLAG_ATAPI)
|
|
fis.cf_cmd = ATA_CMD_IDENTIFY_PACKET;
|
|
else
|
|
fis.cf_cmd = ATA_CMD_IDENTIFY;
|
|
|
|
prd.vp_addr = ps->tmp_phys;
|
|
prd.vp_size = ATA_ID_SIZE;
|
|
|
|
/* Start the command, and possibly wait for the result. */
|
|
port_set_cmd(ps, 0, &fis, NULL /*packet*/, &prd, 1, FALSE /*write*/);
|
|
|
|
if (blocking)
|
|
return port_exec(ps, 0, ahci_command_timeout);
|
|
|
|
port_issue(ps, 0, ahci_command_timeout);
|
|
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* gen_flush_wcache *
|
|
*===========================================================================*/
|
|
static int gen_flush_wcache(struct port_state *ps)
|
|
{
|
|
/* Flush the device's write cache.
|
|
*/
|
|
cmd_fis_t fis;
|
|
|
|
/* The FLUSH CACHE command may not be supported by all (writable ATAPI)
|
|
* devices.
|
|
*/
|
|
if (!(ps->flags & FLAG_HAS_FLUSH))
|
|
return EINVAL;
|
|
|
|
/* Use the FLUSH CACHE command for both ATA and ATAPI. We are not
|
|
* interested in the disk location of a failure, so there is no reason
|
|
* to use the ATA-only FLUSH CACHE EXT command. Either way, the command
|
|
* may indeed fail due to a disk error, in which case it should be
|
|
* repeated. For now, we shift this responsibility onto the caller.
|
|
*/
|
|
memset(&fis, 0, sizeof(fis));
|
|
fis.cf_cmd = ATA_CMD_FLUSH_CACHE;
|
|
|
|
/* Start the command, and wait for it to complete or fail.
|
|
* The flush command may take longer than regular I/O commands.
|
|
*/
|
|
port_set_cmd(ps, 0, &fis, NULL /*packet*/, NULL /*prdt*/, 0,
|
|
FALSE /*write*/);
|
|
|
|
return port_exec(ps, 0, ahci_flush_timeout);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* gen_get_wcache *
|
|
*===========================================================================*/
|
|
static int gen_get_wcache(struct port_state *ps, int *val)
|
|
{
|
|
/* Retrieve the status of the device's write cache.
|
|
*/
|
|
int r;
|
|
|
|
/* Write caches are not mandatory. */
|
|
if (!(ps->flags & FLAG_HAS_WCACHE))
|
|
return EINVAL;
|
|
|
|
/* Retrieve information about the device. */
|
|
if ((r = gen_identify(ps, TRUE /*blocking*/)) != OK)
|
|
return r;
|
|
|
|
/* Return the current setting. */
|
|
*val = !!(((u16_t *) ps->tmp_base)[ATA_ID_ENA0] & ATA_ID_ENA0_WCACHE);
|
|
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* gen_set_wcache *
|
|
*===========================================================================*/
|
|
static int gen_set_wcache(struct port_state *ps, int enable)
|
|
{
|
|
/* Enable or disable the device's write cache.
|
|
*/
|
|
cmd_fis_t fis;
|
|
clock_t timeout;
|
|
|
|
/* Write caches are not mandatory. */
|
|
if (!(ps->flags & FLAG_HAS_WCACHE))
|
|
return EINVAL;
|
|
|
|
/* Disabling the write cache causes a (blocking) cache flush. Cache
|
|
* flushes may take much longer than regular commands.
|
|
*/
|
|
timeout = enable ? ahci_command_timeout : ahci_flush_timeout;
|
|
|
|
/* Set up a command. */
|
|
memset(&fis, 0, sizeof(fis));
|
|
fis.cf_cmd = ATA_CMD_SET_FEATURES;
|
|
fis.cf_feat = enable ? ATA_SF_EN_WCACHE : ATA_SF_DI_WCACHE;
|
|
|
|
/* Start the command, and wait for it to complete or fail. */
|
|
port_set_cmd(ps, 0, &fis, NULL /*packet*/, NULL /*prdt*/, 0,
|
|
FALSE /*write*/);
|
|
|
|
return port_exec(ps, 0, timeout);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ct_set_fis *
|
|
*===========================================================================*/
|
|
static vir_bytes ct_set_fis(u8_t *ct, cmd_fis_t *fis, unsigned int tag)
|
|
{
|
|
/* Fill in the Frame Information Structure part of a command table,
|
|
* and return the resulting FIS size (in bytes). We only support the
|
|
* command Register - Host to Device FIS type.
|
|
*/
|
|
|
|
memset(ct, 0, ATA_H2D_SIZE);
|
|
ct[ATA_FIS_TYPE] = ATA_FIS_TYPE_H2D;
|
|
ct[ATA_H2D_FLAGS] = ATA_H2D_FLAGS_C;
|
|
ct[ATA_H2D_CMD] = fis->cf_cmd;
|
|
ct[ATA_H2D_LBA_LOW] = fis->cf_lba & 0xFF;
|
|
ct[ATA_H2D_LBA_MID] = (fis->cf_lba >> 8) & 0xFF;
|
|
ct[ATA_H2D_LBA_HIGH] = (fis->cf_lba >> 16) & 0xFF;
|
|
ct[ATA_H2D_DEV] = fis->cf_dev;
|
|
ct[ATA_H2D_LBA_LOW_EXP] = fis->cf_lba_exp & 0xFF;
|
|
ct[ATA_H2D_LBA_MID_EXP] = (fis->cf_lba_exp >> 8) & 0xFF;
|
|
ct[ATA_H2D_LBA_HIGH_EXP] = (fis->cf_lba_exp >> 16) & 0xFF;
|
|
ct[ATA_H2D_CTL] = fis->cf_ctl;
|
|
|
|
if (ATA_IS_FPDMA_CMD(fis->cf_cmd)) {
|
|
ct[ATA_H2D_FEAT] = fis->cf_sec;
|
|
ct[ATA_H2D_FEAT_EXP] = fis->cf_sec_exp;
|
|
ct[ATA_H2D_SEC] = tag << ATA_SEC_TAG_SHIFT;
|
|
ct[ATA_H2D_SEC_EXP] = 0;
|
|
} else {
|
|
ct[ATA_H2D_FEAT] = fis->cf_feat;
|
|
ct[ATA_H2D_FEAT_EXP] = fis->cf_feat_exp;
|
|
ct[ATA_H2D_SEC] = fis->cf_sec;
|
|
ct[ATA_H2D_SEC_EXP] = fis->cf_sec_exp;
|
|
}
|
|
|
|
return ATA_H2D_SIZE;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ct_set_packet *
|
|
*===========================================================================*/
|
|
static void ct_set_packet(u8_t *ct, u8_t packet[ATAPI_PACKET_SIZE])
|
|
{
|
|
/* Fill in the packet part of a command table.
|
|
*/
|
|
|
|
memcpy(&ct[AHCI_CT_PACKET_OFF], packet, ATAPI_PACKET_SIZE);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ct_set_prdt *
|
|
*===========================================================================*/
|
|
static void ct_set_prdt(u8_t *ct, prd_t *prdt, int nr_prds)
|
|
{
|
|
/* Fill in the PRDT part of a command table.
|
|
*/
|
|
u32_t *p;
|
|
int i;
|
|
|
|
p = (u32_t *) &ct[AHCI_CT_PRDT_OFF];
|
|
|
|
for (i = 0; i < nr_prds; i++, prdt++) {
|
|
*p++ = prdt->vp_addr;
|
|
*p++ = 0;
|
|
*p++ = 0;
|
|
*p++ = prdt->vp_size - 1;
|
|
}
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_set_cmd *
|
|
*===========================================================================*/
|
|
static void port_set_cmd(struct port_state *ps, int cmd, cmd_fis_t *fis,
|
|
u8_t packet[ATAPI_PACKET_SIZE], prd_t *prdt, int nr_prds, int write)
|
|
{
|
|
/* Prepare the given command for execution, by constructing a command
|
|
* table and setting up a command list entry pointing to the table.
|
|
*/
|
|
u8_t *ct;
|
|
u32_t *cl;
|
|
vir_bytes size;
|
|
|
|
/* Set a port-specific flag that tells us if the command being
|
|
* processed is a NCQ command or not.
|
|
*/
|
|
if (ATA_IS_FPDMA_CMD(fis->cf_cmd)) {
|
|
ps->flags |= FLAG_NCQ_MODE;
|
|
} else {
|
|
assert(!ps->pend_mask);
|
|
ps->flags &= ~FLAG_NCQ_MODE;
|
|
}
|
|
|
|
/* Construct a command table, consisting of a command FIS, optionally
|
|
* a packet, and optionally a number of PRDs (making up the actual PRD
|
|
* table).
|
|
*/
|
|
ct = ps->ct_base[cmd];
|
|
|
|
assert(ct != NULL);
|
|
assert(nr_prds <= NR_PRDS);
|
|
|
|
size = ct_set_fis(ct, fis, cmd);
|
|
|
|
if (packet != NULL)
|
|
ct_set_packet(ct, packet);
|
|
|
|
ct_set_prdt(ct, prdt, nr_prds);
|
|
|
|
/* Construct a command list entry, pointing to the command's table.
|
|
* Current assumptions: callers always provide a Register - Host to
|
|
* Device type FIS, and all non-NCQ commands are prefetchable.
|
|
*/
|
|
cl = &ps->cl_base[cmd * AHCI_CL_ENTRY_DWORDS];
|
|
|
|
memset(cl, 0, AHCI_CL_ENTRY_SIZE);
|
|
cl[0] = (nr_prds << AHCI_CL_PRDTL_SHIFT) |
|
|
((!ATA_IS_FPDMA_CMD(fis->cf_cmd) &&
|
|
(nr_prds > 0 || packet != NULL)) ? AHCI_CL_PREFETCHABLE : 0) |
|
|
(write ? AHCI_CL_WRITE : 0) |
|
|
((packet != NULL) ? AHCI_CL_ATAPI : 0) |
|
|
((size / sizeof(u32_t)) << AHCI_CL_CFL_SHIFT);
|
|
cl[2] = ps->ct_phys[cmd];
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_finish_cmd *
|
|
*===========================================================================*/
|
|
static void port_finish_cmd(struct port_state *ps, int cmd, int result)
|
|
{
|
|
/* Finish a command that has either succeeded or failed.
|
|
*/
|
|
|
|
assert(cmd < ps->queue_depth);
|
|
|
|
dprintf(V_REQ, ("%s: command %d %s\n", ahci_portname(ps),
|
|
cmd, (result == RESULT_SUCCESS) ? "succeeded" : "failed"));
|
|
|
|
/* Update the command result, and clear it from the pending list. */
|
|
ps->cmd_info[cmd].result = result;
|
|
|
|
assert(ps->pend_mask & (1 << cmd));
|
|
ps->pend_mask &= ~(1 << cmd);
|
|
|
|
/* Wake up the thread, unless it is the main thread. This can happen
|
|
* during initialization, as the gen_identify function is called by the
|
|
* main thread itself.
|
|
*/
|
|
if (ps->state != STATE_WAIT_ID)
|
|
blockdriver_mt_wakeup(ps->cmd_info[cmd].tid);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_fail_cmds *
|
|
*===========================================================================*/
|
|
static void port_fail_cmds(struct port_state *ps)
|
|
{
|
|
/* Fail all ongoing commands for a device.
|
|
*/
|
|
int i;
|
|
|
|
for (i = 0; ps->pend_mask != 0 && i < ps->queue_depth; i++)
|
|
if (ps->pend_mask & (1 << i))
|
|
port_finish_cmd(ps, i, RESULT_FAILURE);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_check_cmds *
|
|
*===========================================================================*/
|
|
static void port_check_cmds(struct port_state *ps)
|
|
{
|
|
/* Check what commands have completed, and finish them.
|
|
*/
|
|
u32_t mask, done;
|
|
int i;
|
|
|
|
/* See which commands have completed. */
|
|
if (ps->flags & FLAG_NCQ_MODE)
|
|
mask = port_read(ps, AHCI_PORT_SACT);
|
|
else
|
|
mask = port_read(ps, AHCI_PORT_CI);
|
|
|
|
/* Wake up threads corresponding to completed commands. */
|
|
done = ps->pend_mask & ~mask;
|
|
|
|
for (i = 0; i < ps->queue_depth; i++)
|
|
if (done & (1 << i))
|
|
port_finish_cmd(ps, i, RESULT_SUCCESS);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_find_cmd *
|
|
*===========================================================================*/
|
|
static int port_find_cmd(struct port_state *ps)
|
|
{
|
|
/* Find a free command tag to queue the current request.
|
|
*/
|
|
int i;
|
|
|
|
for (i = 0; i < ps->queue_depth; i++)
|
|
if (!(ps->pend_mask & (1 << i)))
|
|
break;
|
|
|
|
/* We should always be able to find a free slot, since a thread runs
|
|
* only when it is free, and thus, only because a slot is available.
|
|
*/
|
|
assert(i < ps->queue_depth);
|
|
|
|
return i;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_get_padbuf *
|
|
*===========================================================================*/
|
|
static int port_get_padbuf(struct port_state *ps, size_t size)
|
|
{
|
|
/* Make available a temporary buffer for use by this port. Enlarge the
|
|
* previous buffer if applicable and necessary, potentially changing
|
|
* its physical address.
|
|
*/
|
|
|
|
if (ps->pad_base != NULL && ps->pad_size >= size)
|
|
return OK;
|
|
|
|
if (ps->pad_base != NULL)
|
|
free_contig(ps->pad_base, ps->pad_size);
|
|
|
|
ps->pad_size = size;
|
|
ps->pad_base = alloc_contig(ps->pad_size, 0, &ps->pad_phys);
|
|
|
|
if (ps->pad_base == NULL) {
|
|
dprintf(V_ERR, ("%s: unable to allocate a padding buffer of "
|
|
"size %lu\n", ahci_portname(ps),
|
|
(unsigned long) size));
|
|
|
|
return ENOMEM;
|
|
}
|
|
|
|
dprintf(V_INFO, ("%s: allocated padding buffer of size %lu\n",
|
|
ahci_portname(ps), (unsigned long) size));
|
|
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* sum_iovec *
|
|
*===========================================================================*/
|
|
static int sum_iovec(struct port_state *ps, endpoint_t endpt,
|
|
iovec_s_t *iovec, int nr_req, vir_bytes *total)
|
|
{
|
|
/* Retrieve the total size of the given I/O vector. Check for alignment
|
|
* requirements along the way. Return OK (and the total request size)
|
|
* or an error.
|
|
*/
|
|
vir_bytes size, bytes;
|
|
int i;
|
|
|
|
bytes = 0;
|
|
|
|
for (i = 0; i < nr_req; i++) {
|
|
size = iovec[i].iov_size;
|
|
|
|
if (size == 0 || (size & 1) || size > LONG_MAX) {
|
|
dprintf(V_ERR, ("%s: bad size %lu in iovec from %d\n",
|
|
ahci_portname(ps), size, endpt));
|
|
return EINVAL;
|
|
}
|
|
|
|
bytes += size;
|
|
|
|
if (bytes > LONG_MAX) {
|
|
dprintf(V_ERR, ("%s: iovec size overflow from %d\n",
|
|
ahci_portname(ps), endpt));
|
|
return EINVAL;
|
|
}
|
|
}
|
|
|
|
*total = bytes;
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* setup_prdt *
|
|
*===========================================================================*/
|
|
static int setup_prdt(struct port_state *ps, endpoint_t endpt,
|
|
iovec_s_t *iovec, int nr_req, vir_bytes size, vir_bytes lead,
|
|
int write, prd_t *prdt)
|
|
{
|
|
/* Convert (the first part of) an I/O vector to a Physical Region
|
|
* Descriptor Table describing array that can later be used to set the
|
|
* command's real PRDT. The resulting table as a whole should be
|
|
* sector-aligned; leading and trailing local buffers may have to be
|
|
* used for padding as appropriate. Return the number of PRD entries,
|
|
* or a negative error code.
|
|
*/
|
|
struct vumap_vir vvec[NR_PRDS];
|
|
size_t bytes, trail;
|
|
int i, r, pcount, nr_prds = 0;
|
|
|
|
if (lead > 0) {
|
|
/* Allocate a buffer for the data we don't want. */
|
|
if ((r = port_get_padbuf(ps, ps->sector_size)) != OK)
|
|
return r;
|
|
|
|
prdt[nr_prds].vp_addr = ps->pad_phys;
|
|
prdt[nr_prds].vp_size = lead;
|
|
nr_prds++;
|
|
}
|
|
|
|
/* The sum of lead, size, trail has to be sector-aligned. */
|
|
trail = (ps->sector_size - (lead + size)) % ps->sector_size;
|
|
|
|
/* Get the physical addresses of the given buffers. */
|
|
for (i = 0; i < nr_req && size > 0; i++) {
|
|
bytes = MIN(iovec[i].iov_size, size);
|
|
|
|
if (endpt == SELF)
|
|
vvec[i].vv_addr = (vir_bytes) iovec[i].iov_grant;
|
|
else
|
|
vvec[i].vv_grant = iovec[i].iov_grant;
|
|
|
|
vvec[i].vv_size = bytes;
|
|
|
|
size -= bytes;
|
|
}
|
|
|
|
pcount = i;
|
|
|
|
if ((r = sys_vumap(endpt, vvec, i, 0, write ? VUA_READ : VUA_WRITE,
|
|
&prdt[nr_prds], &pcount)) != OK) {
|
|
dprintf(V_ERR, ("%s: unable to map memory from %d (%d)\n",
|
|
ahci_portname(ps), endpt, r));
|
|
return r;
|
|
}
|
|
|
|
assert(pcount > 0 && pcount <= i);
|
|
|
|
/* Make sure all buffers are physically contiguous and word-aligned. */
|
|
for (i = 0; i < pcount; i++) {
|
|
if (vvec[i].vv_size != prdt[nr_prds].vp_size) {
|
|
dprintf(V_ERR, ("%s: non-contiguous memory from %d\n",
|
|
ahci_portname(ps), endpt));
|
|
return EINVAL;
|
|
}
|
|
|
|
if (prdt[nr_prds].vp_addr & 1) {
|
|
dprintf(V_ERR, ("%s: bad physical address from %d\n",
|
|
ahci_portname(ps), endpt));
|
|
return EINVAL;
|
|
}
|
|
|
|
nr_prds++;
|
|
}
|
|
|
|
if (trail > 0) {
|
|
assert(nr_prds < NR_PRDS);
|
|
prdt[nr_prds].vp_addr = ps->pad_phys + lead;
|
|
prdt[nr_prds].vp_size = trail;
|
|
nr_prds++;
|
|
}
|
|
|
|
return nr_prds;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_transfer *
|
|
*===========================================================================*/
|
|
static ssize_t port_transfer(struct port_state *ps, u64_t pos, u64_t eof,
|
|
endpoint_t endpt, iovec_s_t *iovec, int nr_req, int write, int flags)
|
|
{
|
|
/* Perform an I/O transfer on a port.
|
|
*/
|
|
prd_t prdt[NR_PRDS];
|
|
vir_bytes size, lead;
|
|
unsigned int count, nr_prds;
|
|
u64_t start_lba;
|
|
int r, cmd;
|
|
|
|
/* Get the total request size from the I/O vector. */
|
|
if ((r = sum_iovec(ps, endpt, iovec, nr_req, &size)) != OK)
|
|
return r;
|
|
|
|
dprintf(V_REQ, ("%s: %s for %lu bytes at pos %08lx%08lx\n",
|
|
ahci_portname(ps), write ? "write" : "read", size,
|
|
ex64hi(pos), ex64lo(pos)));
|
|
|
|
assert(ps->state == STATE_GOOD_DEV);
|
|
assert(ps->flags & FLAG_HAS_MEDIUM);
|
|
assert(ps->sector_size > 0);
|
|
|
|
/* Limit the maximum size of a single transfer.
|
|
* See the comments at the top of this file for details.
|
|
*/
|
|
if (size > MAX_TRANSFER)
|
|
size = MAX_TRANSFER;
|
|
|
|
/* If necessary, reduce the request size so that the request does not
|
|
* extend beyond the end of the partition. The caller already
|
|
* guarantees that the starting position lies within the partition.
|
|
*/
|
|
if (cmp64(add64ul(pos, size), eof) >= 0)
|
|
size = (vir_bytes) diff64(eof, pos);
|
|
|
|
start_lba = div64(pos, cvu64(ps->sector_size));
|
|
lead = rem64u(pos, ps->sector_size);
|
|
count = (lead + size + ps->sector_size - 1) / ps->sector_size;
|
|
|
|
/* Position must be word-aligned for read requests, and sector-aligned
|
|
* for write requests. We do not support read-modify-write for writes.
|
|
*/
|
|
if ((lead & 1) || (write && lead != 0)) {
|
|
dprintf(V_ERR, ("%s: unaligned position from %d\n",
|
|
ahci_portname(ps), endpt));
|
|
return EINVAL;
|
|
}
|
|
|
|
/* Write requests must be sector-aligned. Word alignment of the size is
|
|
* already guaranteed by sum_iovec().
|
|
*/
|
|
if (write && (size % ps->sector_size) != 0) {
|
|
dprintf(V_ERR, ("%s: unaligned size %lu from %d\n",
|
|
ahci_portname(ps), size, endpt));
|
|
return EINVAL;
|
|
}
|
|
|
|
/* Create a vector of physical addresses and sizes for the transfer. */
|
|
nr_prds = r = setup_prdt(ps, endpt, iovec, nr_req, size, lead, write,
|
|
prdt);
|
|
|
|
if (r < 0) return r;
|
|
|
|
/* Perform the actual transfer. */
|
|
cmd = port_find_cmd(ps);
|
|
|
|
if (ps->flags & FLAG_ATAPI)
|
|
r = atapi_transfer(ps, cmd, start_lba, count, write, prdt,
|
|
nr_prds);
|
|
else
|
|
r = ata_transfer(ps, cmd, start_lba, count, write,
|
|
!!(flags & BDEV_FORCEWRITE), prdt, nr_prds);
|
|
|
|
if (r != OK) return r;
|
|
|
|
return size;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_hardreset *
|
|
*===========================================================================*/
|
|
static void port_hardreset(struct port_state *ps)
|
|
{
|
|
/* Perform a port-level (hard) reset on the given port.
|
|
*/
|
|
|
|
port_write(ps, AHCI_PORT_SCTL, AHCI_PORT_SCTL_DET_INIT);
|
|
|
|
micro_delay(COMRESET_DELAY * 1000); /* COMRESET_DELAY is in ms */
|
|
|
|
port_write(ps, AHCI_PORT_SCTL, AHCI_PORT_SCTL_DET_NONE);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_override *
|
|
*===========================================================================*/
|
|
static void port_override(struct port_state *ps)
|
|
{
|
|
/* Override the port's BSY and/or DRQ flags. This may only be done
|
|
* prior to starting the port.
|
|
*/
|
|
u32_t cmd;
|
|
|
|
cmd = port_read(ps, AHCI_PORT_CMD);
|
|
port_write(ps, AHCI_PORT_CMD, cmd | AHCI_PORT_CMD_CLO);
|
|
|
|
SPIN_UNTIL(!(port_read(ps, AHCI_PORT_CMD) & AHCI_PORT_CMD_CLO),
|
|
PORTREG_DELAY);
|
|
|
|
dprintf(V_INFO, ("%s: overridden\n", ahci_portname(ps)));
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_start *
|
|
*===========================================================================*/
|
|
static void port_start(struct port_state *ps)
|
|
{
|
|
/* Start the given port, allowing for the execution of commands and the
|
|
* transfer of data on that port.
|
|
*/
|
|
u32_t cmd;
|
|
|
|
/* Reset status registers. */
|
|
port_write(ps, AHCI_PORT_SERR, ~0);
|
|
port_write(ps, AHCI_PORT_IS, ~0);
|
|
|
|
/* Start the port. */
|
|
cmd = port_read(ps, AHCI_PORT_CMD);
|
|
port_write(ps, AHCI_PORT_CMD, cmd | AHCI_PORT_CMD_ST);
|
|
|
|
dprintf(V_INFO, ("%s: started\n", ahci_portname(ps)));
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_stop *
|
|
*===========================================================================*/
|
|
static void port_stop(struct port_state *ps)
|
|
{
|
|
/* Stop the given port, if not already stopped.
|
|
*/
|
|
u32_t cmd;
|
|
|
|
cmd = port_read(ps, AHCI_PORT_CMD);
|
|
|
|
if (cmd & (AHCI_PORT_CMD_CR | AHCI_PORT_CMD_ST)) {
|
|
port_write(ps, AHCI_PORT_CMD, cmd & ~AHCI_PORT_CMD_ST);
|
|
|
|
SPIN_UNTIL(!(port_read(ps, AHCI_PORT_CMD) & AHCI_PORT_CMD_CR),
|
|
PORTREG_DELAY);
|
|
|
|
dprintf(V_INFO, ("%s: stopped\n", ahci_portname(ps)));
|
|
}
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_restart *
|
|
*===========================================================================*/
|
|
static void port_restart(struct port_state *ps)
|
|
{
|
|
/* Restart a port after a fatal error has occurred.
|
|
*/
|
|
|
|
/* Fail all outstanding commands. */
|
|
port_fail_cmds(ps);
|
|
|
|
/* Stop the port. */
|
|
port_stop(ps);
|
|
|
|
/* If the BSY and/or DRQ flags are set, reset the port. */
|
|
if (port_read(ps, AHCI_PORT_TFD) &
|
|
(AHCI_PORT_TFD_STS_BSY | AHCI_PORT_TFD_STS_DRQ)) {
|
|
|
|
dprintf(V_ERR, ("%s: port reset\n", ahci_portname(ps)));
|
|
|
|
/* To keep this driver simple, we do not transparently recover
|
|
* ongoing requests. Instead, we mark the failing device as
|
|
* disconnected, and reset it. If the reset succeeds, the
|
|
* device (or, perhaps, eventually, another device) will come
|
|
* back up. Any current and future requests to this port will
|
|
* be failed until the port is fully closed and reopened.
|
|
*/
|
|
port_disconnect(ps);
|
|
|
|
/* Trigger a port reset. */
|
|
port_hardreset(ps);
|
|
|
|
return;
|
|
}
|
|
|
|
/* Start the port. */
|
|
port_start(ps);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* print_string *
|
|
*===========================================================================*/
|
|
static void print_string(u16_t *buf, int start, int end)
|
|
{
|
|
/* Print a string that is stored as little-endian words and padded with
|
|
* trailing spaces.
|
|
*/
|
|
int i, last = 0;
|
|
|
|
while (end >= start && buf[end] == 0x2020) end--;
|
|
|
|
if (end >= start && (buf[end] & 0xFF) == 0x20) end--, last++;
|
|
|
|
for (i = start; i <= end; i++)
|
|
printf("%c%c", buf[i] >> 8, buf[i] & 0xFF);
|
|
|
|
if (last)
|
|
printf("%c", buf[i] >> 8);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_id_check *
|
|
*===========================================================================*/
|
|
static void port_id_check(struct port_state *ps, int success)
|
|
{
|
|
/* The device identification command has either completed or timed out.
|
|
* Decide whether this device is usable or not, and store some of its
|
|
* properties.
|
|
*/
|
|
u16_t *buf;
|
|
|
|
assert(ps->state == STATE_WAIT_ID);
|
|
|
|
ps->flags &= ~FLAG_BUSY;
|
|
cancel_timer(&ps->cmd_info[0].timer);
|
|
|
|
if (!success) {
|
|
if (!(ps->flags & FLAG_ATAPI) &&
|
|
port_read(ps, AHCI_PORT_SIG) != ATA_SIG_ATA) {
|
|
dprintf(V_INFO, ("%s: may not be ATA, trying ATAPI\n",
|
|
ahci_portname(ps)));
|
|
|
|
ps->flags |= FLAG_ATAPI;
|
|
|
|
(void) gen_identify(ps, FALSE /*blocking*/);
|
|
return;
|
|
}
|
|
|
|
dprintf(V_ERR,
|
|
("%s: unable to identify\n", ahci_portname(ps)));
|
|
}
|
|
|
|
/* If the identify command itself succeeded, check the results and
|
|
* store some properties.
|
|
*/
|
|
if (success) {
|
|
buf = (u16_t *) ps->tmp_base;
|
|
|
|
if (ps->flags & FLAG_ATAPI)
|
|
success = atapi_id_check(ps, buf);
|
|
else
|
|
success = ata_id_check(ps, buf);
|
|
}
|
|
|
|
/* If the device has not been identified successfully, mark it as an
|
|
* unusable device.
|
|
*/
|
|
if (!success) {
|
|
port_stop(ps);
|
|
|
|
ps->state = STATE_BAD_DEV;
|
|
port_write(ps, AHCI_PORT_IE, AHCI_PORT_IE_PRCE);
|
|
|
|
return;
|
|
}
|
|
|
|
/* The device has been identified successfully, and hence usable. */
|
|
ps->state = STATE_GOOD_DEV;
|
|
|
|
/* Print some information about the device. */
|
|
if (ahci_verbose >= V_INFO) {
|
|
printf("%s: ATA%s, ", ahci_portname(ps),
|
|
(ps->flags & FLAG_ATAPI) ? "PI" : "");
|
|
print_string(buf, 27, 46);
|
|
if (ahci_verbose >= V_DEV) {
|
|
printf(" (");
|
|
print_string(buf, 10, 19);
|
|
printf(", ");
|
|
print_string(buf, 23, 26);
|
|
printf(")");
|
|
}
|
|
|
|
if (ps->flags & FLAG_HAS_MEDIUM)
|
|
printf(", %u byte sectors, %lu MB size",
|
|
ps->sector_size, div64u(mul64(ps->lba_count,
|
|
cvu64(ps->sector_size)), 1024*1024));
|
|
|
|
printf("\n");
|
|
}
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_connect *
|
|
*===========================================================================*/
|
|
static void port_connect(struct port_state *ps)
|
|
{
|
|
/* A device has been found to be attached to this port. Start the port,
|
|
* and do timed polling for its signature to become available.
|
|
*/
|
|
u32_t status, sig;
|
|
|
|
dprintf(V_INFO, ("%s: device connected\n", ahci_portname(ps)));
|
|
|
|
port_start(ps);
|
|
|
|
/* The next check covers a purely hypothetical race condition, where
|
|
* the device would disappear right before we try to start it. This is
|
|
* possible because we have to clear PxSERR, and with that, the DIAG.N
|
|
* bit. Double-check the port status, and if it is not as we expect,
|
|
* infer a disconnection.
|
|
*/
|
|
status = port_read(ps, AHCI_PORT_SSTS) & AHCI_PORT_SSTS_DET_MASK;
|
|
|
|
if (status != AHCI_PORT_SSTS_DET_PHY) {
|
|
dprintf(V_ERR, ("%s: device vanished!\n", ahci_portname(ps)));
|
|
|
|
port_stop(ps);
|
|
|
|
ps->state = STATE_NO_DEV;
|
|
ps->flags &= ~FLAG_BUSY;
|
|
|
|
return;
|
|
}
|
|
|
|
/* Clear all state flags except the busy flag, which may be relevant if
|
|
* a BDEV_OPEN call is waiting for the device to become ready; the
|
|
* barrier flag, which prevents access to the device until it is
|
|
* completely closed and (re)opened; and, the thread suspension flag.
|
|
*/
|
|
ps->flags &= (FLAG_BUSY | FLAG_BARRIER | FLAG_SUSPENDED);
|
|
|
|
/* Check the port's signature. We only use the signature to speed up
|
|
* identification; we will try both ATA and ATAPI if the signature is
|
|
* neither ATA nor ATAPI.
|
|
*/
|
|
sig = port_read(ps, AHCI_PORT_SIG);
|
|
|
|
if (sig == ATA_SIG_ATAPI)
|
|
ps->flags |= FLAG_ATAPI;
|
|
|
|
/* Attempt to identify the device. Do this using continuation, because
|
|
* we may already be called from port_wait() here, and could end up
|
|
* confusing the timer expiration procedure.
|
|
*/
|
|
ps->state = STATE_WAIT_ID;
|
|
port_write(ps, AHCI_PORT_IE, AHCI_PORT_IE_MASK);
|
|
|
|
(void) gen_identify(ps, FALSE /*blocking*/);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_disconnect *
|
|
*===========================================================================*/
|
|
static void port_disconnect(struct port_state *ps)
|
|
{
|
|
/* The device has detached from this port. It has already been stopped.
|
|
*/
|
|
|
|
dprintf(V_INFO, ("%s: device disconnected\n", ahci_portname(ps)));
|
|
|
|
ps->state = STATE_NO_DEV;
|
|
port_write(ps, AHCI_PORT_IE, AHCI_PORT_IE_PCE);
|
|
ps->flags &= ~FLAG_BUSY;
|
|
|
|
/* Fail any ongoing request. The caller may already have done this. */
|
|
port_fail_cmds(ps);
|
|
|
|
/* Block any further access until the device is completely closed and
|
|
* reopened. This prevents arbitrary I/O to a newly plugged-in device
|
|
* without upper layers noticing.
|
|
*/
|
|
ps->flags |= FLAG_BARRIER;
|
|
|
|
/* Inform the blockdriver library to reduce the number of threads. */
|
|
blockdriver_mt_set_workers(ps->device, 1);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_dev_check *
|
|
*===========================================================================*/
|
|
static void port_dev_check(struct port_state *ps)
|
|
{
|
|
/* Perform device detection by means of polling.
|
|
*/
|
|
u32_t status, tfd;
|
|
|
|
assert(ps->state == STATE_WAIT_DEV);
|
|
|
|
status = port_read(ps, AHCI_PORT_SSTS) & AHCI_PORT_SSTS_DET_MASK;
|
|
|
|
dprintf(V_DEV, ("%s: polled status %u\n", ahci_portname(ps), status));
|
|
|
|
switch (status) {
|
|
case AHCI_PORT_SSTS_DET_PHY:
|
|
tfd = port_read(ps, AHCI_PORT_TFD);
|
|
|
|
/* If a Phy connection has been established, and the BSY and
|
|
* DRQ flags are cleared, the device is ready.
|
|
*/
|
|
if (!(tfd & (AHCI_PORT_TFD_STS_BSY | AHCI_PORT_TFD_STS_DRQ))) {
|
|
port_connect(ps);
|
|
|
|
return;
|
|
}
|
|
|
|
/* fall-through */
|
|
case AHCI_PORT_SSTS_DET_DET:
|
|
/* A device has been detected, but it is not ready yet. Try for
|
|
* a while before giving up. This may take seconds.
|
|
*/
|
|
if (ps->left > 0) {
|
|
ps->left--;
|
|
set_timer(&ps->cmd_info[0].timer, ahci_device_delay,
|
|
port_timeout, BUILD_ARG(ps - port_state, 0));
|
|
return;
|
|
}
|
|
}
|
|
|
|
dprintf(V_INFO, ("%s: device not ready\n", ahci_portname(ps)));
|
|
|
|
/* We get here on timeout, and if the HBA reports that there is no
|
|
* device present at all. In all cases, we change to another state.
|
|
*/
|
|
if (status == AHCI_PORT_SSTS_DET_PHY) {
|
|
/* Some devices may not correctly clear BSY/DRQ. Upon timeout,
|
|
* if we can override these flags, do so and start the
|
|
* identification process anyway.
|
|
*/
|
|
if (hba_state.has_clo) {
|
|
port_override(ps);
|
|
|
|
port_connect(ps);
|
|
|
|
return;
|
|
}
|
|
|
|
/* A device is present and initialized, but not ready. */
|
|
ps->state = STATE_BAD_DEV;
|
|
port_write(ps, AHCI_PORT_IE, AHCI_PORT_IE_PRCE);
|
|
} else {
|
|
/* A device may or may not be present, but it does not appear
|
|
* to be ready in any case. Ignore it until the next device
|
|
* initialization event.
|
|
*/
|
|
ps->state = STATE_NO_DEV;
|
|
ps->flags &= ~FLAG_BUSY;
|
|
}
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_intr *
|
|
*===========================================================================*/
|
|
static void port_intr(struct port_state *ps)
|
|
{
|
|
/* Process an interrupt on this port.
|
|
*/
|
|
u32_t smask, emask;
|
|
int success;
|
|
|
|
if (ps->state == STATE_NO_PORT) {
|
|
dprintf(V_ERR, ("%s: interrupt for invalid port!\n",
|
|
ahci_portname(ps)));
|
|
|
|
return;
|
|
}
|
|
|
|
smask = port_read(ps, AHCI_PORT_IS);
|
|
emask = smask & port_read(ps, AHCI_PORT_IE);
|
|
|
|
/* Clear the interrupt flags that we saw were set. */
|
|
port_write(ps, AHCI_PORT_IS, smask);
|
|
|
|
dprintf(V_REQ, ("%s: interrupt (%08x)\n", ahci_portname(ps), smask));
|
|
|
|
/* Check if any commands have completed. */
|
|
port_check_cmds(ps);
|
|
|
|
if (emask & AHCI_PORT_IS_PCS) {
|
|
/* Clear the X diagnostics bit to clear this interrupt. */
|
|
port_write(ps, AHCI_PORT_SERR, AHCI_PORT_SERR_DIAG_X);
|
|
|
|
dprintf(V_DEV, ("%s: device attached\n", ahci_portname(ps)));
|
|
|
|
switch (ps->state) {
|
|
case STATE_SPIN_UP:
|
|
case STATE_NO_DEV:
|
|
/* Reportedly, a device has shown up. Start polling its
|
|
* status until it has become ready.
|
|
*/
|
|
|
|
if (ps->state == STATE_SPIN_UP)
|
|
cancel_timer(&ps->cmd_info[0].timer);
|
|
|
|
ps->state = STATE_WAIT_DEV;
|
|
ps->left = ahci_device_checks;
|
|
|
|
port_dev_check(ps);
|
|
|
|
break;
|
|
|
|
case STATE_WAIT_DEV:
|
|
/* Nothing else to do. */
|
|
break;
|
|
|
|
default:
|
|
/* Impossible. */
|
|
assert(0);
|
|
}
|
|
} else if (emask & AHCI_PORT_IS_PRCS) {
|
|
/* Clear the N diagnostics bit to clear this interrupt. */
|
|
port_write(ps, AHCI_PORT_SERR, AHCI_PORT_SERR_DIAG_N);
|
|
|
|
dprintf(V_DEV, ("%s: device detached\n", ahci_portname(ps)));
|
|
|
|
switch (ps->state) {
|
|
case STATE_WAIT_ID:
|
|
case STATE_GOOD_DEV:
|
|
/* The device is no longer ready. Stop the port, cancel
|
|
* ongoing requests, and disconnect the device.
|
|
*/
|
|
port_stop(ps);
|
|
|
|
/* fall-through */
|
|
case STATE_BAD_DEV:
|
|
port_disconnect(ps);
|
|
|
|
/* The device has become unusable to us at this point.
|
|
* Reset the port to make sure that once the device (or
|
|
* another device) becomes usable again, we will get a
|
|
* PCS interrupt as well.
|
|
*/
|
|
port_hardreset(ps);
|
|
|
|
break;
|
|
|
|
default:
|
|
/* Impossible. */
|
|
assert(0);
|
|
}
|
|
} else if (smask & AHCI_PORT_IS_MASK) {
|
|
/* We assume that any other interrupt indicates command
|
|
* completion or (command or device) failure. Unfortunately, if
|
|
* an NCQ command failed, we cannot easily determine which one
|
|
* it was. For that reason, after completing all successfully
|
|
* finished commands (above), we fail all other outstanding
|
|
* commands and restart the port. This can possibly be improved
|
|
* later by obtaining per-command status results from the HBA.
|
|
*/
|
|
|
|
success = !(port_read(ps, AHCI_PORT_TFD) &
|
|
(AHCI_PORT_TFD_STS_ERR | AHCI_PORT_TFD_STS_DF));
|
|
|
|
/* Check now for failure. There are fatal failures, and there
|
|
* are failures that set the TFD.STS.ERR field using a D2H
|
|
* FIS. In both cases, we just restart the port, failing all
|
|
* commands in the process.
|
|
*/
|
|
if ((port_read(ps, AHCI_PORT_TFD) &
|
|
(AHCI_PORT_TFD_STS_ERR | AHCI_PORT_TFD_STS_DF)) ||
|
|
(smask & AHCI_PORT_IS_RESTART)) {
|
|
port_restart(ps);
|
|
}
|
|
|
|
/* If we were waiting for ID verification, check now. */
|
|
if (ps->state == STATE_WAIT_ID)
|
|
port_id_check(ps, success);
|
|
}
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_timeout *
|
|
*===========================================================================*/
|
|
static void port_timeout(struct timer *tp)
|
|
{
|
|
/* A timeout has occurred on this port. Figure out what the timeout is
|
|
* for, and take appropriate action.
|
|
*/
|
|
struct port_state *ps;
|
|
int port, cmd;
|
|
|
|
port = GET_PORT(tmr_arg(tp)->ta_int);
|
|
cmd = GET_TAG(tmr_arg(tp)->ta_int);
|
|
|
|
assert(port >= 0 && port < hba_state.nr_ports);
|
|
|
|
ps = &port_state[port];
|
|
|
|
/* Regardless of the outcome of this timeout, wake up the thread if it
|
|
* is suspended. This applies only during the initialization.
|
|
*/
|
|
if (ps->flags & FLAG_SUSPENDED) {
|
|
assert(cmd == 0);
|
|
blockdriver_mt_wakeup(ps->cmd_info[0].tid);
|
|
}
|
|
|
|
/* If detection of a device after startup timed out, give up on initial
|
|
* detection and only look for hot plug events from now on.
|
|
*/
|
|
if (ps->state == STATE_SPIN_UP) {
|
|
/* One exception: if the PCS interrupt bit is set here, then we
|
|
* are probably running on VirtualBox, which is currently not
|
|
* always raising interrupts when setting interrupt bits (!).
|
|
*/
|
|
if (port_read(ps, AHCI_PORT_IS) & AHCI_PORT_IS_PCS) {
|
|
dprintf(V_INFO, ("%s: bad controller, no interrupt\n",
|
|
ahci_portname(ps)));
|
|
|
|
ps->state = STATE_WAIT_DEV;
|
|
ps->left = ahci_device_checks;
|
|
|
|
port_dev_check(ps);
|
|
|
|
return;
|
|
} else {
|
|
dprintf(V_INFO, ("%s: spin-up timeout\n",
|
|
ahci_portname(ps)));
|
|
|
|
/* If the busy flag is set, a BDEV_OPEN request is
|
|
* waiting for the detection to finish; clear the busy
|
|
* flag to return an error to the caller.
|
|
*/
|
|
ps->state = STATE_NO_DEV;
|
|
ps->flags &= ~FLAG_BUSY;
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
/* If we are waiting for a device to become connected and initialized,
|
|
* check now.
|
|
*/
|
|
if (ps->state == STATE_WAIT_DEV) {
|
|
port_dev_check(ps);
|
|
|
|
return;
|
|
}
|
|
|
|
dprintf(V_ERR, ("%s: timeout\n", ahci_portname(ps)));
|
|
|
|
/* Restart the port, failing all current commands. */
|
|
port_restart(ps);
|
|
|
|
/* Finish up the identify operation. */
|
|
if (ps->state == STATE_WAIT_ID)
|
|
port_id_check(ps, FALSE);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_wait *
|
|
*===========================================================================*/
|
|
static void port_wait(struct port_state *ps)
|
|
{
|
|
/* Suspend the current thread until the given port is no longer busy,
|
|
* due to either command completion or timeout.
|
|
*/
|
|
|
|
ps->flags |= FLAG_SUSPENDED;
|
|
|
|
while (ps->flags & FLAG_BUSY)
|
|
blockdriver_mt_sleep();
|
|
|
|
ps->flags &= ~FLAG_SUSPENDED;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_issue *
|
|
*===========================================================================*/
|
|
static void port_issue(struct port_state *ps, int cmd, clock_t timeout)
|
|
{
|
|
/* Issue a command to the port, and set a timer to trigger a timeout
|
|
* if the command takes too long to complete.
|
|
*/
|
|
|
|
/* Set the corresponding NCQ command bit, if applicable. */
|
|
if (ps->flags & FLAG_HAS_NCQ)
|
|
port_write(ps, AHCI_PORT_SACT, 1 << cmd);
|
|
|
|
/* Make sure that the compiler does not delay any previous write
|
|
* operations until after the write to the command issue register.
|
|
*/
|
|
__insn_barrier();
|
|
|
|
/* Tell the controller that a new command is ready. */
|
|
port_write(ps, AHCI_PORT_CI, 1 << cmd);
|
|
|
|
/* Update pending commands. */
|
|
ps->pend_mask |= 1 << cmd;
|
|
|
|
/* Set a timer in case the command does not complete at all. */
|
|
set_timer(&ps->cmd_info[cmd].timer, timeout, port_timeout,
|
|
BUILD_ARG(ps - port_state, cmd));
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_exec *
|
|
*===========================================================================*/
|
|
static int port_exec(struct port_state *ps, int cmd, clock_t timeout)
|
|
{
|
|
/* Execute a command on a port, wait for the command to complete or for
|
|
* a timeout, and return whether the command succeeded or not.
|
|
*/
|
|
|
|
port_issue(ps, cmd, timeout);
|
|
|
|
/* Put the thread to sleep until a timeout or a command completion
|
|
* happens. Earlier, we used to call port_wait which set the suspended
|
|
* flag. We now abandon it since the flag has to work on a per-thread,
|
|
* and hence per-tag basis and not on a per-port basis. Instead, we
|
|
* retain that call only to defer open calls during device/driver
|
|
* initialization. Instead, we call sleep here directly. Before
|
|
* sleeping, we register the thread.
|
|
*/
|
|
ps->cmd_info[cmd].tid = blockdriver_mt_get_tid();
|
|
|
|
blockdriver_mt_sleep();
|
|
|
|
/* Cancelling a timer that just triggered, does no harm. */
|
|
cancel_timer(&ps->cmd_info[cmd].timer);
|
|
|
|
assert(!(ps->flags & FLAG_BUSY));
|
|
|
|
dprintf(V_REQ, ("%s: end of command -- %s\n", ahci_portname(ps),
|
|
(ps->cmd_info[cmd].result == RESULT_FAILURE) ?
|
|
"failure" : "success"));
|
|
|
|
if (ps->cmd_info[cmd].result == RESULT_FAILURE)
|
|
return EIO;
|
|
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_alloc *
|
|
*===========================================================================*/
|
|
static void port_alloc(struct port_state *ps)
|
|
{
|
|
/* Allocate memory for the given port, and enable FIS receipt. We try
|
|
* to cram everything into one 4K-page in order to limit memory usage
|
|
* as much as possible. More memory may be allocated on demand later,
|
|
* but allocation failure should be fatal only here. Note that we do
|
|
* not allocate memory for sector padding here, because we do not know
|
|
* the device's sector size yet.
|
|
*/
|
|
size_t fis_off, tmp_off, ct_off; int i;
|
|
size_t ct_offs[NR_CMDS];
|
|
u32_t cmd;
|
|
|
|
fis_off = AHCI_CL_SIZE + AHCI_FIS_SIZE - 1;
|
|
fis_off -= fis_off % AHCI_FIS_SIZE;
|
|
|
|
tmp_off = fis_off + AHCI_FIS_SIZE + AHCI_TMP_ALIGN - 1;
|
|
tmp_off -= tmp_off % AHCI_TMP_ALIGN;
|
|
|
|
/* Allocate memory for all the commands. */
|
|
ct_off = tmp_off + AHCI_TMP_SIZE;
|
|
for (i = 0; i < NR_CMDS; i++) {
|
|
ct_off += AHCI_CT_ALIGN - 1;
|
|
ct_off -= ct_off % AHCI_CT_ALIGN;
|
|
ct_offs[i] = ct_off;
|
|
ps->mem_size = ct_off + AHCI_CT_SIZE;
|
|
ct_off = ps->mem_size;
|
|
}
|
|
|
|
ps->mem_base = alloc_contig(ps->mem_size, AC_ALIGN4K, &ps->mem_phys);
|
|
if (ps->mem_base == NULL)
|
|
panic("unable to allocate port memory");
|
|
memset(ps->mem_base, 0, ps->mem_size);
|
|
|
|
ps->cl_base = (u32_t *) ps->mem_base;
|
|
ps->cl_phys = ps->mem_phys;
|
|
assert(ps->cl_phys % AHCI_CL_SIZE == 0);
|
|
|
|
ps->fis_base = (u32_t *) (ps->mem_base + fis_off);
|
|
ps->fis_phys = ps->mem_phys + fis_off;
|
|
assert(ps->fis_phys % AHCI_FIS_SIZE == 0);
|
|
|
|
ps->tmp_base = (u8_t *) (ps->mem_base + tmp_off);
|
|
ps->tmp_phys = ps->mem_phys + tmp_off;
|
|
assert(ps->tmp_phys % AHCI_TMP_ALIGN == 0);
|
|
|
|
for (i = 0; i < NR_CMDS; i++) {
|
|
ps->ct_base[i] = ps->mem_base + ct_offs[i];
|
|
ps->ct_phys[i] = ps->mem_phys + ct_offs[i];
|
|
assert(ps->ct_phys[i] % AHCI_CT_ALIGN == 0);
|
|
}
|
|
|
|
/* Tell the controller about some of the physical addresses. */
|
|
port_write(ps, AHCI_PORT_FBU, 0);
|
|
port_write(ps, AHCI_PORT_FB, ps->fis_phys);
|
|
|
|
port_write(ps, AHCI_PORT_CLBU, 0);
|
|
port_write(ps, AHCI_PORT_CLB, ps->cl_phys);
|
|
|
|
/* Enable FIS receive. */
|
|
cmd = port_read(ps, AHCI_PORT_CMD);
|
|
port_write(ps, AHCI_PORT_CMD, cmd | AHCI_PORT_CMD_FRE);
|
|
|
|
ps->pad_base = NULL;
|
|
ps->pad_size = 0;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_free *
|
|
*===========================================================================*/
|
|
static void port_free(struct port_state *ps)
|
|
{
|
|
/* Disable FIS receipt for the given port, and free previously
|
|
* allocated memory.
|
|
*/
|
|
u32_t cmd;
|
|
int i;
|
|
|
|
/* Disable FIS receive. */
|
|
cmd = port_read(ps, AHCI_PORT_CMD);
|
|
|
|
if (cmd & (AHCI_PORT_CMD_FR | AHCI_PORT_CMD_FRE)) {
|
|
port_write(ps, AHCI_PORT_CMD, cmd & ~AHCI_PORT_CMD_FRE);
|
|
|
|
SPIN_UNTIL(!(port_read(ps, AHCI_PORT_CMD) & AHCI_PORT_CMD_FR),
|
|
PORTREG_DELAY);
|
|
}
|
|
|
|
if (ps->pad_base != NULL)
|
|
free_contig(ps->pad_base, ps->pad_size);
|
|
|
|
/* The first command table is part of the primary memory page. */
|
|
for (i = 1; i < hba_state.nr_cmds; i++)
|
|
if (ps->ct_base[i] != NULL)
|
|
free_contig(ps->ct_base[i], AHCI_CT_SIZE);
|
|
|
|
free_contig(ps->mem_base, ps->mem_size);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* port_init *
|
|
*===========================================================================*/
|
|
static void port_init(struct port_state *ps)
|
|
{
|
|
/* Initialize the given port.
|
|
*/
|
|
u32_t cmd;
|
|
int i;
|
|
|
|
/* Initialize the port state structure. */
|
|
ps->queue_depth = 1;
|
|
ps->state = STATE_SPIN_UP;
|
|
ps->flags = FLAG_BUSY;
|
|
ps->sector_size = 0;
|
|
ps->open_count = 0;
|
|
ps->pend_mask = 0;
|
|
for (i = 0; i < NR_CMDS; i++)
|
|
init_timer(&ps->cmd_info[i].timer);
|
|
|
|
ps->reg = (u32_t *) ((u8_t *) hba_state.base +
|
|
AHCI_MEM_BASE_SIZE + AHCI_MEM_PORT_SIZE * (ps - port_state));
|
|
|
|
/* Allocate memory for the port. */
|
|
port_alloc(ps);
|
|
|
|
/* Just listen for device connection events for now. */
|
|
port_write(ps, AHCI_PORT_IE, AHCI_PORT_IE_PCE);
|
|
|
|
/* Enable device spin-up for HBAs that support staggered spin-up.
|
|
* This is a no-op for HBAs that do not support it.
|
|
*/
|
|
cmd = port_read(ps, AHCI_PORT_CMD);
|
|
port_write(ps, AHCI_PORT_CMD, cmd | AHCI_PORT_CMD_SUD);
|
|
|
|
/* Trigger a port reset. */
|
|
port_hardreset(ps);
|
|
|
|
set_timer(&ps->cmd_info[0].timer, ahci_spinup_timeout,
|
|
port_timeout, BUILD_ARG(ps - port_state, 0));
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_probe *
|
|
*===========================================================================*/
|
|
static int ahci_probe(int skip)
|
|
{
|
|
/* Find a matching PCI device.
|
|
*/
|
|
int r, devind;
|
|
u16_t vid, did;
|
|
|
|
pci_init();
|
|
|
|
r = pci_first_dev(&devind, &vid, &did);
|
|
if (r <= 0)
|
|
return -1;
|
|
|
|
while (skip--) {
|
|
r = pci_next_dev(&devind, &vid, &did);
|
|
if (r <= 0)
|
|
return -1;
|
|
}
|
|
|
|
pci_reserve(devind);
|
|
|
|
return devind;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_reset *
|
|
*===========================================================================*/
|
|
static void ahci_reset(void)
|
|
{
|
|
/* Reset the HBA. Do not enable AHCI mode afterwards.
|
|
*/
|
|
u32_t ghc;
|
|
|
|
ghc = hba_read(AHCI_HBA_GHC);
|
|
|
|
hba_write(AHCI_HBA_GHC, ghc | AHCI_HBA_GHC_AE);
|
|
|
|
hba_write(AHCI_HBA_GHC, ghc | AHCI_HBA_GHC_AE | AHCI_HBA_GHC_HR);
|
|
|
|
SPIN_UNTIL(!(hba_read(AHCI_HBA_GHC) & AHCI_HBA_GHC_HR), RESET_DELAY);
|
|
|
|
if (hba_read(AHCI_HBA_GHC) & AHCI_HBA_GHC_HR)
|
|
panic("unable to reset HBA");
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_init *
|
|
*===========================================================================*/
|
|
static void ahci_init(int devind)
|
|
{
|
|
/* Initialize the device.
|
|
*/
|
|
u32_t base, size, cap, ghc, mask;
|
|
int r, port, ioflag;
|
|
|
|
if ((r = pci_get_bar(devind, PCI_BAR_6, &base, &size, &ioflag)) != OK)
|
|
panic("unable to retrieve BAR: %d", r);
|
|
|
|
if (ioflag)
|
|
panic("invalid BAR type");
|
|
|
|
/* There must be at least one port, and at most NR_PORTS ports. Limit
|
|
* the actual total number of ports to the size of the exposed area.
|
|
*/
|
|
if (size < AHCI_MEM_BASE_SIZE + AHCI_MEM_PORT_SIZE)
|
|
panic("HBA memory size too small: %u", size);
|
|
|
|
size = MIN(size, AHCI_MEM_BASE_SIZE + AHCI_MEM_PORT_SIZE * NR_PORTS);
|
|
|
|
hba_state.nr_ports = (size - AHCI_MEM_BASE_SIZE) / AHCI_MEM_PORT_SIZE;
|
|
|
|
/* Map the register area into local memory. */
|
|
hba_state.base = (u32_t *) vm_map_phys(SELF, (void *) base, size);
|
|
hba_state.size = size;
|
|
if (hba_state.base == MAP_FAILED)
|
|
panic("unable to map HBA memory");
|
|
|
|
/* Retrieve, allocate and enable the controller's IRQ. */
|
|
hba_state.irq = pci_attr_r8(devind, PCI_ILR);
|
|
hba_state.hook_id = 0;
|
|
|
|
if ((r = sys_irqsetpolicy(hba_state.irq, 0, &hba_state.hook_id)) != OK)
|
|
panic("unable to register IRQ: %d", r);
|
|
|
|
if ((r = sys_irqenable(&hba_state.hook_id)) != OK)
|
|
panic("unable to enable IRQ: %d", r);
|
|
|
|
/* Reset the HBA. */
|
|
ahci_reset();
|
|
|
|
/* Enable AHCI and interrupts. */
|
|
ghc = hba_read(AHCI_HBA_GHC);
|
|
hba_write(AHCI_HBA_GHC, ghc | AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE);
|
|
|
|
/* Limit the maximum number of commands to the controller's value. */
|
|
/* Note that we currently use only one command anyway. */
|
|
cap = hba_read(AHCI_HBA_CAP);
|
|
hba_state.has_ncq = !!(cap & AHCI_HBA_CAP_SNCQ);
|
|
hba_state.has_clo = !!(cap & AHCI_HBA_CAP_SCLO);
|
|
hba_state.nr_cmds = MIN(NR_CMDS,
|
|
((cap >> AHCI_HBA_CAP_NCS_SHIFT) & AHCI_HBA_CAP_NCS_MASK) + 1);
|
|
|
|
dprintf(V_INFO, ("AHCI%u: HBA v%d.%d%d, %ld ports, %ld commands, "
|
|
"%s queuing, IRQ %d\n",
|
|
ahci_instance,
|
|
(int) (hba_read(AHCI_HBA_VS) >> 16),
|
|
(int) ((hba_read(AHCI_HBA_VS) >> 8) & 0xFF),
|
|
(int) (hba_read(AHCI_HBA_VS) & 0xFF),
|
|
((cap >> AHCI_HBA_CAP_NP_SHIFT) & AHCI_HBA_CAP_NP_MASK) + 1,
|
|
((cap >> AHCI_HBA_CAP_NCS_SHIFT) & AHCI_HBA_CAP_NCS_MASK) + 1,
|
|
hba_state.has_ncq ? "supports" : "no", hba_state.irq));
|
|
|
|
dprintf(V_INFO, ("AHCI%u: CAP %08x, CAP2 %08x, PI %08x\n",
|
|
ahci_instance, cap, hba_read(AHCI_HBA_CAP2),
|
|
hba_read(AHCI_HBA_PI)));
|
|
|
|
/* Initialize each of the implemented ports. We ignore CAP.NP. */
|
|
mask = hba_read(AHCI_HBA_PI);
|
|
|
|
for (port = 0; port < hba_state.nr_ports; port++) {
|
|
port_state[port].device = NO_DEVICE;
|
|
port_state[port].state = STATE_NO_PORT;
|
|
|
|
if (mask & (1 << port))
|
|
port_init(&port_state[port]);
|
|
}
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_stop *
|
|
*===========================================================================*/
|
|
static void ahci_stop(void)
|
|
{
|
|
/* Disable AHCI, and clean up resources to the extent possible.
|
|
*/
|
|
struct port_state *ps;
|
|
int r, port;
|
|
|
|
for (port = 0; port < hba_state.nr_ports; port++) {
|
|
ps = &port_state[port];
|
|
|
|
if (ps->state != STATE_NO_PORT) {
|
|
port_stop(ps);
|
|
|
|
port_free(ps);
|
|
}
|
|
}
|
|
|
|
ahci_reset();
|
|
|
|
if ((r = vm_unmap_phys(SELF, (void *) hba_state.base,
|
|
hba_state.size)) != OK)
|
|
panic("unable to unmap HBA memory: %d", r);
|
|
|
|
if ((r = sys_irqrmpolicy(&hba_state.hook_id)) != OK)
|
|
panic("unable to deregister IRQ: %d", r);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_alarm *
|
|
*===========================================================================*/
|
|
static void ahci_alarm(clock_t stamp)
|
|
{
|
|
/* Process an alarm.
|
|
*/
|
|
|
|
/* Call the port-specific handler for each port that timed out. */
|
|
expire_timers(stamp);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_intr *
|
|
*===========================================================================*/
|
|
static void ahci_intr(unsigned int UNUSED(mask))
|
|
{
|
|
/* Process an interrupt.
|
|
*/
|
|
struct port_state *ps;
|
|
u32_t mask;
|
|
int r, port;
|
|
|
|
/* Handle an interrupt for each port that has the interrupt bit set. */
|
|
mask = hba_read(AHCI_HBA_IS);
|
|
|
|
for (port = 0; port < hba_state.nr_ports; port++) {
|
|
if (mask & (1 << port)) {
|
|
ps = &port_state[port];
|
|
|
|
port_intr(ps);
|
|
|
|
/* After processing an interrupt, wake up the device
|
|
* thread if it is suspended and now no longer busy.
|
|
*/
|
|
if ((ps->flags & (FLAG_SUSPENDED | FLAG_BUSY)) ==
|
|
FLAG_SUSPENDED)
|
|
blockdriver_mt_wakeup(ps->cmd_info[0].tid);
|
|
}
|
|
}
|
|
|
|
/* Clear the bits that we processed. */
|
|
hba_write(AHCI_HBA_IS, mask);
|
|
|
|
/* Reenable the interrupt. */
|
|
if ((r = sys_irqenable(&hba_state.hook_id)) != OK)
|
|
panic("unable to enable IRQ: %d", r);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_get_params *
|
|
*===========================================================================*/
|
|
static void ahci_get_params(void)
|
|
{
|
|
/* Retrieve and parse parameters passed to this driver, except the
|
|
* device-to-port mapping, which has to be parsed later.
|
|
*/
|
|
long v;
|
|
int i;
|
|
|
|
/* Find out which driver instance we are. */
|
|
v = 0;
|
|
(void) env_parse("instance", "d", 0, &v, 0, 255);
|
|
ahci_instance = (int) v;
|
|
|
|
/* Initialize the verbosity level. */
|
|
v = V_ERR;
|
|
(void) env_parse("ahci_verbose", "d", 0, &v, V_NONE, V_REQ);
|
|
ahci_verbose = (int) v;
|
|
|
|
/* Initialize timeout-related values. */
|
|
for (i = 0; i < sizeof(ahci_timevar) / sizeof(ahci_timevar[0]); i++) {
|
|
v = ahci_timevar[i].default_ms;
|
|
|
|
(void) env_parse(ahci_timevar[i].name, "d", 0, &v, 1,
|
|
LONG_MAX);
|
|
|
|
*ahci_timevar[i].ptr = millis_to_hz(v);
|
|
}
|
|
|
|
ahci_device_delay = millis_to_hz(DEVICE_DELAY);
|
|
ahci_device_checks = (ahci_device_timeout + ahci_device_delay - 1) /
|
|
ahci_device_delay;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_set_mapping *
|
|
*===========================================================================*/
|
|
static void ahci_set_mapping(void)
|
|
{
|
|
/* Construct a mapping from device nodes to port numbers.
|
|
*/
|
|
char key[16], val[32], *p;
|
|
unsigned int port;
|
|
int i, j;
|
|
|
|
/* Start off with a mapping that includes implemented ports only, in
|
|
* order. We choose this mapping over an identity mapping to maximize
|
|
* the chance that the user will be able to access the first MAX_DRIVES
|
|
* devices. Note that we can only do this after initializing the HBA.
|
|
*/
|
|
for (i = j = 0; i < NR_PORTS && j < MAX_DRIVES; i++)
|
|
if (port_state[i].state != STATE_NO_PORT)
|
|
ahci_map[j++] = i;
|
|
|
|
for ( ; j < MAX_DRIVES; j++)
|
|
ahci_map[j] = NO_PORT;
|
|
|
|
/* See if the user specified a custom mapping. Unlike all other
|
|
* configuration options, this is a per-instance setting.
|
|
*/
|
|
strlcpy(key, "ahci0_map", sizeof(key));
|
|
key[4] += ahci_instance;
|
|
|
|
if (env_get_param(key, val, sizeof(val)) == OK) {
|
|
/* Parse the mapping, which is assumed to be a comma-separated
|
|
* list of zero-based port numbers.
|
|
*/
|
|
p = val;
|
|
|
|
for (i = 0; i < MAX_DRIVES; i++) {
|
|
if (*p) {
|
|
port = (unsigned int) strtoul(p, &p, 0);
|
|
|
|
if (*p) p++;
|
|
|
|
ahci_map[i] = port % NR_PORTS;
|
|
}
|
|
else ahci_map[i] = NO_PORT;
|
|
}
|
|
}
|
|
|
|
/* Create a reverse mapping. */
|
|
for (i = 0; i < MAX_DRIVES; i++)
|
|
if ((j = ahci_map[i]) != NO_PORT)
|
|
port_state[j].device = i;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* sef_cb_init_fresh *
|
|
*===========================================================================*/
|
|
static int sef_cb_init_fresh(int type, sef_init_info_t *UNUSED(info))
|
|
{
|
|
/* Initialize the driver.
|
|
*/
|
|
int devind;
|
|
|
|
/* Get command line parameters. */
|
|
ahci_get_params();
|
|
|
|
/* Probe for recognized devices, skipping matches as appropriate. */
|
|
devind = ahci_probe(ahci_instance);
|
|
|
|
if (devind < 0)
|
|
panic("no matching device found");
|
|
|
|
/* Initialize the device we found. */
|
|
ahci_init(devind);
|
|
|
|
/* Create a mapping from device nodes to port numbers. */
|
|
ahci_set_mapping();
|
|
|
|
/* Announce that we are up. */
|
|
blockdriver_announce(type);
|
|
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* sef_cb_signal_handler *
|
|
*===========================================================================*/
|
|
static void sef_cb_signal_handler(int signo)
|
|
{
|
|
/* In case of a termination signal, shut down this driver.
|
|
*/
|
|
int port;
|
|
|
|
if (signo != SIGTERM) return;
|
|
|
|
/* If any ports are still opened, assume that the system is being shut
|
|
* down, and stay up until the last device has been closed.
|
|
*/
|
|
ahci_exiting = TRUE;
|
|
|
|
for (port = 0; port < hba_state.nr_ports; port++)
|
|
if (port_state[port].open_count > 0)
|
|
return;
|
|
|
|
/* If not, stop the driver and exit immediately. */
|
|
ahci_stop();
|
|
|
|
exit(0);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* sef_local_startup *
|
|
*===========================================================================*/
|
|
static void sef_local_startup(void)
|
|
{
|
|
/* Set callbacks and initialize the System Event Framework (SEF).
|
|
*/
|
|
|
|
/* Register init callbacks. */
|
|
sef_setcb_init_fresh(sef_cb_init_fresh);
|
|
sef_setcb_init_lu(sef_cb_init_fresh);
|
|
|
|
/* Register signal callbacks. */
|
|
sef_setcb_signal_handler(sef_cb_signal_handler);
|
|
|
|
/* Let SEF perform startup. */
|
|
sef_startup();
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_portname *
|
|
*===========================================================================*/
|
|
static char *ahci_portname(struct port_state *ps)
|
|
{
|
|
/* Return a printable name for the given port. Whenever we can, print a
|
|
* "Dx" device number rather than a "Pxx" port number, because the user
|
|
* may not be aware of the mapping currently in use.
|
|
*/
|
|
static char name[] = "AHCI0-P00";
|
|
|
|
name[4] = '0' + ahci_instance;
|
|
|
|
if (ps->device == NO_DEVICE) {
|
|
name[6] = 'P';
|
|
name[7] = '0' + (ps - port_state) / 10;
|
|
name[8] = '0' + (ps - port_state) % 10;
|
|
}
|
|
else {
|
|
name[6] = 'D';
|
|
name[7] = '0' + ps->device;
|
|
name[8] = 0;
|
|
}
|
|
|
|
return name;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_map_minor *
|
|
*===========================================================================*/
|
|
static struct port_state *ahci_map_minor(dev_t minor, struct device **dvp)
|
|
{
|
|
/* Map a minor device number to a port and a pointer to the partition's
|
|
* device structure. Return NULL if this minor device number does not
|
|
* identify an actual device.
|
|
*/
|
|
struct port_state *ps;
|
|
int port;
|
|
|
|
ps = NULL;
|
|
|
|
if (minor < NR_MINORS) {
|
|
port = ahci_map[minor / DEV_PER_DRIVE];
|
|
|
|
if (port == NO_PORT)
|
|
return NULL;
|
|
|
|
ps = &port_state[port];
|
|
*dvp = &ps->part[minor % DEV_PER_DRIVE];
|
|
}
|
|
else if ((unsigned) (minor -= MINOR_d0p0s0) < NR_SUBDEVS) {
|
|
port = ahci_map[minor / SUB_PER_DRIVE];
|
|
|
|
if (port == NO_PORT)
|
|
return NULL;
|
|
|
|
ps = &port_state[port];
|
|
*dvp = &ps->subpart[minor % SUB_PER_DRIVE];
|
|
}
|
|
|
|
return ps;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_part *
|
|
*===========================================================================*/
|
|
static struct device *ahci_part(dev_t minor)
|
|
{
|
|
/* Return a pointer to the partition information structure of the given
|
|
* minor device.
|
|
*/
|
|
struct device *dv;
|
|
|
|
if (ahci_map_minor(minor, &dv) == NULL)
|
|
return NULL;
|
|
|
|
return dv;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_open *
|
|
*===========================================================================*/
|
|
static int ahci_open(dev_t minor, int access)
|
|
{
|
|
/* Open a device.
|
|
*/
|
|
struct port_state *ps;
|
|
int r;
|
|
|
|
ps = ahci_get_port(minor);
|
|
|
|
/* Only one open request can be processed at a time, due to the fact
|
|
* that it is an exclusive operation. The thread that handles this call
|
|
* can therefore freely register itself at slot zero.
|
|
*/
|
|
ps->cmd_info[0].tid = blockdriver_mt_get_tid();
|
|
|
|
/* If we are still in the process of initializing this port or device,
|
|
* wait for completion of that phase first.
|
|
*/
|
|
if (ps->flags & FLAG_BUSY)
|
|
port_wait(ps);
|
|
|
|
/* The device may only be opened if it is now properly functioning. */
|
|
if (ps->state != STATE_GOOD_DEV)
|
|
return ENXIO;
|
|
|
|
/* Some devices may only be opened in read-only mode. */
|
|
if ((ps->flags & FLAG_READONLY) && (access & W_BIT))
|
|
return EACCES;
|
|
|
|
if (ps->open_count == 0) {
|
|
/* The first open request. Clear the barrier flag, if set. */
|
|
ps->flags &= ~FLAG_BARRIER;
|
|
|
|
/* Recheck media only when nobody is using the device. */
|
|
if ((ps->flags & FLAG_ATAPI) &&
|
|
(r = atapi_check_medium(ps, 0)) != OK)
|
|
return r;
|
|
|
|
/* After rechecking the media, the partition table must always
|
|
* be read. This is also a convenient time to do it for
|
|
* nonremovable devices. Start by resetting the partition
|
|
* tables and setting the working size of the entire device.
|
|
*/
|
|
memset(ps->part, 0, sizeof(ps->part));
|
|
memset(ps->subpart, 0, sizeof(ps->subpart));
|
|
|
|
ps->part[0].dv_size =
|
|
mul64(ps->lba_count, cvu64(ps->sector_size));
|
|
|
|
partition(&ahci_dtab, ps->device * DEV_PER_DRIVE, P_PRIMARY,
|
|
!!(ps->flags & FLAG_ATAPI));
|
|
|
|
blockdriver_mt_set_workers(ps->device, ps->queue_depth);
|
|
}
|
|
else {
|
|
/* If the barrier flag is set, deny new open requests until the
|
|
* device is fully closed first.
|
|
*/
|
|
if (ps->flags & FLAG_BARRIER)
|
|
return ENXIO;
|
|
}
|
|
|
|
ps->open_count++;
|
|
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_close *
|
|
*===========================================================================*/
|
|
static int ahci_close(dev_t minor)
|
|
{
|
|
/* Close a device.
|
|
*/
|
|
struct port_state *ps;
|
|
int port;
|
|
|
|
ps = ahci_get_port(minor);
|
|
|
|
/* Decrease the open count. */
|
|
if (ps->open_count <= 0) {
|
|
dprintf(V_ERR, ("%s: closing already-closed port\n",
|
|
ahci_portname(ps)));
|
|
|
|
return EINVAL;
|
|
}
|
|
|
|
ps->open_count--;
|
|
|
|
if (ps->open_count > 0)
|
|
return OK;
|
|
|
|
/* The device is now fully closed. That also means that the threads for
|
|
* this device are not needed anymore, so we reduce the count to one.
|
|
*/
|
|
blockdriver_mt_set_workers(ps->device, 1);
|
|
|
|
if (ps->state == STATE_GOOD_DEV && !(ps->flags & FLAG_BARRIER)) {
|
|
dprintf(V_INFO, ("%s: flushing write cache\n",
|
|
ahci_portname(ps)));
|
|
|
|
(void) gen_flush_wcache(ps);
|
|
}
|
|
|
|
/* If the entire driver has been told to terminate, check whether all
|
|
* devices are now closed. If so, tell libblockdriver to quit after
|
|
* replying to the close request.
|
|
*/
|
|
if (ahci_exiting) {
|
|
for (port = 0; port < hba_state.nr_ports; port++)
|
|
if (port_state[port].open_count > 0)
|
|
break;
|
|
|
|
if (port == hba_state.nr_ports) {
|
|
ahci_stop();
|
|
|
|
blockdriver_mt_terminate();
|
|
}
|
|
}
|
|
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_transfer *
|
|
*===========================================================================*/
|
|
static ssize_t ahci_transfer(dev_t minor, int do_write, u64_t position,
|
|
endpoint_t endpt, iovec_t *iovec, unsigned int count, int flags)
|
|
{
|
|
/* Perform data transfer on the selected device.
|
|
*/
|
|
struct port_state *ps;
|
|
struct device *dv;
|
|
u64_t pos, eof;
|
|
|
|
ps = ahci_get_port(minor);
|
|
dv = ahci_part(minor);
|
|
|
|
if (ps->state != STATE_GOOD_DEV || (ps->flags & FLAG_BARRIER))
|
|
return EIO;
|
|
|
|
if (count > NR_IOREQS)
|
|
return EINVAL;
|
|
|
|
/* Check for basic end-of-partition condition: if the start position of
|
|
* the request is outside the partition, return success immediately.
|
|
* The size of the request is obtained, and possibly reduced, later.
|
|
*/
|
|
if (cmp64(position, dv->dv_size) >= 0)
|
|
return OK;
|
|
|
|
pos = add64(dv->dv_base, position);
|
|
eof = add64(dv->dv_base, dv->dv_size);
|
|
|
|
return port_transfer(ps, pos, eof, endpt, (iovec_s_t *) iovec, count,
|
|
do_write, flags);
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_ioctl *
|
|
*===========================================================================*/
|
|
static int ahci_ioctl(dev_t minor, unsigned int request, endpoint_t endpt,
|
|
cp_grant_id_t grant)
|
|
{
|
|
/* Process I/O control requests.
|
|
*/
|
|
struct port_state *ps;
|
|
int r, val;
|
|
|
|
ps = ahci_get_port(minor);
|
|
|
|
switch (request) {
|
|
case DIOCEJECT:
|
|
if (ps->state != STATE_GOOD_DEV || (ps->flags & FLAG_BARRIER))
|
|
return EIO;
|
|
|
|
if (!(ps->flags & FLAG_ATAPI))
|
|
return EINVAL;
|
|
|
|
return atapi_load_eject(ps, 0, FALSE /*load*/);
|
|
|
|
case DIOCOPENCT:
|
|
return sys_safecopyto(endpt, grant, 0,
|
|
(vir_bytes) &ps->open_count, sizeof(ps->open_count));
|
|
|
|
case DIOCFLUSH:
|
|
if (ps->state != STATE_GOOD_DEV || (ps->flags & FLAG_BARRIER))
|
|
return EIO;
|
|
|
|
return gen_flush_wcache(ps);
|
|
|
|
case DIOCSETWC:
|
|
if (ps->state != STATE_GOOD_DEV || (ps->flags & FLAG_BARRIER))
|
|
return EIO;
|
|
|
|
if ((r = sys_safecopyfrom(endpt, grant, 0, (vir_bytes) &val,
|
|
sizeof(val))) != OK)
|
|
return r;
|
|
|
|
return gen_set_wcache(ps, val);
|
|
|
|
case DIOCGETWC:
|
|
if (ps->state != STATE_GOOD_DEV || (ps->flags & FLAG_BARRIER))
|
|
return EIO;
|
|
|
|
if ((r = gen_get_wcache(ps, &val)) != OK)
|
|
return r;
|
|
|
|
return sys_safecopyto(endpt, grant, 0, (vir_bytes) &val,
|
|
sizeof(val));
|
|
}
|
|
|
|
return EINVAL;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_device *
|
|
*===========================================================================*/
|
|
static int ahci_device(dev_t minor, device_id_t *id)
|
|
{
|
|
/* Map a minor device number to a device ID.
|
|
*/
|
|
struct port_state *ps;
|
|
struct device *dv;
|
|
|
|
if ((ps = ahci_map_minor(minor, &dv)) == NULL)
|
|
return ENXIO;
|
|
|
|
*id = ps->device;
|
|
|
|
return OK;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* ahci_get_port *
|
|
*===========================================================================*/
|
|
static struct port_state *ahci_get_port(dev_t minor)
|
|
{
|
|
/* Get the port structure associated with the given minor device.
|
|
* Called only from worker threads, so the minor device is already
|
|
* guaranteed to map to a port.
|
|
*/
|
|
struct port_state *ps;
|
|
struct device *dv;
|
|
|
|
if ((ps = ahci_map_minor(minor, &dv)) == NULL)
|
|
panic("device mapping for minor %d disappeared", minor);
|
|
|
|
return ps;
|
|
}
|
|
|
|
/*===========================================================================*
|
|
* main *
|
|
*===========================================================================*/
|
|
int main(int argc, char **argv)
|
|
{
|
|
/* Driver task.
|
|
*/
|
|
|
|
env_setargs(argc, argv);
|
|
sef_local_startup();
|
|
|
|
blockdriver_mt_task(&ahci_dtab);
|
|
|
|
return 0;
|
|
}
|