| /* | |
| * Copyright 2017, Data61 | |
| * Commonwealth Scientific and Industrial Research Organisation (CSIRO) | |
| * ABN 41 687 119 230. | |
| * | |
| * This software may be distributed and modified according to the terms of | |
| * the BSD 2-Clause license. Note that NO WARRANTY is provided. | |
| * See "LICENSE_BSD2.txt" for details. | |
| * | |
| * @TAG(DATA61_BSD) | |
| */ | |
| /** | |
| * @brief EHCI asynchronous schedule implementation. | |
| */ | |
| #include <stdint.h> | |
| #include <stdio.h> | |
| #include <string.h> | |
| #include <usb/drivers/usbhub.h> | |
| #include "ehci.h" | |
| #include "../services.h" | |
| enum usb_xact_status qtd_get_status(volatile struct TD* qtd) | |
| { | |
| uint32_t t = qtd->token; | |
| if (t & TDTOK_SACTIVE) { | |
| /* Note that we have already returned an error code | |
| * if this TD is still pending due to an error in | |
| * a previous TD */ | |
| return XACTSTAT_PENDING; | |
| } else if (t & TDTOK_SHALTED) { | |
| if (t & TDTOK_SXACTERR) { | |
| return XACTSTAT_ERROR; | |
| } else if (t & TDTOK_ERROR) { | |
| return XACTSTAT_HOSTERROR; | |
| } | |
| ZF_LOGF("EHCI: Unknown QTD error code 0x%x\n", t); | |
| return XACTSTAT_HOSTERROR; | |
| } else { | |
| return XACTSTAT_SUCCESS; | |
| } | |
| } | |
| enum usb_xact_status qhn_get_status(struct QHn * qhn) | |
| { | |
| struct TDn *tdn; | |
| tdn = qhn->tdns; | |
| while (tdn) { | |
| enum usb_xact_status stat; | |
| stat = qtd_get_status(tdn->td); | |
| if (stat != XACTSTAT_SUCCESS) { | |
| return stat; | |
| } | |
| tdn = tdn->next; | |
| } | |
| /* All TDs complete, return the status of the QH */ | |
| return qtd_get_status(&qhn->qh->td_overlay); | |
| } | |
| /**************************** | |
| **** Queue manipulation **** | |
| ****************************/ | |
| /* | |
| * TODO: The current data structure assumes one xact per TD, which means the | |
| * length of an xact can not exceed 20KB. | |
| */ | |
| struct TDn* | |
| qtd_alloc(struct ehci_host *edev, enum usb_speed speed, struct endpoint *ep, | |
| struct xact *xact, int nxact, usb_cb_t cb, void *token) | |
| { | |
| struct TDn *head_tdn = NULL, *prev_tdn, *tdn = NULL; | |
| int buf_filled, cnt, total_bytes = 0; | |
| int xact_stage = 0; | |
| if (!xact || nxact <=0) { | |
| ZF_LOGF("Invalid arguments\n"); | |
| } | |
| prev_tdn = NULL; | |
| for (int i = 0; i < nxact; i++) { | |
| tdn = calloc(1, sizeof(struct TDn)); | |
| if (!tdn) { | |
| ZF_LOGE("Out of memory"); | |
| return NULL; | |
| } | |
| /* Allocate TD overlay */ | |
| tdn->td = ps_dma_alloc_pinned(edev->dman, sizeof(*tdn->td), 32, 0, | |
| PS_MEM_NORMAL, &tdn->ptd); | |
| if (!tdn->td) { | |
| ZF_LOGE("Out of DMA memory"); | |
| return NULL; | |
| } | |
| memset((void*)tdn->td, 0, sizeof(*tdn->td)); | |
| /* Fill in the TD */ | |
| if (prev_tdn) { | |
| prev_tdn->td->next = tdn->ptd; | |
| } else { | |
| head_tdn = tdn; | |
| } | |
| tdn->td->alt = TDLP_INVALID; | |
| /* The Control endpoint manages its own data toggle */ | |
| if (ep->type == EP_CONTROL) { | |
| if (ep->max_pkt & (ep->max_pkt + total_bytes - 1)) { | |
| tdn->td->token = TDTOK_DT; | |
| } | |
| } | |
| tdn->td->token |= TDTOK_BYTES(xact[i].len); | |
| tdn->td->token |= TDTOK_C_ERR(0x3); //Maximize retries | |
| switch (xact[i].type) { | |
| case PID_SETUP: | |
| tdn->td->token |= TDTOK_PID_SETUP; | |
| xact_stage |= TDTOK_PID_SETUP; | |
| break; | |
| case PID_IN: | |
| tdn->td->token |= TDTOK_PID_IN; | |
| xact_stage |= TDTOK_PID_IN; | |
| break; | |
| case PID_OUT: | |
| tdn->td->token |= TDTOK_PID_OUT; | |
| xact_stage |= TDTOK_PID_OUT; | |
| break; | |
| default: | |
| ZF_LOGF("Invalid PID!\n"); | |
| break; | |
| } | |
| tdn->td->token |= TDTOK_SHALTED; | |
| /* Ping control */ | |
| if (speed == USBSPEED_HIGH && xact[i].type == PID_OUT) { | |
| tdn->td->token |= TDTOK_PINGSTATE; | |
| } | |
| /* Fill in the buffer */ | |
| cnt = 0; | |
| tdn->td->buf[cnt] = xact[i].paddr; //First buffer has offset | |
| buf_filled = 0x1000 - (xact[i].paddr & 0xFFF); | |
| /* All following buffers are page aligned */ | |
| while (buf_filled < xact[i].len) { | |
| cnt++; | |
| tdn->td->buf[cnt] = (xact[i].paddr + 0x1000 * cnt) & ~0xFFF; | |
| buf_filled += 0x1000; | |
| } | |
| /* We only have 5 page-sized buffers */ | |
| if (cnt > 4) { | |
| ZF_LOGF("Too many buffers\n"); | |
| } | |
| /* Total data transferred */ | |
| total_bytes += xact[i].len; | |
| if (prev_tdn) { | |
| prev_tdn->next = tdn; | |
| } | |
| prev_tdn = tdn; | |
| } | |
| /* | |
| * Zero length packet | |
| * XXX: It is unclear that the exact condition of when the zero length | |
| * packet is required. The following implementation is partially based | |
| * on observation. According to USB 2.0 spec(5.5.3), a zero length | |
| * packet shouldn't be needed under some of the situations below, but | |
| * apparently, some devices don't always follow the spec. | |
| */ | |
| if (((xact_stage & TDTOK_PID_OUT) && !(total_bytes % ep->max_pkt)) || | |
| ep->type == EP_CONTROL) { | |
| /* Allocate TD for the zero length packet */ | |
| tdn = calloc(1, sizeof(struct TDn)); | |
| /* Allocate TD overlay */ | |
| tdn->td = ps_dma_alloc_pinned(edev->dman, sizeof(*tdn->td), | |
| 32, 0, PS_MEM_NORMAL, &tdn->ptd); | |
| if (!tdn->td) { | |
| ZF_LOGF("Out of DMA memory\n"); | |
| } | |
| memset((void*)tdn->td, 0, sizeof(*tdn->td)); | |
| /* Fill in the TD */ | |
| tdn->td->alt = TDLP_INVALID; | |
| tdn->td->token = TDTOK_C_ERR(0x3) | TDTOK_SHALTED; | |
| if (xact_stage & TDTOK_PID_SETUP) { | |
| /* Flip the PID, if there is no data stage, then IN */ | |
| if (xact_stage & TDTOK_PID_IN) { | |
| tdn->td->token |= TDTOK_PID_OUT; | |
| } else { | |
| tdn->td->token |= TDTOK_PID_IN; | |
| } | |
| /* Force DATA1 */ | |
| tdn->td->token |= TDTOK_DT; | |
| } else { | |
| /* Bulk out transfer */ | |
| tdn->td->token |= TDTOK_PID_OUT; | |
| /* XXX: Flip the data toggle? */ | |
| if (!(prev_tdn->td->token & TDTOK_DT)) { | |
| tdn->td->token |= TDTOK_DT; | |
| } | |
| } | |
| /* Add to the list */ | |
| prev_tdn->td->next = tdn->ptd; | |
| prev_tdn->next = tdn; | |
| } | |
| /* Send IRQ when finished processing the last TD */ | |
| tdn->td->token |= TDTOK_IOC; //TODO: Maybe disable IRQ when cb == NULL | |
| tdn->cb = cb; | |
| tdn->token = token; | |
| /* Mark the last TD as terminate TD */ | |
| tdn->td->next |= TDLP_INVALID; | |
| return head_tdn; | |
| } | |
| /* | |
| * Allocate generic queue head for both periodic and asynchronous schedule | |
| * Note that the link pointer and reclamation flag bit are set when inserting | |
| * the queue head to asynchronous schedule. | |
| * | |
| * XXX: For some unknown reason, we cannot initialize the link pointer and head | |
| * type here, it seems that the register can be only written once. So we'll fill | |
| * the register when adding the queue head to the schedule. | |
| */ | |
| struct QHn* | |
| qhn_alloc(struct ehci_host *edev, uint8_t address, uint8_t hub_addr, | |
| uint8_t hub_port, enum usb_speed speed, struct endpoint *ep) | |
| { | |
| struct QHn *qhn; | |
| volatile struct QH *qh; | |
| qhn = calloc(1, sizeof(struct QHn)); | |
| if (!qhn) { | |
| ZF_LOGF("Out of memory\n"); | |
| } | |
| /* Allocate queue head */ | |
| qhn->qh = ps_dma_alloc_pinned(edev->dman, sizeof(*qh), 32, 0, | |
| PS_MEM_NORMAL, &qhn->pqh); | |
| if (!qhn->qh) { | |
| ZF_LOGF("Out of DMA memory\n"); | |
| } | |
| memset((void*)qhn->qh, 0, sizeof(*qh)); | |
| /* Fill in the queue head */ | |
| qh = qhn->qh; | |
| qh->qhlptr = QHLP_INVALID; | |
| /* epc0 */ | |
| switch (speed) { | |
| case USBSPEED_HIGH: | |
| qh->epc[0] = QHEPC0_HSPEED; | |
| break; | |
| case USBSPEED_FULL: | |
| qh->epc[0] = QHEPC0_FSPEED; | |
| break; | |
| case USBSPEED_LOW : | |
| qh->epc[0] = QHEPC0_LSPEED; | |
| break; | |
| default: | |
| ZF_LOGF("Invalid speed\n"); | |
| } | |
| qh->epc[0] |= QHEPC0_MAXPKTLEN(ep->max_pkt) | QHEPC0_ADDR(address) | | |
| QHEPC0_EP(ep->num); | |
| /* | |
| * Nak counter must NOT be used for interrupt endpoint | |
| * EHCI spec chapter 4.9(Nak "Not Used" mode) | |
| */ | |
| if (ep->type == EP_INTERRUPT) { | |
| qh->epc[0] |= QHEPC0_NAKCNT_RL(0); | |
| } else { | |
| qh->epc[0] |= QHEPC0_NAKCNT_RL(0x8); | |
| } | |
| /* Control endpoint manages its own data toggle */ | |
| if (ep->type == EP_CONTROL) { | |
| qh->epc[0] |= QHEPC0_DTC; | |
| /* For full/low speed control endpoint */ | |
| if (speed != USBSPEED_HIGH) { | |
| qh->epc[0] |= QHEPC0_C; | |
| } | |
| } | |
| /* epc1 */ | |
| qh->epc[1] = QHEPC1_MULT(1); | |
| if (speed != USBSPEED_HIGH) { | |
| qh->epc[1] |= QHEPC1_HUB_ADDR(hub_addr) | QHEPC1_PORT(hub_port); | |
| } | |
| /* TODO: Check CMASK and SMASK */ | |
| if ((ep->type == EP_INTERRUPT || ep->type == EP_ISOCHRONOUS) && | |
| speed != USBSPEED_HIGH) { | |
| qh->epc[1] |= QHEPC1_UFRAME_CMASK(0x1C); | |
| } | |
| if (ep->type == EP_INTERRUPT) { | |
| qh->epc[1] |= QHEPC1_UFRAME_SMASK(1); | |
| } | |
| qh->td_overlay.next = TDLP_INVALID; | |
| qh->td_overlay.alt = TDLP_INVALID; | |
| return qhn; | |
| } | |
| void | |
| qhn_update(struct QHn *qhn, uint8_t address, struct endpoint *ep) | |
| { | |
| uint32_t epc0; | |
| if (!qhn || !ep) { | |
| ZF_LOGF("Invalid arguments\n"); | |
| } | |
| /* | |
| * We only care about the control endpoint, because all other | |
| * endpoints' info is extracted from the endpoint descriptor, and by the | |
| * time the core driver reads the descriptors, the device's address | |
| * should have settled already. | |
| */ | |
| if (ep->type != EP_CONTROL) { | |
| return; | |
| } | |
| /* Update maximum packet size */ | |
| epc0 = qhn->qh->epc[0]; | |
| if (unlikely(QHEPC0_GET_MAXPKT(epc0) != ep->max_pkt)) { | |
| epc0 &= ~QHEPC0_MAXPKT_MASK; | |
| epc0 |= QHEPC0_MAXPKTLEN(ep->max_pkt); | |
| } | |
| /* Update device address */ | |
| if (unlikely(QHEPC0_GET_ADDR(epc0) != address)) { | |
| epc0 &= ~QHEPC0_ADDR_MASK; | |
| epc0 |= QHEPC0_ADDR(address); | |
| } | |
| qhn->qh->epc[0] = epc0; | |
| } | |
| void | |
| qtd_enqueue(struct ehci_host *edev, struct QHn *qhn, struct TDn *tdn) | |
| { | |
| struct TDn *last_tdn; | |
| if (!qhn || !tdn) { | |
| ZF_LOGF("Invalid arguments\n"); | |
| } | |
| /* If the queue is empty, point the TD overlay to the first TD */ | |
| if (!qhn->tdns) { | |
| qhn->qh->td_overlay.next = tdn->ptd; | |
| qhn->tdns = tdn; | |
| } else { | |
| ps_mutex_lock(edev->sync, qhn->lock); | |
| /* Find the last TD */ | |
| last_tdn = qhn->tdns; | |
| while (last_tdn->next) { | |
| last_tdn = last_tdn->next; | |
| } | |
| if (qhn->qh->td_cur == last_tdn->ptd && | |
| qhn->qh->td_overlay.next == TDLP_INVALID) { | |
| qhn->qh->td_overlay.next = tdn->ptd; | |
| } | |
| /* Add new TD to the queue and update the termination bit */ | |
| last_tdn->next = tdn; | |
| last_tdn->td->next = tdn->ptd & ~TDLP_INVALID; | |
| ps_mutex_unlock(edev->sync, qhn->lock); | |
| } | |
| /* Enable all TDs */ | |
| while (tdn) { | |
| tdn->td->token &= ~TDTOK_SHALTED; | |
| tdn->td->token |= TDTOK_SACTIVE; | |
| tdn = tdn->next; | |
| } | |
| /* Make sure the controller sees all the active TDs */ | |
| dsb(); | |
| } | |
| void qhn_destroy(ps_dma_man_t* dman, struct QHn* qhn) | |
| { | |
| struct TDn *tdn, *tmp; | |
| tdn = qhn->tdns; | |
| while (tdn) { | |
| tmp = tdn; | |
| tdn = tdn->next; | |
| if (tmp->cb) { | |
| tmp->cb(tmp->token, XACTSTAT_CANCELLED, 0); | |
| } | |
| ps_dma_free_pinned(dman, (void*)tmp->td, sizeof(struct TD)); | |
| usb_free(tmp); | |
| } | |
| ps_dma_free_pinned(dman, (void*)qhn->qh, sizeof(struct QH)); | |
| usb_free(qhn); | |
| } | |
| void ehci_async_complete(struct ehci_host *edev) | |
| { | |
| struct QHn *qhn; | |
| struct TDn *tdn, *head, *tmp; | |
| int sum; | |
| qhn = edev->alist_tail; | |
| /* Nothing to do if the queue is empty */ | |
| if (!qhn) { | |
| return; | |
| } | |
| do { | |
| ps_mutex_lock(edev->sync, qhn->lock); | |
| tdn = qhn->tdns; | |
| sum = 0; | |
| head = tdn; | |
| while (tdn != NULL && | |
| qtd_get_status(tdn->td) == XACTSTAT_SUCCESS) { | |
| sum += TDTOK_GET_BYTES(tdn->td->token); | |
| if (tdn->td->token & TDTOK_IOC) { | |
| if (tdn->cb) { | |
| tdn->cb(tdn->token, XACTSTAT_SUCCESS, sum); | |
| sum = 0; | |
| } | |
| qhn->tdns = tdn->next; | |
| /* | |
| * Update the QH if we are about to dequeue the | |
| * "previous" last TD in the queue. This happens | |
| * when the last TD gets partially processed | |
| * while we enqueue new TDs. | |
| */ | |
| if (qhn->tdns && | |
| qhn->qh->td_cur == tdn->ptd && | |
| qhn->qh->td_overlay.next == TDLP_INVALID) { | |
| qhn->qh->td_overlay.next = tdn->next->ptd; | |
| dsb(); | |
| } | |
| /* Free */ | |
| while (head != tdn->next) { | |
| tmp = head; | |
| head = head->next; | |
| ps_dma_free_pinned(edev->dman, | |
| (void*)tmp->td, | |
| sizeof(struct TD)); | |
| usb_free(tmp); | |
| } | |
| } | |
| tdn = tdn->next; | |
| } | |
| ps_mutex_unlock(edev->sync, qhn->lock); | |
| qhn = qhn->next; | |
| } while (qhn != edev->alist_tail); | |
| } | |
| void ehci_add_qhn_async(struct ehci_host *edev, struct QHn *qhn) | |
| { | |
| /* Add new queue head to async queue */ | |
| if (edev->alist_tail) { | |
| /* Update the hardware queue */ | |
| qhn->qh->qhlptr = edev->alist_tail->qh->qhlptr; | |
| edev->alist_tail->qh->qhlptr = qhn->pqh | QHLP_TYPE_QH; | |
| /* Update the Software queue */ | |
| qhn->next = edev->alist_tail->next; | |
| edev->alist_tail->next = qhn; | |
| edev->alist_tail = qhn; | |
| } else { | |
| edev->alist_tail = qhn; | |
| edev->alist_tail->next = qhn; | |
| qhn->qh->qhlptr = qhn->pqh | QHLP_TYPE_QH; | |
| } | |
| dsb(); | |
| } | |
| void ehci_del_qhn_async(struct ehci_host *edev, struct QHn *qhn) | |
| { | |
| struct TDn *tdn; | |
| struct QHn *prev; | |
| /* | |
| * The EHCI spec(section 4.8.2) gives the instructions of removing a QH | |
| * from the active async queue. However, the instructions have been | |
| * proved racy and impractical. See the following link for more | |
| * information, | |
| * | |
| * https://marc.info/?l=linux-usb&m=144554811526940&w=2 | |
| * | |
| * This implementation combines part of the spec instructions and | |
| * Linux's double IAA cycle approach. Hope it would minimize the chances | |
| * of trigging serious problems. Good Luck! | |
| */ | |
| /* | |
| * Deactivate all active TDs | |
| * | |
| * This is the racy part of the EHCI spec. The host may have already in | |
| * the middle of processing the TD overlay when we change the token. | |
| * But we've got no choice otherwise the host would follow the TD list | |
| * and would cause further errors. | |
| */ | |
| tdn = qhn->tdns; | |
| while (tdn) { | |
| if (tdn->td->token & TDTOK_SACTIVE) { | |
| tdn->td->token &= ~TDTOK_SACTIVE; | |
| tdn->td->token |= TDTOK_SHALTED; | |
| } | |
| tdn = tdn->next; | |
| } | |
| dsb(); | |
| /* Select another queue head to set its H-bit */ | |
| if ((qhn->qh->epc[0] & QHEPC0_H) && qhn->next) { | |
| qhn->next->qh->epc[0] |= QHEPC0_H; | |
| } | |
| /* Remove the queue head from async list */ | |
| prev = edev->alist_tail; | |
| while (prev->next != qhn) { | |
| prev = prev->next; | |
| } | |
| prev->qh->qhlptr = qhn->qh->qhlptr; | |
| prev->next = qhn->next; | |
| if (edev->alist_tail == qhn) { | |
| edev->alist_tail = qhn->next; | |
| } | |
| /* Put the queue head to the recycle queue */ | |
| if (edev->db_pending) { | |
| prev = edev->db_pending; | |
| while (prev->next != NULL) { | |
| prev = prev->next; | |
| } | |
| prev->next = qhn; | |
| } else { | |
| edev->db_pending = qhn; | |
| } | |
| qhn->next = NULL; | |
| qhn->was_cancelled = 1; | |
| /* Ring the doorbell */ | |
| edev->op_regs->usbcmd |= EHCICMD_ASYNC_DB; | |
| } | |
| int ehci_wait_for_completion(struct TDn *tdn) | |
| { | |
| int status; | |
| int cnt, sum = 0; | |
| while (tdn) { | |
| cnt = 3000; | |
| do { | |
| status = tdn->td->token & 0xFF; | |
| if (status == 0 || status == 1) { | |
| sum += TDTOK_GET_BYTES(tdn->td->token); | |
| break; | |
| } | |
| if (cnt <= 0) { | |
| ZF_LOGF("Timeout(%p, %p)\n", tdn->td, | |
| (void*)tdn->ptd); | |
| return -1; | |
| } | |
| ps_mdelay(1); | |
| cnt--; | |
| } while (status); | |
| tdn = tdn->next; | |
| } | |
| return sum; | |
| } | |
| /* TODO: Is it okay to use alist_tail and remove qhn */ | |
| void ehci_schedule_async(struct ehci_host* edev, struct QHn* qhn) | |
| { | |
| /* Make sure we are safe to write to the register */ | |
| while (((edev->op_regs->usbsts & EHCISTS_ASYNC_EN) >> 15) | |
| ^ ((edev->op_regs->usbcmd & EHCICMD_ASYNC_EN) >> 5)); | |
| /* Enable the async scheduling */ | |
| if (!(edev->op_regs->usbsts & EHCISTS_ASYNC_EN)) { | |
| qhn->qh->epc[0] |= QHEPC0_H; | |
| edev->op_regs->asynclistaddr = qhn->pqh; | |
| edev->op_regs->usbcmd |= EHCICMD_ASYNC_EN; | |
| while (edev->op_regs->usbsts & EHCISTS_ASYNC_EN) break; | |
| } | |
| } | |
| void check_doorbell(struct ehci_host* edev) | |
| { | |
| int again = 0; | |
| struct QHn *qhn, *tmp, *tail = NULL; | |
| qhn = edev->db_pending; | |
| edev->db_pending = NULL; | |
| while (qhn) { | |
| tmp = qhn; | |
| qhn = qhn->next; | |
| /* Two IAA cycles have passed, safe to remove */ | |
| if (tmp->was_cancelled > 1) { | |
| qhn_destroy(edev->dman, tmp); | |
| } else { | |
| tmp->was_cancelled++; | |
| if (!edev->db_pending) { | |
| edev->db_pending = tmp; | |
| } else { | |
| tail->next = tmp; | |
| } | |
| tail = tmp; | |
| again = 1; | |
| } | |
| } | |
| if (again) { | |
| edev->op_regs->usbcmd |= EHCICMD_ASYNC_DB; | |
| } | |
| } | |