1450N/A/*
1450N/A * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved.
1450N/A */
1450N/A
1450N/A/*
1450N/A * Copyright (c) 2012, 2013, Intel Corporation. All rights reserved.
1450N/A */
1450N/A
1450N/A/*
1450N/A * Copyright © 2010 Daniel Vetter
1450N/A *
1450N/A * Permission is hereby granted, free of charge, to any person obtaining a
1450N/A * copy of this software and associated documentation files (the "Software"),
1450N/A * to deal in the Software without restriction, including without limitation
1450N/A * the rights to use, copy, modify, merge, publish, distribute, sublicense,
1450N/A * and/or sell copies of the Software, and to permit persons to whom the
1450N/A * Software is furnished to do so, subject to the following conditions:
1450N/A *
1450N/A * The above copyright notice and this permission notice (including the next
1450N/A * paragraph) shall be included in all copies or substantial portions of the
1450N/A * Software.
1450N/A *
1450N/A * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1450N/A * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1450N/A * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
1450N/A * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1450N/A * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
1450N/A * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
1450N/A * IN THE SOFTWARE.
1450N/A *
1450N/A */
1450N/A
1450N/A#include "drmP.h"
1450N/A#include "drm.h"
1450N/A#include "i915_drm.h"
1450N/A#include "i915_drv.h"
1450N/A#include "intel_drv.h"
1450N/A
1450N/Atypedef uint32_t gtt_pte_t;
1450N/A
1450N/A/* PPGTT stuff */
1450N/A#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0))
1450N/A
1450N/A#define GEN6_PDE_VALID (1 << 0)
1450N/A/* gen6+ has bit 11-4 for physical addr bit 39-32 */
1450N/A#define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
1450N/A
1450N/A#define GEN6_PTE_VALID (1 << 0)
1450N/A#define GEN6_PTE_UNCACHED (1 << 1)
1450N/A#define HSW_PTE_UNCACHED (0)
1450N/A#define GEN6_PTE_CACHE_LLC (2 << 1)
1450N/A#define GEN6_PTE_CACHE_LLC_MLC (3 << 1)
1450N/A#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
1450N/A
1450N/Astatic gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev,
1450N/A uint64_t addr,
1450N/A enum i915_cache_level level)
1450N/A{
1450N/A gen6_gtt_pte_t pte = GEN6_PTE_VALID;
1450N/A pte |= GEN6_PTE_ADDR_ENCODE(addr);
1450N/A
1450N/A switch (level) {
1450N/A case I915_CACHE_LLC_MLC:
1450N/A pte |= GEN6_PTE_CACHE_LLC_MLC;
1450N/A break;
1450N/A case I915_CACHE_LLC:
1450N/A pte |= GEN6_PTE_CACHE_LLC;
1450N/A break;
1450N/A case I915_CACHE_NONE:
1450N/A pte |= GEN6_PTE_UNCACHED;
1450N/A break;
1450N/A default:
1450N/A BUG();
1450N/A }
1450N/A
1450N/A return pte;
1450N/A}
1450N/A
1450N/A#define BYT_PTE_WRITEABLE (1 << 1)
1450N/A#define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2)
1450N/A
1450N/Astatic gen6_gtt_pte_t byt_pte_encode(struct drm_device *dev,
1450N/A uint64_t addr,
1450N/A enum i915_cache_level level)
1450N/A{
1450N/A gen6_gtt_pte_t pte = GEN6_PTE_VALID;
1450N/A pte |= GEN6_PTE_ADDR_ENCODE(addr);
1450N/A
1450N/A /* Mark the page as writeable. Other platforms don't have a
1450N/A * setting for read-only/writable, so this matches that behavior.
1450N/A */
1450N/A pte |= BYT_PTE_WRITEABLE;
1450N/A
1450N/A if (level != I915_CACHE_NONE)
1450N/A pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
1450N/A
1450N/A return pte;
1450N/A}
1450N/A
1450N/Astatic gen6_gtt_pte_t hsw_pte_encode(struct drm_device *dev,
1450N/A uint64_t addr,
1450N/A enum i915_cache_level level)
1450N/A{
1450N/A gen6_gtt_pte_t pte = GEN6_PTE_VALID;
1450N/A pte |= GEN6_PTE_ADDR_ENCODE(addr);
1450N/A
1450N/A if (level != I915_CACHE_NONE)
1450N/A pte |= GEN6_PTE_CACHE_LLC;
1450N/A
1450N/A return pte;
1450N/A}
1450N/A
1450N/Astatic ddi_dma_attr_t ppgt_dma_attr = {
1450N/A DMA_ATTR_V0,
1450N/A 0xff000U, /* dma_attr_addr_lo */
1450N/A 0xffffffffU, /* dma_attr_addr_hi */
1450N/A 0xffffffffU, /* dma_attr_count_max */
1450N/A 4096, /* dma_attr_align */
1450N/A 0x1fffU, /* dma_attr_burstsizes */
1450N/A 1, /* dma_attr_minxfer */
1450N/A 0xffffffffU, /* dma_attr_maxxfer */
1450N/A 0xffffffffU, /* dma_attr_seg */
1450N/A 1, /* dma_attr_sgllen, variable */
1450N/A 4, /* dma_attr_granular */
1450N/A DDI_DMA_FLAGERR, /* dma_attr_flags */
1450N/A};
1450N/Astatic ddi_device_acc_attr_t ppgt_acc_attr = {
1450N/A DDI_DEVICE_ATTR_V0,
1450N/A DDI_NEVERSWAP_ACC,
1450N/A DDI_MERGING_OK_ACC,
1450N/A DDI_FLAGERR_ACC
1450N/A};
1450N/A
1450N/Astatic int
1450N/Ai915_ppgtt_page_alloc(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,
1450N/A int pgcnt)
1450N/A{
1450N/A ddi_dma_cookie_t cookie;
1450N/A uint_t cookie_cnt;
1450N/A pgcnt_t real_pgcnt;
1450N/A uint32_t paddr, cookie_end;
1450N/A int i, n;
1450N/A
1450N/A ppgt_dma_attr.dma_attr_sgllen = (int)pgcnt;
1450N/A
1450N/A if (ddi_dma_alloc_handle(dev->devinfo, &ppgt_dma_attr,
1450N/A DDI_DMA_DONTWAIT, NULL, &ppgtt->dma_hdl)) {
1450N/A DRM_ERROR("i915_ppgtt_page_alloc: "
1450N/A "ddi_dma_alloc_handle failed");
1450N/A goto err1;
1450N/A }
1450N/A if (ddi_dma_mem_alloc(ppgtt->dma_hdl, ptob(pgcnt), &ppgt_acc_attr,
1450N/A IOMEM_DATA_UC_WR_COMBINE, DDI_DMA_DONTWAIT, NULL,
1450N/A &ppgtt->kaddr, &ppgtt->real_size, &ppgtt->acc_hdl)) {
1450N/A DRM_ERROR("drm_gem_object_alloc: "
1450N/A "ddi_dma_mem_alloc failed");
1450N/A goto err2;
1450N/A }
1450N/A if (ddi_dma_addr_bind_handle(ppgtt->dma_hdl, NULL,
1450N/A ppgtt->kaddr, ppgtt->real_size, DDI_DMA_RDWR,
1450N/A DDI_DMA_DONTWAIT, NULL, &cookie, &cookie_cnt)
1450N/A != DDI_DMA_MAPPED) {
1450N/A DRM_ERROR("drm_gem_object_alloc: "
1450N/A "ddi_dma_addr_bind_handle failed");
1450N/A goto err3;
1450N/A }
1450N/A
1450N/A real_pgcnt = btopr(ppgtt->real_size);
1450N/A
1450N/A ppgtt->pt_pages = kmem_zalloc(real_pgcnt * sizeof (pfn_t), KM_NOSLEEP);
1450N/A if (ppgtt->pt_pages == NULL) {
1450N/A DRM_DEBUG("pfnarray == NULL");
1450N/A goto err4;
1450N/A }
1450N/A
1450N/A for (n = 0, i = 1; ; i++) {
1450N/A for (paddr = cookie.dmac_address,
1450N/A cookie_end = cookie.dmac_address + cookie.dmac_size;
1450N/A paddr < cookie_end;
1450N/A paddr += PAGESIZE) {
1450N/A ppgtt->pt_pages[n++] = btop(paddr);
1450N/A if (n >= real_pgcnt)
1450N/A return (0);
1450N/A }
1450N/A if (i >= cookie_cnt)
1450N/A break;
1450N/A ddi_dma_nextcookie(ppgtt->dma_hdl, &cookie);
1450N/A }
1450N/A
1450N/Aerr4:
1450N/A (void) ddi_dma_unbind_handle(ppgtt->dma_hdl);
1450N/Aerr3:
1450N/A ddi_dma_mem_free(&ppgtt->acc_hdl);
1450N/Aerr2:
1450N/A ddi_dma_free_handle(&ppgtt->dma_hdl);
1450N/Aerr1:
1450N/A return (-1);
1450N/A
1450N/A}
1450N/A
1450N/Astatic void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = ppgtt->dev->dev_private;
1450N/A gen6_gtt_pte_t *pd_addr;
1450N/A uint32_t pt_addr;
1450N/A uint32_t pd_entry;
1450N/A int i;
1450N/A
1450N/A WARN_ON(ppgtt->pd_offset & 0x3f);
1450N/A for (i = 0; i < ppgtt->num_pd_entries; i++) {
1450N/A pd_addr = (gen6_gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt
1450N/A + ppgtt->pd_offset + i * sizeof(gtt_pte_t));
1450N/A
1450N/A
1450N/A pt_addr = ppgtt->pt_pages[i] << PAGE_SHIFT;
1450N/A pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
1450N/A pd_entry |= GEN6_PDE_VALID;
1450N/A
1450N/A ddi_put32(dev_priv->gtt.gtt_mapping.acc_handle,
1450N/A pd_addr, pd_entry);
1450N/A
1450N/A }
1450N/A (void) ddi_get32(dev_priv->gtt.gtt_mapping.acc_handle,
1450N/A (gen6_gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt + ppgtt->pd_offset));
1450N/A}
1450N/A
1450N/Astatic int gen6_ppgtt_enable(struct drm_device *dev)
1450N/A{
1450N/A drm_i915_private_t *dev_priv = dev->dev_private;
1450N/A uint32_t pd_offset;
1450N/A struct intel_ring_buffer *ring;
1450N/A struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1450N/A int i;
1450N/A
1450N/A BUG_ON(ppgtt->pd_offset & 0x3f);
1450N/A
1450N/A gen6_write_pdes(ppgtt);
1450N/A
1450N/A pd_offset = ppgtt->pd_offset;
1450N/A pd_offset /= 64; /* in cachelines, */
1450N/A pd_offset <<= 16;
1450N/A
1450N/A if (INTEL_INFO(dev)->gen == 6) {
1450N/A uint32_t ecochk, gab_ctl, ecobits;
1450N/A
1450N/A ecobits = I915_READ(GAC_ECO_BITS);
1450N/A I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1450N/A ECOBITS_PPGTT_CACHE64B);
1450N/A
1450N/A gab_ctl = I915_READ(GAB_CTL);
1450N/A I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1450N/A
1450N/A ecochk = I915_READ(GAM_ECOCHK);
1450N/A I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
1450N/A ECOCHK_PPGTT_CACHE64B);
1450N/A I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1450N/A } else if (INTEL_INFO(dev)->gen >= 7) {
1450N/A uint32_t ecochk, ecobits;
1450N/A
1450N/A ecobits = I915_READ(GAC_ECO_BITS);
1450N/A I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1450N/A
1450N/A ecochk = I915_READ(GAM_ECOCHK);
1450N/A if (IS_HASWELL(dev)) {
1450N/A ecochk |= ECOCHK_PPGTT_WB_HSW;
1450N/A } else {
1450N/A ecochk |= ECOCHK_PPGTT_LLC_IVB;
1450N/A ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1450N/A }
1450N/A I915_WRITE(GAM_ECOCHK, ecochk);
1450N/A /* GFX_MODE is per-ring on gen7+ */
1450N/A }
1450N/A
1450N/A for_each_ring(ring, dev_priv, i) {
1450N/A if (INTEL_INFO(dev)->gen >= 7)
1450N/A I915_WRITE(RING_MODE_GEN7(ring),
1450N/A _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1450N/A
1450N/A I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
1450N/A I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
1450N/A }
1450N/A return 0;
1450N/A}
1450N/A
1450N/A/* PPGTT support for Sandybdrige/Gen6 and later */
1450N/Astatic void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
1450N/A unsigned first_entry,
1450N/A unsigned num_entries)
1450N/A{
1450N/A gen6_gtt_pte_t *pt_vaddr, scratch_pte;
1450N/A unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
1450N/A unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
1450N/A unsigned last_pte, i;
1450N/A
1450N/A scratch_pte = ppgtt->pte_encode(ppgtt->dev,
1450N/A ppgtt->scratch_page_paddr,
1450N/A I915_CACHE_LLC);
1450N/A
1450N/A while (num_entries) {
1450N/A last_pte = first_pte + num_entries;
1450N/A if (last_pte > I915_PPGTT_PT_ENTRIES)
1450N/A last_pte = I915_PPGTT_PT_ENTRIES;
1450N/A
1450N/A pt_vaddr = (uint32_t*)(uintptr_t)(ppgtt->kaddr + act_pt*PAGE_SIZE);
1450N/A
1450N/A for (i = first_pte; i < last_pte; i++)
1450N/A pt_vaddr[i] = scratch_pte;
1450N/A
1450N/A num_entries -= last_pte - first_pte;
1450N/A first_pte = 0;
1450N/A act_pt++;
1450N/A }
1450N/A}
1450N/A
1450N/Astatic void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt,
1450N/A unsigned first_entry, unsigned num_entries,
1450N/A pfn_t *pages, enum i915_cache_level cache_level)
1450N/A{
1450N/A gen6_gtt_pte_t *pt_vaddr;
1450N/A unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
1450N/A unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
1450N/A unsigned i, j;
1450N/A
1450N/A pt_vaddr = (gen6_gtt_pte_t *)(uintptr_t)(ppgtt->kaddr + act_pt*PAGE_SIZE);
1450N/A for (i = first_entry, j = 0; i < ( first_entry + num_entries); i++, j++) {
1450N/A
1450N/A pt_vaddr[act_pte] = ppgtt->pte_encode(ppgtt->dev, pages[j] << PAGE_SHIFT,
1450N/A cache_level);
1450N/A if (++act_pte == I915_PPGTT_PT_ENTRIES) {
1450N/A act_pt++;
1450N/A pt_vaddr = (gen6_gtt_pte_t *)(uintptr_t)(ppgtt->kaddr + act_pt*PAGE_SIZE);
1450N/A act_pte = 0;
1450N/A
1450N/A }
1450N/A }
1450N/A}
1450N/A
1450N/Astatic void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt)
1450N/A{
1450N/A kmem_free(ppgtt->pt_pages, btopr(ppgtt->real_size) * sizeof (pfn_t));
1450N/A
1450N/A (void) ddi_dma_unbind_handle(ppgtt->dma_hdl);
1450N/A ddi_dma_mem_free(&ppgtt->acc_hdl);
1450N/A ddi_dma_free_handle(&ppgtt->dma_hdl);
1450N/A kfree(ppgtt, sizeof(*ppgtt));
1450N/A}
1450N/A
1450N/Astatic int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1450N/A{
1450N/A struct drm_device *dev = ppgtt->dev;
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A unsigned first_pd_entry_in_global_pt;
1450N/A int ret = -ENOMEM;
1450N/A
1450N/A /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
1450N/A * entries. For aliasing ppgtt support we just steal them at the end for
1450N/A * now. */
1450N/A first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
1450N/A
1450N/A if (IS_HASWELL(dev)) {
1450N/A ppgtt->pte_encode = hsw_pte_encode;
1450N/A } else if (IS_VALLEYVIEW(dev)) {
1450N/A ppgtt->pte_encode = byt_pte_encode;
1450N/A } else {
1450N/A ppgtt->pte_encode = gen6_pte_encode;
1450N/A }
1450N/A ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
1450N/A ppgtt->enable = gen6_ppgtt_enable;
1450N/A ppgtt->clear_range = gen6_ppgtt_clear_range;
1450N/A ppgtt->insert_entries = gen6_ppgtt_insert_entries;
1450N/A ppgtt->cleanup = gen6_ppgtt_cleanup;
1450N/A ret = i915_ppgtt_page_alloc(dev, ppgtt, ppgtt->num_pd_entries);
1450N/A if (ret)
1450N/A return (-ENOMEM);
1450N/A
1450N/A ppgtt->clear_range(ppgtt, 0,
1450N/A ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
1450N/A
1450N/A ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
1450N/A
1450N/A return 0;
1450N/A
1450N/A}
1450N/A
1450N/Astatic int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A struct i915_hw_ppgtt *ppgtt;
1450N/A int ret = -EINVAL;
1450N/A
1450N/A ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1450N/A if (!ppgtt)
1450N/A return -ENOMEM;
1450N/A
1450N/A ppgtt->dev = dev;
1450N/A ppgtt->scratch_page_paddr = ptob(dev_priv->gtt.scratch_page->pfnarray[0]);
1450N/A
1450N/A if (INTEL_INFO(dev)->gen < 8)
1450N/A ret = gen6_ppgtt_init(ppgtt);
1450N/A else {
1450N/A BUG();
1450N/A DRM_ERROR("ppgtt is not supported");
1450N/A }
1450N/A
1450N/A if (ret)
1450N/A kfree(ppgtt, sizeof(*ppgtt));
1450N/A else
1450N/A dev_priv->mm.aliasing_ppgtt = ppgtt;
1450N/A
1450N/A return ret;
1450N/A}
1450N/A
1450N/Avoid i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1450N/A
1450N/A if (!ppgtt)
1450N/A return;
1450N/A
1450N/A ppgtt->cleanup(ppgtt);
1450N/A dev_priv->mm.aliasing_ppgtt = NULL;
1450N/A}
1450N/A
1450N/Avoid i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
1450N/A struct drm_i915_gem_object *obj,
1450N/A enum i915_cache_level cache_level)
1450N/A{
1450N/A
1450N/A ppgtt->insert_entries(ppgtt,
1450N/A obj->gtt_space->start >> PAGE_SHIFT,
1450N/A obj->base.size >> PAGE_SHIFT,
1450N/A obj->base.pfnarray,
1450N/A cache_level);
1450N/A}
1450N/A
1450N/Avoid i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
1450N/A struct drm_i915_gem_object *obj)
1450N/A{
1450N/A ppgtt->clear_range(ppgtt,
1450N/A obj->gtt_space->start >> PAGE_SHIFT,
1450N/A obj->base.size >> PAGE_SHIFT);
1450N/A}
1450N/A
1450N/Avoid i915_gem_restore_gtt_mappings(struct drm_device *dev)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A struct drm_i915_gem_object *obj;
1450N/A
1450N/A /* First fill our portion of the GTT with scratch pages */
1450N/A/*
1450N/A i915_ggtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
1450N/A (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
1450N/A*/
1450N/A
1450N/A list_for_each_entry(obj, struct drm_i915_gem_object, &dev_priv->mm.bound_list, global_list) {
1450N/A i915_gem_clflush_object(obj);
1450N/A i915_gem_gtt_bind_object(obj, obj->cache_level);
1450N/A }
1450N/A
1450N/A i915_gem_chipset_flush(dev);
1450N/A}
1450N/A
1450N/Aint i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
1450N/A{
1450N/A#if 0
1450N/A if (obj->has_dma_mapping)
1450N/A return 0;
1450N/A
1450N/A if (!dma_map_sg(&obj->base.dev->pdev->dev,
1450N/A obj->pages->sgl, obj->pages->nents,
1450N/A PCI_DMA_BIDIRECTIONAL))
1450N/A return -ENOSPC;
1450N/A
1450N/A#endif
1450N/A return 0;
1450N/A}
1450N/A
1450N/A/*
1450N/A * Binds an object into the global gtt with the specified cache level. The object
1450N/A * will be accessible to the GPU via commands whose operands reference offsets
1450N/A * within the global GTT as well as accessible by the GPU through the GMADR
1450N/A * mapped BAR (dev_priv->mm.gtt->gtt).
1450N/A */
1450N/Astatic void gen6_ggtt_insert_entries(struct drm_i915_gem_object *obj,
1450N/A enum i915_cache_level level)
1450N/A{
1450N/A struct drm_device *dev = obj->base.dev;
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A unsigned first_entry = obj->gtt_offset >> PAGE_SHIFT;
1450N/A unsigned num_entries = obj->base.size / PAGE_SIZE;
1450N/A /* LINTED */
1450N/A const int max_entries = gtt_total_entries(dev_priv->gtt);
1450N/A uint64_t page_addr;
1450N/A gtt_pte_t *gtt_addr;
1450N/A int i, j;
1450N/A
1450N/A for (i = first_entry, j = 0; i < ( first_entry + num_entries); i++, j++) {
1450N/A gtt_addr = (gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt + i * sizeof(gtt_pte_t));
1450N/A page_addr = obj->base.pfnarray[j] << PAGE_SHIFT;
1450N/A ddi_put32(dev_priv->gtt.gtt_mapping.acc_handle,
1450N/A gtt_addr,
1450N/A dev_priv->gtt.pte_encode(dev, page_addr, level));
1450N/A }
1450N/A
1450N/A BUG_ON(i > max_entries);
1450N/A BUG_ON(j != obj->base.size / PAGE_SIZE);
1450N/A
1450N/A /* XXX: This serves as a posting read to make sure that the PTE has
1450N/A * actually been updated. There is some concern that even though
1450N/A * registers and PTEs are within the same BAR that they are potentially
1450N/A * of NUMA access patterns. Therefore, even with the way we assume
1450N/A * hardware should work, we must keep this posting read for paranoia.
1450N/A */
1450N/A if (i != 0) {
1450N/A gtt_addr = (gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt + (i-1)*sizeof(gtt_pte_t));
1450N/A page_addr = obj->base.pfnarray[j-1] << PAGE_SHIFT;
1450N/A WARN_ON(ddi_get32(dev_priv->gtt.gtt_mapping.acc_handle, gtt_addr) !=
1450N/A dev_priv->gtt.pte_encode(dev, page_addr, level));
1450N/A }
1450N/A
1450N/A /* This next bit makes the above posting read even more important. We
1450N/A * want to flush the TLBs only after we're certain all the PTE updates
1450N/A * have finished.
1450N/A */
1450N/A I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1450N/A POSTING_READ(GFX_FLSH_CNTL_GEN6);
1450N/A}
1450N/A
1450N/Astatic void gen6_ggtt_clear_range(struct drm_device *dev,
1450N/A struct drm_i915_gem_object *obj,
1450N/A uint32_t type)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A gtt_pte_t scratch_pte;
1450N/A unsigned first_entry = obj->gtt_offset >> PAGE_SHIFT;
1450N/A unsigned num_entries = obj->base.size / PAGE_SIZE;
1450N/A const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1450N/A uint64_t scratch_page_addr = dev_priv->gtt.scratch_page->pfnarray[0] << PAGE_SHIFT;
1450N/A gtt_pte_t *gtt_addr;
1450N/A int i;
1450N/A
1450N/A if (num_entries > max_entries) {
1450N/A DRM_ERROR("First entry = %d; Num entries = %d (max=%d)\n",
1450N/A first_entry, num_entries, max_entries);
1450N/A num_entries = max_entries;
1450N/A }
1450N/A
1450N/A for (i = first_entry ; i < ( first_entry + num_entries); i++) {
1450N/A gtt_addr = (gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt + i * sizeof(gtt_pte_t));
1450N/A scratch_pte = dev_priv->gtt.pte_encode(dev, scratch_page_addr, I915_CACHE_LLC);
1450N/A ddi_put32(dev_priv->gtt.gtt_mapping.acc_handle,
1450N/A gtt_addr, scratch_pte);
1450N/A }
1450N/A}
1450N/A
1450N/Avoid i915_ggtt_insert_entries(struct drm_i915_gem_object *obj,
1450N/A enum i915_cache_level cache_level)
1450N/A{
1450N/A struct drm_device *dev = obj->base.dev;
1450N/A
1450N/A unsigned int flags = (cache_level == I915_CACHE_NONE) ?
1450N/A AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
1450N/A
1450N/A (void) drm_agp_bind_pages(dev,
1450N/A obj->base.pfnarray,
1450N/A obj->base.size >> PAGE_SHIFT,
1450N/A obj->gtt_offset,
1450N/A flags);
1450N/A}
1450N/A
1450N/Astatic void i915_ggtt_clear_range(struct drm_device *dev,
1450N/A struct drm_i915_gem_object *obj,
1450N/A uint32_t type)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A
1450N/A if (!obj->agp_mem) {
1450N/A (void) drm_agp_unbind_pages(dev, obj->base.pfnarray,
1450N/A (obj->base.size / PAGE_SIZE), obj->gtt_offset,
1450N/A dev_priv->gtt.scratch_page->pfnarray[0], type);
1450N/A obj->agp_mem = -1;
1450N/A }
1450N/A return;
1450N/A
1450N/A}
1450N/A
1450N/Avoid i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
1450N/A enum i915_cache_level cache_level)
1450N/A{
1450N/A struct drm_device *dev = obj->base.dev;
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A
1450N/A dev_priv->gtt.gtt_insert_entries(obj, cache_level);
1450N/A obj->has_global_gtt_mapping = 1;
1450N/A}
1450N/A
1450N/Avoid i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj, uint32_t type)
1450N/A{
1450N/A struct drm_device *dev = obj->base.dev;
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A
1450N/A dev_priv->gtt.gtt_clear_range(dev, obj, type);
1450N/A
1450N/A obj->has_global_gtt_mapping = 0;
1450N/A}
1450N/A
1450N/Avoid i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
1450N/A{
1450N/A}
1450N/A
1450N/Astatic void i915_gtt_color_adjust(struct drm_mm_node *node,
1450N/A unsigned long color,
1450N/A unsigned long *start,
1450N/A unsigned long *end)
1450N/A{
1450N/A if (node->color != color)
1450N/A *start += 4096;
1450N/A
1450N/A if (!list_empty(&node->node_list)) {
1450N/A node = list_entry(node->node_list.next,
1450N/A struct drm_mm_node,
1450N/A node_list);
1450N/A if (node->allocated && node->color != color)
1450N/A *end -= 4096;
1450N/A }
1450N/A}
1450N/A
1450N/Avoid i915_gem_setup_global_gtt(struct drm_device *dev,
1450N/A unsigned long start,
1450N/A unsigned long mappable_end,
1450N/A unsigned long end)
1450N/A{
1450N/A /* Let GEM Manage all of the aperture.
1450N/A *
1450N/A * However, leave one page at the end still bound to the scratch page.
1450N/A * There are a number of places where the hardware apparently prefetches
1450N/A * past the end of the object, and we've seen multiple hangs with the
1450N/A * GPU head pointer stuck in a batchbuffer bound at the last page of the
1450N/A * aperture. One page should be enough to keep any prefetching inside
1450N/A * of the aperture.
1450N/A */
1450N/A drm_i915_private_t *dev_priv = dev->dev_private;
1450N/A struct drm_i915_gem_object *obj;
1450N/A
1450N/A /* Substract the guard page ... */
1450N/A drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
1450N/A if (!HAS_LLC(dev))
1450N/A dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
1450N/A
1450N/A /* Mark any preallocated objects as occupied */
1450N/A list_for_each_entry(obj, struct drm_i915_gem_object,
1450N/A &dev_priv->mm.bound_list, global_list) {
1450N/A DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n",
1450N/A obj->gtt_offset, obj->base.size);
1450N/A
1450N/A BUG_ON(obj->gtt_space != I915_GTT_RESERVED);
1450N/A obj->gtt_space = drm_mm_create_block(&dev_priv->mm.gtt_space,
1450N/A obj->gtt_offset,
1450N/A obj->base.size,
1450N/A false);
1450N/A obj->has_global_gtt_mapping = 1;
1450N/A }
1450N/A
1450N/A dev_priv->gtt.start = start;
1450N/A dev_priv->gtt.total = end - start;
1450N/A}
1450N/A
1450N/Astatic bool
1450N/Aintel_enable_ppgtt(struct drm_device *dev)
1450N/A{
1450N/A if (i915_enable_ppgtt >= 0)
1450N/A return i915_enable_ppgtt;
1450N/A
1450N/A /* Disable ppgtt on SNB if VT-d is on. */
1450N/A if (INTEL_INFO(dev)->gen == 6)
1450N/A return false;
1450N/A
1450N/A return true;
1450N/A}
1450N/A
1450N/Avoid i915_gem_init_global_gtt(struct drm_device *dev)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A unsigned long gtt_size, mappable_size;
1450N/A
1450N/A gtt_size = dev_priv->gtt.total;
1450N/A mappable_size = dev_priv->gtt.mappable_end;
1450N/A
1450N/A if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
1450N/A
1450N/A if (INTEL_INFO(dev)->gen <= 7) {
1450N/A /* PPGTT pdes are stolen from global gtt ptes, so shrink the
1450N/A * aperture accordingly when using aliasing ppgtt. */
1450N/A gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
1450N/A }
1450N/A }
1450N/A i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
1450N/A
1450N/A setup_scratch_page(dev);
1450N/A
1450N/A if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
1450N/A int ret;
1450N/A ret = i915_gem_init_aliasing_ppgtt(dev);
1450N/A if (!ret)
1450N/A return;
1450N/A DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
1450N/A drm_mm_takedown(&dev_priv->mm.gtt_space);
1450N/A gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
1450N/A }
1450N/A
1450N/A}
1450N/A
1450N/Aint setup_scratch_page(struct drm_device *dev)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A int gen;
1450N/A
1450N/A /* setup scratch page */
1450N/A dev_priv->gtt.scratch_page = kzalloc(sizeof(struct drm_gem_object), GFP_KERNEL);
1450N/A if (dev_priv->gtt.scratch_page == NULL)
1450N/A return (-ENOMEM);
1450N/A
1450N/A if (IS_G33(dev))
1450N/A gen = 33;
1450N/A else
1450N/A gen = INTEL_INFO(dev)->gen * 10;
1450N/A
1450N/A if (drm_gem_object_init(dev, dev_priv->gtt.scratch_page, DRM_PAGE_SIZE, gen) != 0) {
1450N/A kmem_free(dev_priv->gtt.scratch_page, sizeof (struct drm_gem_object));
1450N/A return (-ENOMEM);
1450N/A }
1450N/A (void) memset(dev_priv->gtt.scratch_page->kaddr, 0, DRM_PAGE_SIZE);
1450N/A
1450N/A return 0;
1450N/A}
1450N/A
1450N/Avoid teardown_scratch_page(struct drm_device *dev)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A
1450N/A drm_gem_object_release(dev_priv->gtt.scratch_page);
1450N/A kmem_free(dev_priv->gtt.scratch_page, sizeof (struct drm_gem_object));
1450N/A}
1450N/A
1450N/Astatic inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
1450N/A{
1450N/A snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
1450N/A snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
1450N/A return snb_gmch_ctl << 20;
1450N/A}
1450N/A
1450N/Astatic inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
1450N/A{
1450N/A snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
1450N/A snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
1450N/A return snb_gmch_ctl << 25; /* 32 MB units */
1450N/A}
1450N/A
1450N/Astatic ddi_device_acc_attr_t gtt_attr = {
1450N/A DDI_DEVICE_ATTR_V0,
1450N/A DDI_NEVERSWAP_ACC,
1450N/A DDI_STRICTORDER_ACC,
1450N/A};
1450N/A
1450N/A#define GEN6_GTTMMADR 1
1450N/A#define GEN6_GTT_OFFSET 0x200000
1450N/A
1450N/Astatic void i915_gen6_gtt_ioremap(struct drm_local_map *map, struct drm_device *dev)
1450N/A{
1450N/A caddr_t base;
1450N/A int ret;
1450N/A
1450N/A ret = ddi_regs_map_setup(dev->devinfo, GEN6_GTTMMADR,
1450N/A (caddr_t *)&base, 0, 0,
1450N/A &gtt_attr, &map->acc_handle);
1450N/A
1450N/A if (ret != DDI_SUCCESS) {
1450N/A DRM_ERROR("failed to map GTT");
1450N/A }
1450N/A
1450N/A map->handle = (void *)(base + GEN6_GTT_OFFSET);
1450N/A}
1450N/A
1450N/Astatic int gen6_gmch_probe(struct drm_device *dev,
1450N/A size_t *gtt_total,
1450N/A size_t *stolen)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A unsigned int gtt_size;
1450N/A u16 snb_gmch_ctl;
1450N/A
1450N/A dev_priv->gtt.mappable_end = pci_resource_len(dev->pdev, 1);
1450N/A
1450N/A /* 64/512MB is the current min/max we actually know of, but this is just
1450N/A * a coarse sanity check.
1450N/A */
1450N/A if ((dev_priv->gtt.mappable_end < (64<<20) ||
1450N/A (dev_priv->gtt.mappable_end > (512<<20)))) {
1450N/A DRM_ERROR("Unknown GMADR size (%lx)\n",
1450N/A dev_priv->gtt.mappable_end);
1450N/A return -ENXIO;
1450N/A }
1450N/A
1450N/A pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1450N/A gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
1450N/A
1450N/A *stolen = gen6_get_stolen_size(snb_gmch_ctl);
1450N/A
1450N/A *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
1450N/A
1450N/A /* For GEN6+ the PTEs for the ggtt live at 2MB + BAR0 */
1450N/A dev_priv->gtt.gtt_mapping.offset = dev->agp_aperbase;
1450N/A dev_priv->gtt.gtt_mapping.size = gtt_size;
1450N/A dev_priv->gtt.gtt_mapping.type = 0;
1450N/A dev_priv->gtt.gtt_mapping.flags = 0;
1450N/A dev_priv->gtt.gtt_mapping.mtrr = 0;
1450N/A
1450N/A i915_gen6_gtt_ioremap(&dev_priv->gtt.gtt_mapping, dev);
1450N/A
1450N/A if (dev_priv->gtt.gtt_mapping.handle == NULL) {
1450N/A DRM_ERROR("Failed to map the gtt page table");
1450N/A return -ENOMEM;
1450N/A }
1450N/A dev_priv->gtt.virtual_gtt = (caddr_t)dev_priv->gtt.gtt_mapping.handle;
1450N/A
1450N/A dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range;
1450N/A dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries;
1450N/A
1450N/A return 0;
1450N/A}
1450N/A
1450N/Astatic void gen6_gmch_remove(struct drm_device *dev)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A drm_core_ioremapfree(&dev_priv->gtt.gtt_mapping, dev);
1450N/A}
1450N/A
1450N/Astatic unsigned int
1450N/Aintel_gtt_stolen_size(struct drm_device *dev)
1450N/A{
1450N/A u16 gmch_ctrl;
1450N/A u8 rdct;
1450N/A int local = 0;
1450N/A static const int ddt[4] = { 0, 16, 32, 64 };
1450N/A unsigned int stolen_size = 0;
1450N/A drm_i915_private_t *dev_priv = dev->dev_private;
1450N/A
1450N/A if ( INTEL_INFO(dev)->gen == 1)
1450N/A return 0; /* no stolen mem on i81x */
1450N/A
1450N/A if (i915_bridge_dev_read_config_word(&dev_priv->bridge_dev, INTEL_GMCH_CTRL, &gmch_ctrl))
1450N/A return 0;
1450N/A
1450N/A if (dev->pdev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
1450N/A dev->pdev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
1450N/A switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
1450N/A case I830_GMCH_GMS_STOLEN_512:
1450N/A stolen_size = KB(512);
1450N/A break;
1450N/A case I830_GMCH_GMS_STOLEN_1024:
1450N/A stolen_size = MB(1);
1450N/A break;
1450N/A case I830_GMCH_GMS_STOLEN_8192:
1450N/A stolen_size = MB(8);
1450N/A break;
1450N/A case I830_GMCH_GMS_LOCAL:
1450N/A rdct = I915_READ8(I830_RDRAM_CHANNEL_TYPE);
1450N/A stolen_size = (I830_RDRAM_ND(rdct) + 1) *
1450N/A MB(ddt[I830_RDRAM_DDT(rdct)]);
1450N/A local = 1;
1450N/A break;
1450N/A default:
1450N/A stolen_size = 0;
1450N/A break;
1450N/A }
1450N/A } else {
1450N/A switch (gmch_ctrl & INTEL_GMCH_GMS_MASK) {
1450N/A case INTEL_855_GMCH_GMS_STOLEN_1M:
1450N/A stolen_size = MB(1);
1450N/A break;
1450N/A case INTEL_855_GMCH_GMS_STOLEN_4M:
1450N/A stolen_size = MB(4);
1450N/A break;
1450N/A case INTEL_855_GMCH_GMS_STOLEN_8M:
1450N/A stolen_size = MB(8);
1450N/A break;
1450N/A case INTEL_855_GMCH_GMS_STOLEN_16M:
1450N/A stolen_size = MB(16);
1450N/A break;
1450N/A case INTEL_855_GMCH_GMS_STOLEN_32M:
1450N/A stolen_size = MB(32);
1450N/A break;
1450N/A case INTEL_915G_GMCH_GMS_STOLEN_48M:
1450N/A stolen_size = MB(48);
1450N/A break;
1450N/A case INTEL_915G_GMCH_GMS_STOLEN_64M:
1450N/A stolen_size = MB(64);
1450N/A break;
1450N/A case INTEL_GMCH_GMS_STOLEN_128M:
1450N/A stolen_size = MB(128);
1450N/A break;
1450N/A case INTEL_GMCH_GMS_STOLEN_256M:
1450N/A stolen_size = MB(256);
1450N/A break;
1450N/A case INTEL_GMCH_GMS_STOLEN_96M:
1450N/A stolen_size = MB(96);
1450N/A break;
1450N/A case INTEL_GMCH_GMS_STOLEN_160M:
1450N/A stolen_size = MB(160);
1450N/A break;
1450N/A case INTEL_GMCH_GMS_STOLEN_224M:
1450N/A stolen_size = MB(224);
1450N/A break;
1450N/A case INTEL_GMCH_GMS_STOLEN_352M:
1450N/A stolen_size = MB(352);
1450N/A break;
1450N/A default:
1450N/A stolen_size = 0;
1450N/A break;
1450N/A }
1450N/A }
1450N/A
1450N/A if (stolen_size > 0) {
1450N/A DRM_INFO("detected %dK %s memory\n",
1450N/A stolen_size / KB(1), local ? "local" : "stolen");
1450N/A } else {
1450N/A DRM_INFO("no pre-allocated video memory detected\n");
1450N/A stolen_size = 0;
1450N/A }
1450N/A
1450N/A return stolen_size;
1450N/A}
1450N/A
1450N/Astatic unsigned int
1450N/Aintel_gtt_mappable_entries(struct drm_device *dev)
1450N/A{
1450N/A unsigned int aperture_size;
1450N/A drm_i915_private_t *dev_priv = dev->dev_private;
1450N/A
1450N/A if (INTEL_INFO(dev)->gen == 1) {
1450N/A u32 smram_miscc;
1450N/A
1450N/A pci_read_config_dword(dev->pdev,
1450N/A I810_SMRAM_MISCC, &smram_miscc);
1450N/A
1450N/A if ((smram_miscc & I810_GFX_MEM_WIN_SIZE)
1450N/A == I810_GFX_MEM_WIN_32M)
1450N/A aperture_size = MB(32);
1450N/A else
1450N/A aperture_size = MB(64);
1450N/A } else if (INTEL_INFO(dev)->gen == 2) {
1450N/A u16 gmch_ctrl;
1450N/A
1450N/A if (i915_bridge_dev_read_config_word(&dev_priv->bridge_dev, INTEL_GMCH_CTRL, &gmch_ctrl))
1450N/A return 0;
1450N/A
1450N/A if ((gmch_ctrl & INTEL_GMCH_MEM_MASK) == INTEL_GMCH_MEM_64M)
1450N/A aperture_size = MB(64);
1450N/A else
1450N/A aperture_size = MB(128);
1450N/A } else {
1450N/A /* 9xx supports large sizes, just look at the length */
1450N/A aperture_size = pci_resource_len(dev->pdev,
1450N/A (IS_G33(dev) || IS_I945GM(dev)) ? 2 : 1); /* OSOL_i915 */
1450N/A DRM_DEBUG("aperture_size=%d", aperture_size);
1450N/A }
1450N/A
1450N/A return aperture_size >> PAGE_SHIFT;
1450N/A}
1450N/A
1450N/Astatic void
1450N/Ai965_adjust_pgetbl_size(struct drm_device *dev, unsigned int size_flag)
1450N/A{
1450N/A u32 pgetbl_ctl, pgetbl_ctl2;
1450N/A drm_i915_private_t *dev_priv = dev->dev_private;
1450N/A
1450N/A /* ensure that ppgtt is disabled */
1450N/A pgetbl_ctl2 = I915_READ(I965_PGETBL_CTL2);
1450N/A pgetbl_ctl2 &= ~I810_PGETBL_ENABLED;
1450N/A I915_WRITE(pgetbl_ctl2, I965_PGETBL_CTL2);
1450N/A
1450N/A /* write the new ggtt size */
1450N/A pgetbl_ctl = I915_READ(I810_PGETBL_CTL);
1450N/A pgetbl_ctl &= ~I965_PGETBL_SIZE_MASK;
1450N/A pgetbl_ctl |= size_flag;
1450N/A I915_WRITE(pgetbl_ctl, I810_PGETBL_CTL);
1450N/A}
1450N/A
1450N/Astatic unsigned int
1450N/Ai965_gtt_total_entries(struct drm_device *dev)
1450N/A{
1450N/A int size;
1450N/A u32 pgetbl_ctl;
1450N/A u16 gmch_ctl;
1450N/A drm_i915_private_t *dev_priv = dev->dev_private;
1450N/A
1450N/A if (i915_bridge_dev_read_config_word(&dev_priv->bridge_dev, INTEL_GMCH_CTRL, &gmch_ctl))
1450N/A return 0;
1450N/A
1450N/A if (INTEL_INFO(dev)->gen == 5) {
1450N/A switch (gmch_ctl & G4x_GMCH_SIZE_MASK) {
1450N/A case G4x_GMCH_SIZE_1M:
1450N/A case G4x_GMCH_SIZE_VT_1M:
1450N/A i965_adjust_pgetbl_size(dev, I965_PGETBL_SIZE_1MB);
1450N/A break;
1450N/A case G4x_GMCH_SIZE_VT_1_5M:
1450N/A i965_adjust_pgetbl_size(dev, I965_PGETBL_SIZE_1_5MB);
1450N/A break;
1450N/A case G4x_GMCH_SIZE_2M:
1450N/A case G4x_GMCH_SIZE_VT_2M:
1450N/A i965_adjust_pgetbl_size(dev, I965_PGETBL_SIZE_2MB);
1450N/A break;
1450N/A }
1450N/A }
1450N/A
1450N/A pgetbl_ctl = I915_READ(I810_PGETBL_CTL);
1450N/A
1450N/A switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
1450N/A case I965_PGETBL_SIZE_128KB:
1450N/A size = KB(128);
1450N/A break;
1450N/A case I965_PGETBL_SIZE_256KB:
1450N/A size = KB(256);
1450N/A break;
1450N/A case I965_PGETBL_SIZE_512KB:
1450N/A size = KB(512);
1450N/A break;
1450N/A /* GTT pagetable sizes bigger than 512KB are not possible on G33! */
1450N/A case I965_PGETBL_SIZE_1MB:
1450N/A size = KB(1024);
1450N/A break;
1450N/A case I965_PGETBL_SIZE_2MB:
1450N/A size = KB(2048);
1450N/A break;
1450N/A case I965_PGETBL_SIZE_1_5MB:
1450N/A size = KB(1024 + 512);
1450N/A break;
1450N/A default:
1450N/A DRM_DEBUG("unknown page table size, assuming 512KB");
1450N/A size = KB(512);
1450N/A }
1450N/A
1450N/A return size/4;
1450N/A}
1450N/A
1450N/Astatic unsigned int
1450N/Aintel_gtt_total_entries(struct drm_device *dev)
1450N/A{
1450N/A drm_i915_private_t *dev_priv = dev->dev_private;
1450N/A
1450N/A if (IS_G33(dev) || INTEL_INFO(dev)->gen == 4 || INTEL_INFO(dev)->gen == 5)
1450N/A return i965_gtt_total_entries(dev);
1450N/A else {
1450N/A /* On previous hardware, the GTT size was just what was
1450N/A * required to map the aperture.
1450N/A */
1450N/A return (dev_priv->gtt.mappable_end >> PAGE_SHIFT);
1450N/A }
1450N/A}
1450N/A
1450N/Astatic int i915_gmch_probe(struct drm_device *dev,
1450N/A size_t *gtt_total,
1450N/A size_t *stolen)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A
1450N/A dev_priv->gtt.mappable_end = intel_gtt_mappable_entries(dev) << PAGE_SHIFT;
1450N/A *gtt_total = intel_gtt_total_entries(dev) << PAGE_SHIFT;
1450N/A *stolen = intel_gtt_stolen_size(dev);
1450N/A
1450N/A dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range;
1450N/A dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries;
1450N/A
1450N/A return 0;
1450N/A}
1450N/A
1450N/Astatic void i915_gmch_remove(struct drm_device *dev)
1450N/A{
1450N/A
1450N/A}
1450N/A
1450N/Aint i915_gem_gtt_init(struct drm_device *dev)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A struct i915_gtt *gtt = &dev_priv->gtt;
1450N/A int ret;
1450N/A
1450N/A gtt->mappable_base = dev->agp_aperbase;
1450N/A
1450N/A if (INTEL_INFO(dev)->gen <= 5) {
1450N/A dev_priv->gtt.gtt_probe = i915_gmch_probe;
1450N/A dev_priv->gtt.gtt_remove = i915_gmch_remove;
1450N/A } else {
1450N/A dev_priv->gtt.gtt_probe = gen6_gmch_probe;
1450N/A dev_priv->gtt.gtt_remove = gen6_gmch_remove;
1450N/A if (IS_HASWELL(dev)) {
1450N/A dev_priv->gtt.pte_encode = hsw_pte_encode;
1450N/A } else if (IS_VALLEYVIEW(dev)) {
1450N/A dev_priv->gtt.pte_encode = byt_pte_encode;
1450N/A } else {
1450N/A dev_priv->gtt.pte_encode = gen6_pte_encode;
1450N/A }
1450N/A }
1450N/A
1450N/A ret = dev_priv->gtt.gtt_probe(dev, &dev_priv->gtt.total,
1450N/A &dev_priv->gtt.stolen_size);
1450N/A if (ret)
1450N/A return ret;
1450N/A
1450N/A /* GMADR is the PCI mmio aperture into the global GTT. */
1450N/A DRM_INFO("Memory usable by graphics device = %dM\n",
1450N/A dev_priv->gtt.total >> 20);
1450N/A DRM_DEBUG_DRIVER("GMADR size = %ldM\n",
1450N/A dev_priv->gtt.mappable_end >> 20);
1450N/A DRM_DEBUG_DRIVER("GTT stolen size = %dM\n",
1450N/A dev_priv->gtt.stolen_size >> 20);
1450N/A
1450N/A return 0;
1450N/A}
1450N/A
1450N/Avoid
1450N/Aintel_rw_gtt(struct drm_device *dev,
1450N/A size_t size,
1450N/A uint32_t gtt_offset,
1450N/A void *gttp,
1450N/A uint32_t type)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A unsigned first_entry = gtt_offset >> PAGE_SHIFT;
1450N/A unsigned num_entries = size / PAGE_SIZE;
1450N/A uint32_t *gtt_addr;
1450N/A uint32_t *old_gtt = (uint32_t *)gttp;
1450N/A int i, j;
1450N/A
1450N/A if (INTEL_INFO(dev)->gen <= 5) {
1450N/A (void) drm_agp_rw_gtt(dev, size/PAGE_SIZE, gtt_offset,
1450N/A gttp, type);
1450N/A } else {
1450N/A if (type) {
1450N/A for (i = first_entry, j = 0; i < ( first_entry + num_entries); i++, j++) {
1450N/A gtt_addr = (uint32_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt
1450N/A + i * sizeof(uint32_t));
1450N/A ddi_put32(dev_priv->gtt.gtt_mapping.acc_handle,
1450N/A gtt_addr, old_gtt[j]);
1450N/A }
1450N/A } else {
1450N/A for (i = first_entry; i < ( first_entry + num_entries); i++) {
1450N/A gtt_addr = (uint32_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt
1450N/A + i * sizeof(uint32_t));
1450N/A old_gtt[i] = ddi_get32(dev_priv->gtt.gtt_mapping.acc_handle,gtt_addr);
1450N/A }
1450N/A }
1450N/A }
1450N/A}
1450N/A
1450N/Avoid
1450N/Ai915_clean_gtt(struct drm_device *dev, size_t offset)
1450N/A{
1450N/A struct drm_i915_private *dev_priv = dev->dev_private;
1450N/A unsigned first_entry = offset >> PAGE_SHIFT;
1450N/A unsigned num_entries = (dev_priv->gtt.total - offset - PAGE_SIZE) >> PAGE_SHIFT;
1450N/A uint64_t scratch_page_addr = dev_priv->gtt.scratch_page->pfnarray[0] << PAGE_SHIFT;
1450N/A gtt_pte_t *gtt_addr, scratch_pte;
1450N/A int i;
1450N/A
1450N/A if (INTEL_INFO(dev)->gen <= 5) {
1450N/A (void) drm_agp_unbind_pages(dev, NULL, num_entries,
1450N/A offset, dev_priv->gtt.scratch_page->pfnarray[0] ,1);
1450N/A } else {
1450N/A for (i = first_entry ; i < ( first_entry + num_entries); i++) {
1450N/A gtt_addr = (gtt_pte_t *)(uintptr_t)((caddr_t)dev_priv->gtt.virtual_gtt
1450N/A + i * sizeof(gtt_pte_t));
1450N/A scratch_pte = gen6_pte_encode(dev, scratch_page_addr, I915_CACHE_LLC);
1450N/A ddi_put32(dev_priv->gtt.gtt_mapping.acc_handle,
1450N/A gtt_addr, scratch_pte);
1450N/A }
1450N/A }
1450N/A}
1450N/A