umem_genasm.c revision 4f364e7c95ee7fd9d5bbeddc1940e92405bb0e72
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2014 Joyent, Inc. All rights reserved.
*/
/*
* Don't Panic! If you find the blocks of assembly that follow confusing and
* you're questioning why they exist, please go read section 8 of the umem.c big
* theory statement. Next familiarize yourself with the malloc and free
* implementations in libumem's malloc.c.
*
* What follows is the i386 implementation of the thread caching automatic
* assembly generation. With i386 a function only has three registers it's
* allowed to change without restoring them: eax, ecx, and edx. All others have
* to be preserved. Since the set of registers we have available is so small, we
* have to make use of esi, ebx, and edi and save their original values to the
* stack.
*
* Malloc register usage:
* o. esi: Size of the malloc (passed into us and modified)
* o. edi: Size of the cache
* o. eax: Buffer to return
* o. ebx: Scratch space and temporary values
* o. ecx: Pointer to the tmem_t in the ulwp_t.
* o. edx: Pointer to the tmem_t array of roots
*
* Free register usage:
* o. esi: Size of the malloc (passed into us and modified)
* o. edi: Size of the cache
* o. eax: Buffer to free
* o. ebx: Scratch space and temporary values
* o. ecx: Pointer to the tmem_t in the ulwp_t.
* o. edx: Pointer to the tmem_t array of roots
*
* Once we determine what cache we are using, we increment %edx to the
* appropriate offset and set %edi with the size of the cache. This means that
* when we break out to the normal buffer allocation point %edx contains the
* head of the linked list and %edi is the amount that we have to adjust the
* total amount cached by the thread.
*
* Each block of assembly has psuedocode that describes its purpose.
*/
#include <inttypes.h>
#include <strings.h>
#include <umem_impl.h>
#include "umem_base.h"
#include <atomic.h>
const int umem_genasm_supported = 1;
/*
* The maximum number of caches we can support. We use a single byte addl so
* this is 255 (UINT8_MAX) / sizeof (uintptr_t). In this case 63
*/
#define UMEM_GENASM_MAX32 63
#define PTC_ROOT_SIZE sizeof (uintptr_t)
#define MULTINOP 0x0000441f0f
/*
* void *ptcmalloc(size_t orig_size);
*
* size_t size = orig_size + 8;
*
* if (size < orig_size)
* goto tomalloc; ! This is overflow
*
* if (size > cache_size)
* goto tomalloc;
*
* tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
* void **roots = t->tm_roots;
*/
#define PTC_MALINIT_JOUT 0x0e
#define PTC_MALINIT_MCS 0x14
#define PTC_MALINIT_JOV 0x1a
#define PTC_MALINIT_SOFF 0x27
0x55, /* pushl %ebp */
0x89, 0xe5, /* movl %esp, %ebp */
0x57, /* pushl %edi */
0x56, /* pushl %esi */
0x53, /* pushl %ebx */
0x8b, 0x75, 0x08, /* movl 0x8(%ebp), %esi */
0x83, 0xc6, 0x08, /* addl $0x8,%esi */
0x0f, 0x82, 0x00, 0x00, 0x00, 0x00, /* jc +$JMP (errout) */
0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, /* cmpl sizeof ($C0), %esi */
0x0f, 0x87, 0x00, 0x00, 0x00, 0x00, /* ja +$JMP (errout) */
0x65, 0x8b, 0x0d, 0x00, 0x00, 0x00, 0x00, /* movl %gs:0x0,%ecx */
0x81, 0xc1, 0x00, 0x00, 0x00, 0x00, /* addl $OFF, %ecx */
0x8d, 0x51, 0x04 /* leal 0x4(%ecx), %edx */
};
/*
* void ptcfree(void *buf);
*
* if (buf == NULL)
* return;
*
* malloc_data_t *tag = buf;
* tag--;
* int size = tag->malloc_size;
* int tagtval = UMEM_MALLOC_DECODE(tag->malloc_tag, size);
*
* if (tagval != MALLOC_MAGIC)
* goto tofree;
*
* if (size > cache_max)
* goto tofree;
*
* tmem_t *t = (uintptr_t)curthread() + umem_thr_offset;
* void **roots = t->tm_roots;
*/
#define PTC_FRINI_JDONE 0x0d
#define PTC_FRINI_JFREE 0x23
#define PTC_FRINI_MCS 0x29
#define PTC_FRINI_JOV 0x2f
#define PTC_FRINI_SOFF 0x3c
0x55, /* pushl %ebp */
0x89, 0xe5, /* movl %esp, %ebp */
0x57, /* pushl %edi */
0x56, /* pushl %esi */
0x53, /* pushl %ebx */
0x8b, 0x45, 0x08, /* movl 0x8(%ebp), %eax */
0x85, 0xc0, /* testl %eax, %eax */
0x0f, 0x84, 0x00, 0x00, 0x00, 0x00, /* je $JDONE (done) */
0x83, 0xe8, 0x08, /* subl $0x8,%eax */
0x8b, 0x30, /* movl (%eax),%esi */
0x8b, 0x50, 0x04, /* movl 0x4(%eax),%edx */
0x01, 0xf2, /* addl %esi,%edx */
0x81, 0xfa, 0x00, 0xc0, 0x10, 0x3a, /* cmpl MAGIC32, %edx */
0x0f, 0x85, 0x00, 0x00, 0x00, 0x00, /* jne +JFREE (goto freebuf) */
0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, /* cmpl sizeof ($C0), %esi */
0x0f, 0x87, 0x00, 0x00, 0x00, 0x00, /* ja +$JMP (errout) */
0x65, 0x8b, 0x0d, 0x00, 0x0, 0x00, 0x00, /* movl %gs:0x0,%ecx */
0x81, 0xc1, 0x00, 0x00, 0x00, 0x00, /* addl $0xOFF, %ecx */
0x8d, 0x51, 0x04 /* leal 0x4(%ecx),%edx */
};
/*
* if (size <= $CACHE_SIZE) {
* csize = $CACHE_SIZE;
* } else ... ! goto next cache
*/
#define PTC_INICACHE_CMP 0x02
#define PTC_INICACHE_SIZE 0x09
#define PTC_INICACHE_JMP 0x0e
0x81, 0xfe, 0xff, 0x00, 0x00, 0x00, /* cmpl sizeof ($C0), %esi */
0x77, 0x0a, /* ja +0xa */
0xbf, 0xff, 0x00, 0x00, 0x00, /* movl sizeof ($C0), %edi */
0xe9, 0x00, 0x00, 0x00, 0x00 /* jmp +$JMP (allocbuf) */
};
/*
* if (size <= $CACHE_SIZE) {
* csize = $CACHE_SIZE;
* roots += $CACHE_NUM;
* } else ... ! goto next cache
*/
#define PTC_GENCACHE_CMP 0x02
#define PTC_GENCACHE_NUM 0x0a
#define PTC_GENCACHE_SIZE 0x0c
#define PTC_GENCACHE_JMP 0x11
0x81, 0xfe, 0x00, 0x00, 0x00, 0x00, /* cmpl sizeof ($CACHE), %esi */
0x77, 0x0d, /* ja +0xd (next cache) */
0x83, 0xc2, 0x00, /* addl $4*$ii, %edx */
0xbf, 0x00, 0x00, 0x00, 0x00, /* movl sizeof ($CACHE), %edi */
0xe9, 0x00, 0x00, 0x00, 0x00 /* jmp +$JMP (allocbuf) */
};
/*
* else if (size <= $CACHE_SIZE) {
* csize = $CACHE_SIZE;
* roots += $CACHE_NUM;
* } else {
* goto tofunc; ! goto tomalloc if ptcmalloc.
* } ! goto tofree if ptcfree.
*/
#define PTC_FINCACHE_CMP 0x02
#define PTC_FINCACHE_JMP 0x07
#define PTC_FINCACHE_NUM 0x0a
#define PTC_FINCACHE_SIZE 0x0c
0x81, 0xfe, 0xff, 0x00, 0x00, 0x00, /* cmpl sizeof ($CLAST), %esi */
0x77, 0x00, /* ja +$JMP (to errout) */
0x83, 0xc2, 0x00, /* addl $4*($NCACHES-1), %edx */
0xbf, 0x00, 0x00, 0x00, 0x00, /* movl sizeof ($CLAST), %edi */
};
/*
* if (*root == NULL)
* goto tomalloc;
*
* malloc_data_t *ret = *root;
* *root = *(void **)ret;
* t->tm_size += csize;
* ret->malloc_size = size;
*
* ret->malloc_data = UMEM_MALLOC_ENCODE(MALLOC_SECOND_MAGIC, size);
* ret++;
*
* return ((void *)ret);
* tomalloc:
* return (malloc(orig_size));
*/
#define PTC_MALFINI_ALLABEL 0x00
#define PTC_MALFINI_JMLABEL 0x20
#define PTC_MALFINI_JMADDR 0x25
/* allocbuf: */
0x8b, 0x02, /* movl (%edx), %eax */
0x85, 0xc0, /* testl %eax, %eax */
0x74, 0x1a, /* je +0x1a (errout) */
0x8b, 0x18, /* movl (%eax), %esi */
0x89, 0x1a, /* movl %esi, (%edx) */
0x29, 0x39, /* subl %edi, (%ecx) */
0x89, 0x30, /* movl %esi, ($eax) */
0xba, 0x00, 0xc0, 0x10, 0x3a, /* movl $0x3a10c000,%edx */
0x29, 0xf2, /* subl %esi, %edx */
0x89, 0x50, 0x04, /* movl %edx, 0x4(%eax) */
0x83, 0xc0, 0x08, /* addl %0x8, %eax */
0x5b, /* popl %ebx */
0x5e, /* popl %esi */
0x5f, /* popl %edi */
0xc9, /* leave */
0xc3, /* ret */
/* errout: */
0x5b, /* popl %ebx */
0x5e, /* popl %esi */
0x5f, /* popl %edi */
0xc9, /* leave */
0xe9, 0x00, 0x00, 0x00, 0x00 /* jmp $malloc */
};
/*
* if (t->tm_size + csize > umem_ptc_size)
* goto tofree;
*
* t->tm_size += csize
* *(void **)tag = *root;
* *root = tag;
* return;
* tofree:
* free(buf);
* return;
*/
#define PTC_FRFINI_RBUFLABEL 0x00
#define PTC_FRFINI_CACHEMAX 0x06
#define PTC_FRFINI_DONELABEL 0x14
#define PTC_FRFINI_JFLABEL 0x19
#define PTC_FRFINI_JFADDR 0x1e
/* freebuf: */
0x8b, 0x19, /* movl (%ecx),%ebx */
0x01, 0xfb, /* addl %edi,%ebx */
0x81, 0xfb, 0x00, 0x00, 0x00, 0x00, /* cmpl maxsize, %ebx */
0x73, 0x0d, /* jae +0xd <tofree> */
0x01, 0x39, /* addl %edi,(%ecx) */
0x8b, 0x3a, /* movl (%edx),%edi */
0x89, 0x38, /* movl %edi,(%eax) */
0x89, 0x02, /* movl %eax,(%edx) */
/* done: */
0x5b, /* popl %ebx */
0x5e, /* popl %esi */
0x5f, /* popl %edi */
0xc9, /* leave */
0xc3, /* ret */
/* realfree: */
0x5b, /* popl %ebx */
0x5e, /* popl %esi */
0x5f, /* popl %edi */
0xc9, /* leave */
0xe9, 0x00, 0x00, 0x00, 0x00 /* jmp free */
};
/*
* Construct the initial part of malloc. off contains the offset from curthread
* to the root of the tmem structure. ep is the address of the label to error
* and jump to free. csize is the size of the largest umem_cache in ptcumem.
*/
static int
{
return (sizeof (malinit));
}
static int
{
return (sizeof (freeinit));
}
/*
* Create the initial cache entry of the specified size. The value of ap tells
* us what the address of the label to try and allocate a buffer. This value is
* an offset from the current base to that value.
*/
static int
{
return (sizeof (inicache));
}
static int
{
return (sizeof (gencache));
}
static int
{
return (sizeof (fincache));
}
static int
{
return (sizeof (malfini));
}
static int
{
return (sizeof (freefini));
}
/*
* The malloc inline assembly is constructed as follows:
*
* o Malloc prologue assembly
* o Generic first-cache check
* o n Generic cache checks (where n = _tmem_get_entries() - 2)
* o Generic last-cache check
* o Malloc epilogue assembly
*
* Generally there are at least three caches. When there is only one cache we
* only use the generic last-cache. In the case where there are two caches, we
* just leave out the middle ones.
*/
static int
{
if (nents >= 2)
return (1);
if (nents > 1) {
}
}
erroff);
return (0);
}
static int
{
/* Assume that nents has already been audited for us */
if (nents >= 2)
return (1);
if (nents > 1) {
}
}
erroff);
return (0);
}
int
{
int nents, i;
if (umem_genasm_mptr == 0 || umem_genasm_msize == 0 ||
umem_genasm_fptr == 0 || umem_genasm_fsize == 0)
return (1);
/*
* The total number of caches that we can service is the minimum of:
* o the amount supported by libc
* o the total number of umem caches
* o we use a single byte addl, so it's 255 / sizeof (uintptr_t). For
* 32-bit, this is 63.
*/
nents = _tmem_get_nentries();
if (UMEM_GENASM_MAX32 < nents)
/* Based on our constraints, this is not an error */
if (nents == 0 || umem_ptc_size == 0)
return (0);
/* Take into account the jump */
alloc_sizes) != 0)
return (1);
alloc_sizes) != 0)
return (1);
/* nop out the jump with a multibyte jump */
vptr = (void *)umem_genasm_mptr;
v = MULTINOP;
(void) atomic_swap_64(vptr, v);
vptr = (void *)umem_genasm_fptr;
v = MULTINOP;
(void) atomic_swap_64(vptr, v);
for (i = 0; i < nents; i++)
return (0);
}