/* "Bag-of-pages" garbage collector for the GNU compiler.
|
/* "Bag-of-pages" garbage collector for the GNU compiler.
|
Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2007
|
Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2007
|
Free Software Foundation, Inc.
|
Free Software Foundation, Inc.
|
|
|
This file is part of GCC.
|
This file is part of GCC.
|
|
|
GCC is free software; you can redistribute it and/or modify it under
|
GCC is free software; you can redistribute it and/or modify it under
|
the terms of the GNU General Public License as published by the Free
|
the terms of the GNU General Public License as published by the Free
|
Software Foundation; either version 3, or (at your option) any later
|
Software Foundation; either version 3, or (at your option) any later
|
version.
|
version.
|
|
|
GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
for more details.
|
for more details.
|
|
|
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
along with GCC; see the file COPYING3. If not see
|
along with GCC; see the file COPYING3. If not see
|
<http://www.gnu.org/licenses/>. */
|
<http://www.gnu.org/licenses/>. */
|
|
|
#include "config.h"
|
#include "config.h"
|
#include "system.h"
|
#include "system.h"
|
#include "coretypes.h"
|
#include "coretypes.h"
|
#include "tm.h"
|
#include "tm.h"
|
#include "tree.h"
|
#include "tree.h"
|
#include "rtl.h"
|
#include "rtl.h"
|
#include "tm_p.h"
|
#include "tm_p.h"
|
#include "toplev.h"
|
#include "toplev.h"
|
#include "flags.h"
|
#include "flags.h"
|
#include "ggc.h"
|
#include "ggc.h"
|
#include "timevar.h"
|
#include "timevar.h"
|
#include "params.h"
|
#include "params.h"
|
#include "tree-flow.h"
|
#include "tree-flow.h"
|
#ifdef ENABLE_VALGRIND_CHECKING
|
#ifdef ENABLE_VALGRIND_CHECKING
|
# ifdef HAVE_VALGRIND_MEMCHECK_H
|
# ifdef HAVE_VALGRIND_MEMCHECK_H
|
# include <valgrind/memcheck.h>
|
# include <valgrind/memcheck.h>
|
# elif defined HAVE_MEMCHECK_H
|
# elif defined HAVE_MEMCHECK_H
|
# include <memcheck.h>
|
# include <memcheck.h>
|
# else
|
# else
|
# include <valgrind.h>
|
# include <valgrind.h>
|
# endif
|
# endif
|
#else
|
#else
|
/* Avoid #ifdef:s when we can help it. */
|
/* Avoid #ifdef:s when we can help it. */
|
#define VALGRIND_DISCARD(x)
|
#define VALGRIND_DISCARD(x)
|
#endif
|
#endif
|
|
|
/* Prefer MAP_ANON(YMOUS) to /dev/zero, since we don't need to keep a
|
/* Prefer MAP_ANON(YMOUS) to /dev/zero, since we don't need to keep a
|
file open. Prefer either to valloc. */
|
file open. Prefer either to valloc. */
|
#ifdef HAVE_MMAP_ANON
|
#ifdef HAVE_MMAP_ANON
|
# undef HAVE_MMAP_DEV_ZERO
|
# undef HAVE_MMAP_DEV_ZERO
|
|
|
# include <sys/mman.h>
|
# include <sys/mman.h>
|
# ifndef MAP_FAILED
|
# ifndef MAP_FAILED
|
# define MAP_FAILED -1
|
# define MAP_FAILED -1
|
# endif
|
# endif
|
# if !defined (MAP_ANONYMOUS) && defined (MAP_ANON)
|
# if !defined (MAP_ANONYMOUS) && defined (MAP_ANON)
|
# define MAP_ANONYMOUS MAP_ANON
|
# define MAP_ANONYMOUS MAP_ANON
|
# endif
|
# endif
|
# define USING_MMAP
|
# define USING_MMAP
|
|
|
#endif
|
#endif
|
|
|
#ifdef HAVE_MMAP_DEV_ZERO
|
#ifdef HAVE_MMAP_DEV_ZERO
|
|
|
# include <sys/mman.h>
|
# include <sys/mman.h>
|
# ifndef MAP_FAILED
|
# ifndef MAP_FAILED
|
# define MAP_FAILED -1
|
# define MAP_FAILED -1
|
# endif
|
# endif
|
# define USING_MMAP
|
# define USING_MMAP
|
|
|
#endif
|
#endif
|
|
|
#ifndef USING_MMAP
|
#ifndef USING_MMAP
|
#define USING_MALLOC_PAGE_GROUPS
|
#define USING_MALLOC_PAGE_GROUPS
|
#endif
|
#endif
|
|
|
/* Strategy:
|
/* Strategy:
|
|
|
This garbage-collecting allocator allocates objects on one of a set
|
This garbage-collecting allocator allocates objects on one of a set
|
of pages. Each page can allocate objects of a single size only;
|
of pages. Each page can allocate objects of a single size only;
|
available sizes are powers of two starting at four bytes. The size
|
available sizes are powers of two starting at four bytes. The size
|
of an allocation request is rounded up to the next power of two
|
of an allocation request is rounded up to the next power of two
|
(`order'), and satisfied from the appropriate page.
|
(`order'), and satisfied from the appropriate page.
|
|
|
Each page is recorded in a page-entry, which also maintains an
|
Each page is recorded in a page-entry, which also maintains an
|
in-use bitmap of object positions on the page. This allows the
|
in-use bitmap of object positions on the page. This allows the
|
allocation state of a particular object to be flipped without
|
allocation state of a particular object to be flipped without
|
touching the page itself.
|
touching the page itself.
|
|
|
Each page-entry also has a context depth, which is used to track
|
Each page-entry also has a context depth, which is used to track
|
pushing and popping of allocation contexts. Only objects allocated
|
pushing and popping of allocation contexts. Only objects allocated
|
in the current (highest-numbered) context may be collected.
|
in the current (highest-numbered) context may be collected.
|
|
|
Page entries are arranged in an array of singly-linked lists. The
|
Page entries are arranged in an array of singly-linked lists. The
|
array is indexed by the allocation size, in bits, of the pages on
|
array is indexed by the allocation size, in bits, of the pages on
|
it; i.e. all pages on a list allocate objects of the same size.
|
it; i.e. all pages on a list allocate objects of the same size.
|
Pages are ordered on the list such that all non-full pages precede
|
Pages are ordered on the list such that all non-full pages precede
|
all full pages, with non-full pages arranged in order of decreasing
|
all full pages, with non-full pages arranged in order of decreasing
|
context depth.
|
context depth.
|
|
|
Empty pages (of all orders) are kept on a single page cache list,
|
Empty pages (of all orders) are kept on a single page cache list,
|
and are considered first when new pages are required; they are
|
and are considered first when new pages are required; they are
|
deallocated at the start of the next collection if they haven't
|
deallocated at the start of the next collection if they haven't
|
been recycled by then. */
|
been recycled by then. */
|
|
|
/* Define GGC_DEBUG_LEVEL to print debugging information.
|
/* Define GGC_DEBUG_LEVEL to print debugging information.
|
0: No debugging output.
|
0: No debugging output.
|
1: GC statistics only.
|
1: GC statistics only.
|
2: Page-entry allocations/deallocations as well.
|
2: Page-entry allocations/deallocations as well.
|
3: Object allocations as well.
|
3: Object allocations as well.
|
4: Object marks as well. */
|
4: Object marks as well. */
|
#define GGC_DEBUG_LEVEL (0)
|
#define GGC_DEBUG_LEVEL (0)
|
|
|
#ifndef HOST_BITS_PER_PTR
|
#ifndef HOST_BITS_PER_PTR
|
#define HOST_BITS_PER_PTR HOST_BITS_PER_LONG
|
#define HOST_BITS_PER_PTR HOST_BITS_PER_LONG
|
#endif
|
#endif
|
|
|
|
|
/* A two-level tree is used to look up the page-entry for a given
|
/* A two-level tree is used to look up the page-entry for a given
|
pointer. Two chunks of the pointer's bits are extracted to index
|
pointer. Two chunks of the pointer's bits are extracted to index
|
the first and second levels of the tree, as follows:
|
the first and second levels of the tree, as follows:
|
|
|
HOST_PAGE_SIZE_BITS
|
HOST_PAGE_SIZE_BITS
|
32 | |
|
32 | |
|
msb +----------------+----+------+------+ lsb
|
msb +----------------+----+------+------+ lsb
|
| | |
|
| | |
|
PAGE_L1_BITS |
|
PAGE_L1_BITS |
|
| |
|
| |
|
PAGE_L2_BITS
|
PAGE_L2_BITS
|
|
|
The bottommost HOST_PAGE_SIZE_BITS are ignored, since page-entry
|
The bottommost HOST_PAGE_SIZE_BITS are ignored, since page-entry
|
pages are aligned on system page boundaries. The next most
|
pages are aligned on system page boundaries. The next most
|
significant PAGE_L2_BITS and PAGE_L1_BITS are the second and first
|
significant PAGE_L2_BITS and PAGE_L1_BITS are the second and first
|
index values in the lookup table, respectively.
|
index values in the lookup table, respectively.
|
|
|
For 32-bit architectures and the settings below, there are no
|
For 32-bit architectures and the settings below, there are no
|
leftover bits. For architectures with wider pointers, the lookup
|
leftover bits. For architectures with wider pointers, the lookup
|
tree points to a list of pages, which must be scanned to find the
|
tree points to a list of pages, which must be scanned to find the
|
correct one. */
|
correct one. */
|
|
|
#define PAGE_L1_BITS (8)
|
#define PAGE_L1_BITS (8)
|
#define PAGE_L2_BITS (32 - PAGE_L1_BITS - G.lg_pagesize)
|
#define PAGE_L2_BITS (32 - PAGE_L1_BITS - G.lg_pagesize)
|
#define PAGE_L1_SIZE ((size_t) 1 << PAGE_L1_BITS)
|
#define PAGE_L1_SIZE ((size_t) 1 << PAGE_L1_BITS)
|
#define PAGE_L2_SIZE ((size_t) 1 << PAGE_L2_BITS)
|
#define PAGE_L2_SIZE ((size_t) 1 << PAGE_L2_BITS)
|
|
|
#define LOOKUP_L1(p) \
|
#define LOOKUP_L1(p) \
|
(((size_t) (p) >> (32 - PAGE_L1_BITS)) & ((1 << PAGE_L1_BITS) - 1))
|
(((size_t) (p) >> (32 - PAGE_L1_BITS)) & ((1 << PAGE_L1_BITS) - 1))
|
|
|
#define LOOKUP_L2(p) \
|
#define LOOKUP_L2(p) \
|
(((size_t) (p) >> G.lg_pagesize) & ((1 << PAGE_L2_BITS) - 1))
|
(((size_t) (p) >> G.lg_pagesize) & ((1 << PAGE_L2_BITS) - 1))
|
|
|
/* The number of objects per allocation page, for objects on a page of
|
/* The number of objects per allocation page, for objects on a page of
|
the indicated ORDER. */
|
the indicated ORDER. */
|
#define OBJECTS_PER_PAGE(ORDER) objects_per_page_table[ORDER]
|
#define OBJECTS_PER_PAGE(ORDER) objects_per_page_table[ORDER]
|
|
|
/* The number of objects in P. */
|
/* The number of objects in P. */
|
#define OBJECTS_IN_PAGE(P) ((P)->bytes / OBJECT_SIZE ((P)->order))
|
#define OBJECTS_IN_PAGE(P) ((P)->bytes / OBJECT_SIZE ((P)->order))
|
|
|
/* The size of an object on a page of the indicated ORDER. */
|
/* The size of an object on a page of the indicated ORDER. */
|
#define OBJECT_SIZE(ORDER) object_size_table[ORDER]
|
#define OBJECT_SIZE(ORDER) object_size_table[ORDER]
|
|
|
/* For speed, we avoid doing a general integer divide to locate the
|
/* For speed, we avoid doing a general integer divide to locate the
|
offset in the allocation bitmap, by precalculating numbers M, S
|
offset in the allocation bitmap, by precalculating numbers M, S
|
such that (O * M) >> S == O / Z (modulo 2^32), for any offset O
|
such that (O * M) >> S == O / Z (modulo 2^32), for any offset O
|
within the page which is evenly divisible by the object size Z. */
|
within the page which is evenly divisible by the object size Z. */
|
#define DIV_MULT(ORDER) inverse_table[ORDER].mult
|
#define DIV_MULT(ORDER) inverse_table[ORDER].mult
|
#define DIV_SHIFT(ORDER) inverse_table[ORDER].shift
|
#define DIV_SHIFT(ORDER) inverse_table[ORDER].shift
|
#define OFFSET_TO_BIT(OFFSET, ORDER) \
|
#define OFFSET_TO_BIT(OFFSET, ORDER) \
|
(((OFFSET) * DIV_MULT (ORDER)) >> DIV_SHIFT (ORDER))
|
(((OFFSET) * DIV_MULT (ORDER)) >> DIV_SHIFT (ORDER))
|
|
|
/* The number of extra orders, not corresponding to power-of-two sized
|
/* The number of extra orders, not corresponding to power-of-two sized
|
objects. */
|
objects. */
|
|
|
#define NUM_EXTRA_ORDERS ARRAY_SIZE (extra_order_size_table)
|
#define NUM_EXTRA_ORDERS ARRAY_SIZE (extra_order_size_table)
|
|
|
#define RTL_SIZE(NSLOTS) \
|
#define RTL_SIZE(NSLOTS) \
|
(RTX_HDR_SIZE + (NSLOTS) * sizeof (rtunion))
|
(RTX_HDR_SIZE + (NSLOTS) * sizeof (rtunion))
|
|
|
#define TREE_EXP_SIZE(OPS) \
|
#define TREE_EXP_SIZE(OPS) \
|
(sizeof (struct tree_exp) + ((OPS) - 1) * sizeof (tree))
|
(sizeof (struct tree_exp) + ((OPS) - 1) * sizeof (tree))
|
|
|
/* The Ith entry is the maximum size of an object to be stored in the
|
/* The Ith entry is the maximum size of an object to be stored in the
|
Ith extra order. Adding a new entry to this array is the *only*
|
Ith extra order. Adding a new entry to this array is the *only*
|
thing you need to do to add a new special allocation size. */
|
thing you need to do to add a new special allocation size. */
|
|
|
static const size_t extra_order_size_table[] = {
|
static const size_t extra_order_size_table[] = {
|
sizeof (struct stmt_ann_d),
|
sizeof (struct stmt_ann_d),
|
sizeof (struct var_ann_d),
|
sizeof (struct var_ann_d),
|
sizeof (struct tree_decl_non_common),
|
sizeof (struct tree_decl_non_common),
|
sizeof (struct tree_field_decl),
|
sizeof (struct tree_field_decl),
|
sizeof (struct tree_parm_decl),
|
sizeof (struct tree_parm_decl),
|
sizeof (struct tree_var_decl),
|
sizeof (struct tree_var_decl),
|
sizeof (struct tree_list),
|
sizeof (struct tree_list),
|
sizeof (struct tree_ssa_name),
|
sizeof (struct tree_ssa_name),
|
sizeof (struct function),
|
sizeof (struct function),
|
sizeof (struct basic_block_def),
|
sizeof (struct basic_block_def),
|
sizeof (bitmap_element),
|
sizeof (bitmap_element),
|
/* PHI nodes with one to three arguments are already covered by the
|
/* PHI nodes with one to three arguments are already covered by the
|
above sizes. */
|
above sizes. */
|
sizeof (struct tree_phi_node) + sizeof (struct phi_arg_d) * 3,
|
sizeof (struct tree_phi_node) + sizeof (struct phi_arg_d) * 3,
|
TREE_EXP_SIZE (2),
|
TREE_EXP_SIZE (2),
|
RTL_SIZE (2), /* MEM, PLUS, etc. */
|
RTL_SIZE (2), /* MEM, PLUS, etc. */
|
RTL_SIZE (9), /* INSN */
|
RTL_SIZE (9), /* INSN */
|
};
|
};
|
|
|
/* The total number of orders. */
|
/* The total number of orders. */
|
|
|
#define NUM_ORDERS (HOST_BITS_PER_PTR + NUM_EXTRA_ORDERS)
|
#define NUM_ORDERS (HOST_BITS_PER_PTR + NUM_EXTRA_ORDERS)
|
|
|
/* We use this structure to determine the alignment required for
|
/* We use this structure to determine the alignment required for
|
allocations. For power-of-two sized allocations, that's not a
|
allocations. For power-of-two sized allocations, that's not a
|
problem, but it does matter for odd-sized allocations. */
|
problem, but it does matter for odd-sized allocations. */
|
|
|
struct max_alignment {
|
struct max_alignment {
|
char c;
|
char c;
|
union {
|
union {
|
HOST_WIDEST_INT i;
|
HOST_WIDEST_INT i;
|
long double d;
|
long double d;
|
} u;
|
} u;
|
};
|
};
|
|
|
/* The biggest alignment required. */
|
/* The biggest alignment required. */
|
|
|
#define MAX_ALIGNMENT (offsetof (struct max_alignment, u))
|
#define MAX_ALIGNMENT (offsetof (struct max_alignment, u))
|
|
|
/* Compute the smallest nonnegative number which when added to X gives
|
/* Compute the smallest nonnegative number which when added to X gives
|
a multiple of F. */
|
a multiple of F. */
|
|
|
#define ROUND_UP_VALUE(x, f) ((f) - 1 - ((f) - 1 + (x)) % (f))
|
#define ROUND_UP_VALUE(x, f) ((f) - 1 - ((f) - 1 + (x)) % (f))
|
|
|
/* Compute the smallest multiple of F that is >= X. */
|
/* Compute the smallest multiple of F that is >= X. */
|
|
|
#define ROUND_UP(x, f) (CEIL (x, f) * (f))
|
#define ROUND_UP(x, f) (CEIL (x, f) * (f))
|
|
|
/* The Ith entry is the number of objects on a page or order I. */
|
/* The Ith entry is the number of objects on a page or order I. */
|
|
|
static unsigned objects_per_page_table[NUM_ORDERS];
|
static unsigned objects_per_page_table[NUM_ORDERS];
|
|
|
/* The Ith entry is the size of an object on a page of order I. */
|
/* The Ith entry is the size of an object on a page of order I. */
|
|
|
static size_t object_size_table[NUM_ORDERS];
|
static size_t object_size_table[NUM_ORDERS];
|
|
|
/* The Ith entry is a pair of numbers (mult, shift) such that
|
/* The Ith entry is a pair of numbers (mult, shift) such that
|
((k * mult) >> shift) mod 2^32 == (k / OBJECT_SIZE(I)) mod 2^32,
|
((k * mult) >> shift) mod 2^32 == (k / OBJECT_SIZE(I)) mod 2^32,
|
for all k evenly divisible by OBJECT_SIZE(I). */
|
for all k evenly divisible by OBJECT_SIZE(I). */
|
|
|
static struct
|
static struct
|
{
|
{
|
size_t mult;
|
size_t mult;
|
unsigned int shift;
|
unsigned int shift;
|
}
|
}
|
inverse_table[NUM_ORDERS];
|
inverse_table[NUM_ORDERS];
|
|
|
/* A page_entry records the status of an allocation page. This
|
/* A page_entry records the status of an allocation page. This
|
structure is dynamically sized to fit the bitmap in_use_p. */
|
structure is dynamically sized to fit the bitmap in_use_p. */
|
typedef struct page_entry
|
typedef struct page_entry
|
{
|
{
|
/* The next page-entry with objects of the same size, or NULL if
|
/* The next page-entry with objects of the same size, or NULL if
|
this is the last page-entry. */
|
this is the last page-entry. */
|
struct page_entry *next;
|
struct page_entry *next;
|
|
|
/* The previous page-entry with objects of the same size, or NULL if
|
/* The previous page-entry with objects of the same size, or NULL if
|
this is the first page-entry. The PREV pointer exists solely to
|
this is the first page-entry. The PREV pointer exists solely to
|
keep the cost of ggc_free manageable. */
|
keep the cost of ggc_free manageable. */
|
struct page_entry *prev;
|
struct page_entry *prev;
|
|
|
/* The number of bytes allocated. (This will always be a multiple
|
/* The number of bytes allocated. (This will always be a multiple
|
of the host system page size.) */
|
of the host system page size.) */
|
size_t bytes;
|
size_t bytes;
|
|
|
/* The address at which the memory is allocated. */
|
/* The address at which the memory is allocated. */
|
char *page;
|
char *page;
|
|
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
/* Back pointer to the page group this page came from. */
|
/* Back pointer to the page group this page came from. */
|
struct page_group *group;
|
struct page_group *group;
|
#endif
|
#endif
|
|
|
/* This is the index in the by_depth varray where this page table
|
/* This is the index in the by_depth varray where this page table
|
can be found. */
|
can be found. */
|
unsigned long index_by_depth;
|
unsigned long index_by_depth;
|
|
|
/* Context depth of this page. */
|
/* Context depth of this page. */
|
unsigned short context_depth;
|
unsigned short context_depth;
|
|
|
/* The number of free objects remaining on this page. */
|
/* The number of free objects remaining on this page. */
|
unsigned short num_free_objects;
|
unsigned short num_free_objects;
|
|
|
/* A likely candidate for the bit position of a free object for the
|
/* A likely candidate for the bit position of a free object for the
|
next allocation from this page. */
|
next allocation from this page. */
|
unsigned short next_bit_hint;
|
unsigned short next_bit_hint;
|
|
|
/* The lg of size of objects allocated from this page. */
|
/* The lg of size of objects allocated from this page. */
|
unsigned char order;
|
unsigned char order;
|
|
|
/* A bit vector indicating whether or not objects are in use. The
|
/* A bit vector indicating whether or not objects are in use. The
|
Nth bit is one if the Nth object on this page is allocated. This
|
Nth bit is one if the Nth object on this page is allocated. This
|
array is dynamically sized. */
|
array is dynamically sized. */
|
unsigned long in_use_p[1];
|
unsigned long in_use_p[1];
|
} page_entry;
|
} page_entry;
|
|
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
/* A page_group describes a large allocation from malloc, from which
|
/* A page_group describes a large allocation from malloc, from which
|
we parcel out aligned pages. */
|
we parcel out aligned pages. */
|
typedef struct page_group
|
typedef struct page_group
|
{
|
{
|
/* A linked list of all extant page groups. */
|
/* A linked list of all extant page groups. */
|
struct page_group *next;
|
struct page_group *next;
|
|
|
/* The address we received from malloc. */
|
/* The address we received from malloc. */
|
char *allocation;
|
char *allocation;
|
|
|
/* The size of the block. */
|
/* The size of the block. */
|
size_t alloc_size;
|
size_t alloc_size;
|
|
|
/* A bitmask of pages in use. */
|
/* A bitmask of pages in use. */
|
unsigned int in_use;
|
unsigned int in_use;
|
} page_group;
|
} page_group;
|
#endif
|
#endif
|
|
|
#if HOST_BITS_PER_PTR <= 32
|
#if HOST_BITS_PER_PTR <= 32
|
|
|
/* On 32-bit hosts, we use a two level page table, as pictured above. */
|
/* On 32-bit hosts, we use a two level page table, as pictured above. */
|
typedef page_entry **page_table[PAGE_L1_SIZE];
|
typedef page_entry **page_table[PAGE_L1_SIZE];
|
|
|
#else
|
#else
|
|
|
/* On 64-bit hosts, we use the same two level page tables plus a linked
|
/* On 64-bit hosts, we use the same two level page tables plus a linked
|
list that disambiguates the top 32-bits. There will almost always be
|
list that disambiguates the top 32-bits. There will almost always be
|
exactly one entry in the list. */
|
exactly one entry in the list. */
|
typedef struct page_table_chain
|
typedef struct page_table_chain
|
{
|
{
|
struct page_table_chain *next;
|
struct page_table_chain *next;
|
size_t high_bits;
|
size_t high_bits;
|
page_entry **table[PAGE_L1_SIZE];
|
page_entry **table[PAGE_L1_SIZE];
|
} *page_table;
|
} *page_table;
|
|
|
#endif
|
#endif
|
|
|
/* The rest of the global variables. */
|
/* The rest of the global variables. */
|
static struct globals
|
static struct globals
|
{
|
{
|
/* The Nth element in this array is a page with objects of size 2^N.
|
/* The Nth element in this array is a page with objects of size 2^N.
|
If there are any pages with free objects, they will be at the
|
If there are any pages with free objects, they will be at the
|
head of the list. NULL if there are no page-entries for this
|
head of the list. NULL if there are no page-entries for this
|
object size. */
|
object size. */
|
page_entry *pages[NUM_ORDERS];
|
page_entry *pages[NUM_ORDERS];
|
|
|
/* The Nth element in this array is the last page with objects of
|
/* The Nth element in this array is the last page with objects of
|
size 2^N. NULL if there are no page-entries for this object
|
size 2^N. NULL if there are no page-entries for this object
|
size. */
|
size. */
|
page_entry *page_tails[NUM_ORDERS];
|
page_entry *page_tails[NUM_ORDERS];
|
|
|
/* Lookup table for associating allocation pages with object addresses. */
|
/* Lookup table for associating allocation pages with object addresses. */
|
page_table lookup;
|
page_table lookup;
|
|
|
/* The system's page size. */
|
/* The system's page size. */
|
size_t pagesize;
|
size_t pagesize;
|
size_t lg_pagesize;
|
size_t lg_pagesize;
|
|
|
/* Bytes currently allocated. */
|
/* Bytes currently allocated. */
|
size_t allocated;
|
size_t allocated;
|
|
|
/* Bytes currently allocated at the end of the last collection. */
|
/* Bytes currently allocated at the end of the last collection. */
|
size_t allocated_last_gc;
|
size_t allocated_last_gc;
|
|
|
/* Total amount of memory mapped. */
|
/* Total amount of memory mapped. */
|
size_t bytes_mapped;
|
size_t bytes_mapped;
|
|
|
/* Bit N set if any allocations have been done at context depth N. */
|
/* Bit N set if any allocations have been done at context depth N. */
|
unsigned long context_depth_allocations;
|
unsigned long context_depth_allocations;
|
|
|
/* Bit N set if any collections have been done at context depth N. */
|
/* Bit N set if any collections have been done at context depth N. */
|
unsigned long context_depth_collections;
|
unsigned long context_depth_collections;
|
|
|
/* The current depth in the context stack. */
|
/* The current depth in the context stack. */
|
unsigned short context_depth;
|
unsigned short context_depth;
|
|
|
/* A file descriptor open to /dev/zero for reading. */
|
/* A file descriptor open to /dev/zero for reading. */
|
#if defined (HAVE_MMAP_DEV_ZERO)
|
#if defined (HAVE_MMAP_DEV_ZERO)
|
int dev_zero_fd;
|
int dev_zero_fd;
|
#endif
|
#endif
|
|
|
/* A cache of free system pages. */
|
/* A cache of free system pages. */
|
page_entry *free_pages;
|
page_entry *free_pages;
|
|
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
page_group *page_groups;
|
page_group *page_groups;
|
#endif
|
#endif
|
|
|
/* The file descriptor for debugging output. */
|
/* The file descriptor for debugging output. */
|
FILE *debug_file;
|
FILE *debug_file;
|
|
|
/* Current number of elements in use in depth below. */
|
/* Current number of elements in use in depth below. */
|
unsigned int depth_in_use;
|
unsigned int depth_in_use;
|
|
|
/* Maximum number of elements that can be used before resizing. */
|
/* Maximum number of elements that can be used before resizing. */
|
unsigned int depth_max;
|
unsigned int depth_max;
|
|
|
/* Each element of this arry is an index in by_depth where the given
|
/* Each element of this arry is an index in by_depth where the given
|
depth starts. This structure is indexed by that given depth we
|
depth starts. This structure is indexed by that given depth we
|
are interested in. */
|
are interested in. */
|
unsigned int *depth;
|
unsigned int *depth;
|
|
|
/* Current number of elements in use in by_depth below. */
|
/* Current number of elements in use in by_depth below. */
|
unsigned int by_depth_in_use;
|
unsigned int by_depth_in_use;
|
|
|
/* Maximum number of elements that can be used before resizing. */
|
/* Maximum number of elements that can be used before resizing. */
|
unsigned int by_depth_max;
|
unsigned int by_depth_max;
|
|
|
/* Each element of this array is a pointer to a page_entry, all
|
/* Each element of this array is a pointer to a page_entry, all
|
page_entries can be found in here by increasing depth.
|
page_entries can be found in here by increasing depth.
|
index_by_depth in the page_entry is the index into this data
|
index_by_depth in the page_entry is the index into this data
|
structure where that page_entry can be found. This is used to
|
structure where that page_entry can be found. This is used to
|
speed up finding all page_entries at a particular depth. */
|
speed up finding all page_entries at a particular depth. */
|
page_entry **by_depth;
|
page_entry **by_depth;
|
|
|
/* Each element is a pointer to the saved in_use_p bits, if any,
|
/* Each element is a pointer to the saved in_use_p bits, if any,
|
zero otherwise. We allocate them all together, to enable a
|
zero otherwise. We allocate them all together, to enable a
|
better runtime data access pattern. */
|
better runtime data access pattern. */
|
unsigned long **save_in_use;
|
unsigned long **save_in_use;
|
|
|
#ifdef ENABLE_GC_ALWAYS_COLLECT
|
#ifdef ENABLE_GC_ALWAYS_COLLECT
|
/* List of free objects to be verified as actually free on the
|
/* List of free objects to be verified as actually free on the
|
next collection. */
|
next collection. */
|
struct free_object
|
struct free_object
|
{
|
{
|
void *object;
|
void *object;
|
struct free_object *next;
|
struct free_object *next;
|
} *free_object_list;
|
} *free_object_list;
|
#endif
|
#endif
|
|
|
#ifdef GATHER_STATISTICS
|
#ifdef GATHER_STATISTICS
|
struct
|
struct
|
{
|
{
|
/* Total memory allocated with ggc_alloc. */
|
/* Total memory allocated with ggc_alloc. */
|
unsigned long long total_allocated;
|
unsigned long long total_allocated;
|
/* Total overhead for memory to be allocated with ggc_alloc. */
|
/* Total overhead for memory to be allocated with ggc_alloc. */
|
unsigned long long total_overhead;
|
unsigned long long total_overhead;
|
|
|
/* Total allocations and overhead for sizes less than 32, 64 and 128.
|
/* Total allocations and overhead for sizes less than 32, 64 and 128.
|
These sizes are interesting because they are typical cache line
|
These sizes are interesting because they are typical cache line
|
sizes. */
|
sizes. */
|
|
|
unsigned long long total_allocated_under32;
|
unsigned long long total_allocated_under32;
|
unsigned long long total_overhead_under32;
|
unsigned long long total_overhead_under32;
|
|
|
unsigned long long total_allocated_under64;
|
unsigned long long total_allocated_under64;
|
unsigned long long total_overhead_under64;
|
unsigned long long total_overhead_under64;
|
|
|
unsigned long long total_allocated_under128;
|
unsigned long long total_allocated_under128;
|
unsigned long long total_overhead_under128;
|
unsigned long long total_overhead_under128;
|
|
|
/* The allocations for each of the allocation orders. */
|
/* The allocations for each of the allocation orders. */
|
unsigned long long total_allocated_per_order[NUM_ORDERS];
|
unsigned long long total_allocated_per_order[NUM_ORDERS];
|
|
|
/* The overhead for each of the allocation orders. */
|
/* The overhead for each of the allocation orders. */
|
unsigned long long total_overhead_per_order[NUM_ORDERS];
|
unsigned long long total_overhead_per_order[NUM_ORDERS];
|
} stats;
|
} stats;
|
#endif
|
#endif
|
} G;
|
} G;
|
|
|
/* The size in bytes required to maintain a bitmap for the objects
|
/* The size in bytes required to maintain a bitmap for the objects
|
on a page-entry. */
|
on a page-entry. */
|
#define BITMAP_SIZE(Num_objects) \
|
#define BITMAP_SIZE(Num_objects) \
|
(CEIL ((Num_objects), HOST_BITS_PER_LONG) * sizeof(long))
|
(CEIL ((Num_objects), HOST_BITS_PER_LONG) * sizeof(long))
|
|
|
/* Allocate pages in chunks of this size, to throttle calls to memory
|
/* Allocate pages in chunks of this size, to throttle calls to memory
|
allocation routines. The first page is used, the rest go onto the
|
allocation routines. The first page is used, the rest go onto the
|
free list. This cannot be larger than HOST_BITS_PER_INT for the
|
free list. This cannot be larger than HOST_BITS_PER_INT for the
|
in_use bitmask for page_group. Hosts that need a different value
|
in_use bitmask for page_group. Hosts that need a different value
|
can override this by defining GGC_QUIRE_SIZE explicitly. */
|
can override this by defining GGC_QUIRE_SIZE explicitly. */
|
#ifndef GGC_QUIRE_SIZE
|
#ifndef GGC_QUIRE_SIZE
|
# ifdef USING_MMAP
|
# ifdef USING_MMAP
|
# define GGC_QUIRE_SIZE 256
|
# define GGC_QUIRE_SIZE 256
|
# else
|
# else
|
# define GGC_QUIRE_SIZE 16
|
# define GGC_QUIRE_SIZE 16
|
# endif
|
# endif
|
#endif
|
#endif
|
|
|
/* Initial guess as to how many page table entries we might need. */
|
/* Initial guess as to how many page table entries we might need. */
|
#define INITIAL_PTE_COUNT 128
|
#define INITIAL_PTE_COUNT 128
|
|
|
static int ggc_allocated_p (const void *);
|
static int ggc_allocated_p (const void *);
|
static page_entry *lookup_page_table_entry (const void *);
|
static page_entry *lookup_page_table_entry (const void *);
|
static void set_page_table_entry (void *, page_entry *);
|
static void set_page_table_entry (void *, page_entry *);
|
#ifdef USING_MMAP
|
#ifdef USING_MMAP
|
static char *alloc_anon (char *, size_t);
|
static char *alloc_anon (char *, size_t);
|
#endif
|
#endif
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
static size_t page_group_index (char *, char *);
|
static size_t page_group_index (char *, char *);
|
static void set_page_group_in_use (page_group *, char *);
|
static void set_page_group_in_use (page_group *, char *);
|
static void clear_page_group_in_use (page_group *, char *);
|
static void clear_page_group_in_use (page_group *, char *);
|
#endif
|
#endif
|
static struct page_entry * alloc_page (unsigned);
|
static struct page_entry * alloc_page (unsigned);
|
static void free_page (struct page_entry *);
|
static void free_page (struct page_entry *);
|
static void release_pages (void);
|
static void release_pages (void);
|
static void clear_marks (void);
|
static void clear_marks (void);
|
static void sweep_pages (void);
|
static void sweep_pages (void);
|
static void ggc_recalculate_in_use_p (page_entry *);
|
static void ggc_recalculate_in_use_p (page_entry *);
|
static void compute_inverse (unsigned);
|
static void compute_inverse (unsigned);
|
static inline void adjust_depth (void);
|
static inline void adjust_depth (void);
|
static void move_ptes_to_front (int, int);
|
static void move_ptes_to_front (int, int);
|
|
|
void debug_print_page_list (int);
|
void debug_print_page_list (int);
|
static void push_depth (unsigned int);
|
static void push_depth (unsigned int);
|
static void push_by_depth (page_entry *, unsigned long *);
|
static void push_by_depth (page_entry *, unsigned long *);
|
|
|
/* Push an entry onto G.depth. */
|
/* Push an entry onto G.depth. */
|
|
|
inline static void
|
inline static void
|
push_depth (unsigned int i)
|
push_depth (unsigned int i)
|
{
|
{
|
if (G.depth_in_use >= G.depth_max)
|
if (G.depth_in_use >= G.depth_max)
|
{
|
{
|
G.depth_max *= 2;
|
G.depth_max *= 2;
|
G.depth = xrealloc (G.depth, G.depth_max * sizeof (unsigned int));
|
G.depth = xrealloc (G.depth, G.depth_max * sizeof (unsigned int));
|
}
|
}
|
G.depth[G.depth_in_use++] = i;
|
G.depth[G.depth_in_use++] = i;
|
}
|
}
|
|
|
/* Push an entry onto G.by_depth and G.save_in_use. */
|
/* Push an entry onto G.by_depth and G.save_in_use. */
|
|
|
inline static void
|
inline static void
|
push_by_depth (page_entry *p, unsigned long *s)
|
push_by_depth (page_entry *p, unsigned long *s)
|
{
|
{
|
if (G.by_depth_in_use >= G.by_depth_max)
|
if (G.by_depth_in_use >= G.by_depth_max)
|
{
|
{
|
G.by_depth_max *= 2;
|
G.by_depth_max *= 2;
|
G.by_depth = xrealloc (G.by_depth,
|
G.by_depth = xrealloc (G.by_depth,
|
G.by_depth_max * sizeof (page_entry *));
|
G.by_depth_max * sizeof (page_entry *));
|
G.save_in_use = xrealloc (G.save_in_use,
|
G.save_in_use = xrealloc (G.save_in_use,
|
G.by_depth_max * sizeof (unsigned long *));
|
G.by_depth_max * sizeof (unsigned long *));
|
}
|
}
|
G.by_depth[G.by_depth_in_use] = p;
|
G.by_depth[G.by_depth_in_use] = p;
|
G.save_in_use[G.by_depth_in_use++] = s;
|
G.save_in_use[G.by_depth_in_use++] = s;
|
}
|
}
|
|
|
#if (GCC_VERSION < 3001)
|
#if (GCC_VERSION < 3001)
|
#define prefetch(X) ((void) X)
|
#define prefetch(X) ((void) X)
|
#else
|
#else
|
#define prefetch(X) __builtin_prefetch (X)
|
#define prefetch(X) __builtin_prefetch (X)
|
#endif
|
#endif
|
|
|
#define save_in_use_p_i(__i) \
|
#define save_in_use_p_i(__i) \
|
(G.save_in_use[__i])
|
(G.save_in_use[__i])
|
#define save_in_use_p(__p) \
|
#define save_in_use_p(__p) \
|
(save_in_use_p_i (__p->index_by_depth))
|
(save_in_use_p_i (__p->index_by_depth))
|
|
|
/* Returns nonzero if P was allocated in GC'able memory. */
|
/* Returns nonzero if P was allocated in GC'able memory. */
|
|
|
static inline int
|
static inline int
|
ggc_allocated_p (const void *p)
|
ggc_allocated_p (const void *p)
|
{
|
{
|
page_entry ***base;
|
page_entry ***base;
|
size_t L1, L2;
|
size_t L1, L2;
|
|
|
#if HOST_BITS_PER_PTR <= 32
|
#if HOST_BITS_PER_PTR <= 32
|
base = &G.lookup[0];
|
base = &G.lookup[0];
|
#else
|
#else
|
page_table table = G.lookup;
|
page_table table = G.lookup;
|
size_t high_bits = (size_t) p & ~ (size_t) 0xffffffff;
|
size_t high_bits = (size_t) p & ~ (size_t) 0xffffffff;
|
while (1)
|
while (1)
|
{
|
{
|
if (table == NULL)
|
if (table == NULL)
|
return 0;
|
return 0;
|
if (table->high_bits == high_bits)
|
if (table->high_bits == high_bits)
|
break;
|
break;
|
table = table->next;
|
table = table->next;
|
}
|
}
|
base = &table->table[0];
|
base = &table->table[0];
|
#endif
|
#endif
|
|
|
/* Extract the level 1 and 2 indices. */
|
/* Extract the level 1 and 2 indices. */
|
L1 = LOOKUP_L1 (p);
|
L1 = LOOKUP_L1 (p);
|
L2 = LOOKUP_L2 (p);
|
L2 = LOOKUP_L2 (p);
|
|
|
return base[L1] && base[L1][L2];
|
return base[L1] && base[L1][L2];
|
}
|
}
|
|
|
/* Traverse the page table and find the entry for a page.
|
/* Traverse the page table and find the entry for a page.
|
Die (probably) if the object wasn't allocated via GC. */
|
Die (probably) if the object wasn't allocated via GC. */
|
|
|
static inline page_entry *
|
static inline page_entry *
|
lookup_page_table_entry (const void *p)
|
lookup_page_table_entry (const void *p)
|
{
|
{
|
page_entry ***base;
|
page_entry ***base;
|
size_t L1, L2;
|
size_t L1, L2;
|
|
|
#if HOST_BITS_PER_PTR <= 32
|
#if HOST_BITS_PER_PTR <= 32
|
base = &G.lookup[0];
|
base = &G.lookup[0];
|
#else
|
#else
|
page_table table = G.lookup;
|
page_table table = G.lookup;
|
size_t high_bits = (size_t) p & ~ (size_t) 0xffffffff;
|
size_t high_bits = (size_t) p & ~ (size_t) 0xffffffff;
|
while (table->high_bits != high_bits)
|
while (table->high_bits != high_bits)
|
table = table->next;
|
table = table->next;
|
base = &table->table[0];
|
base = &table->table[0];
|
#endif
|
#endif
|
|
|
/* Extract the level 1 and 2 indices. */
|
/* Extract the level 1 and 2 indices. */
|
L1 = LOOKUP_L1 (p);
|
L1 = LOOKUP_L1 (p);
|
L2 = LOOKUP_L2 (p);
|
L2 = LOOKUP_L2 (p);
|
|
|
return base[L1][L2];
|
return base[L1][L2];
|
}
|
}
|
|
|
/* Set the page table entry for a page. */
|
/* Set the page table entry for a page. */
|
|
|
static void
|
static void
|
set_page_table_entry (void *p, page_entry *entry)
|
set_page_table_entry (void *p, page_entry *entry)
|
{
|
{
|
page_entry ***base;
|
page_entry ***base;
|
size_t L1, L2;
|
size_t L1, L2;
|
|
|
#if HOST_BITS_PER_PTR <= 32
|
#if HOST_BITS_PER_PTR <= 32
|
base = &G.lookup[0];
|
base = &G.lookup[0];
|
#else
|
#else
|
page_table table;
|
page_table table;
|
size_t high_bits = (size_t) p & ~ (size_t) 0xffffffff;
|
size_t high_bits = (size_t) p & ~ (size_t) 0xffffffff;
|
for (table = G.lookup; table; table = table->next)
|
for (table = G.lookup; table; table = table->next)
|
if (table->high_bits == high_bits)
|
if (table->high_bits == high_bits)
|
goto found;
|
goto found;
|
|
|
/* Not found -- allocate a new table. */
|
/* Not found -- allocate a new table. */
|
table = xcalloc (1, sizeof(*table));
|
table = xcalloc (1, sizeof(*table));
|
table->next = G.lookup;
|
table->next = G.lookup;
|
table->high_bits = high_bits;
|
table->high_bits = high_bits;
|
G.lookup = table;
|
G.lookup = table;
|
found:
|
found:
|
base = &table->table[0];
|
base = &table->table[0];
|
#endif
|
#endif
|
|
|
/* Extract the level 1 and 2 indices. */
|
/* Extract the level 1 and 2 indices. */
|
L1 = LOOKUP_L1 (p);
|
L1 = LOOKUP_L1 (p);
|
L2 = LOOKUP_L2 (p);
|
L2 = LOOKUP_L2 (p);
|
|
|
if (base[L1] == NULL)
|
if (base[L1] == NULL)
|
base[L1] = XCNEWVEC (page_entry *, PAGE_L2_SIZE);
|
base[L1] = XCNEWVEC (page_entry *, PAGE_L2_SIZE);
|
|
|
base[L1][L2] = entry;
|
base[L1][L2] = entry;
|
}
|
}
|
|
|
/* Prints the page-entry for object size ORDER, for debugging. */
|
/* Prints the page-entry for object size ORDER, for debugging. */
|
|
|
void
|
void
|
debug_print_page_list (int order)
|
debug_print_page_list (int order)
|
{
|
{
|
page_entry *p;
|
page_entry *p;
|
printf ("Head=%p, Tail=%p:\n", (void *) G.pages[order],
|
printf ("Head=%p, Tail=%p:\n", (void *) G.pages[order],
|
(void *) G.page_tails[order]);
|
(void *) G.page_tails[order]);
|
p = G.pages[order];
|
p = G.pages[order];
|
while (p != NULL)
|
while (p != NULL)
|
{
|
{
|
printf ("%p(%1d|%3d) -> ", (void *) p, p->context_depth,
|
printf ("%p(%1d|%3d) -> ", (void *) p, p->context_depth,
|
p->num_free_objects);
|
p->num_free_objects);
|
p = p->next;
|
p = p->next;
|
}
|
}
|
printf ("NULL\n");
|
printf ("NULL\n");
|
fflush (stdout);
|
fflush (stdout);
|
}
|
}
|
|
|
#ifdef USING_MMAP
|
#ifdef USING_MMAP
|
/* Allocate SIZE bytes of anonymous memory, preferably near PREF,
|
/* Allocate SIZE bytes of anonymous memory, preferably near PREF,
|
(if non-null). The ifdef structure here is intended to cause a
|
(if non-null). The ifdef structure here is intended to cause a
|
compile error unless exactly one of the HAVE_* is defined. */
|
compile error unless exactly one of the HAVE_* is defined. */
|
|
|
static inline char *
|
static inline char *
|
alloc_anon (char *pref ATTRIBUTE_UNUSED, size_t size)
|
alloc_anon (char *pref ATTRIBUTE_UNUSED, size_t size)
|
{
|
{
|
#ifdef HAVE_MMAP_ANON
|
#ifdef HAVE_MMAP_ANON
|
char *page = mmap (pref, size, PROT_READ | PROT_WRITE,
|
char *page = mmap (pref, size, PROT_READ | PROT_WRITE,
|
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
#endif
|
#endif
|
#ifdef HAVE_MMAP_DEV_ZERO
|
#ifdef HAVE_MMAP_DEV_ZERO
|
char *page = mmap (pref, size, PROT_READ | PROT_WRITE,
|
char *page = mmap (pref, size, PROT_READ | PROT_WRITE,
|
MAP_PRIVATE, G.dev_zero_fd, 0);
|
MAP_PRIVATE, G.dev_zero_fd, 0);
|
#endif
|
#endif
|
|
|
if (page == (char *) MAP_FAILED)
|
if (page == (char *) MAP_FAILED)
|
{
|
{
|
perror ("virtual memory exhausted");
|
perror ("virtual memory exhausted");
|
exit (FATAL_EXIT_CODE);
|
exit (FATAL_EXIT_CODE);
|
}
|
}
|
|
|
/* Remember that we allocated this memory. */
|
/* Remember that we allocated this memory. */
|
G.bytes_mapped += size;
|
G.bytes_mapped += size;
|
|
|
/* Pretend we don't have access to the allocated pages. We'll enable
|
/* Pretend we don't have access to the allocated pages. We'll enable
|
access to smaller pieces of the area in ggc_alloc. Discard the
|
access to smaller pieces of the area in ggc_alloc. Discard the
|
handle to avoid handle leak. */
|
handle to avoid handle leak. */
|
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (page, size));
|
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (page, size));
|
|
|
return page;
|
return page;
|
}
|
}
|
#endif
|
#endif
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
/* Compute the index for this page into the page group. */
|
/* Compute the index for this page into the page group. */
|
|
|
static inline size_t
|
static inline size_t
|
page_group_index (char *allocation, char *page)
|
page_group_index (char *allocation, char *page)
|
{
|
{
|
return (size_t) (page - allocation) >> G.lg_pagesize;
|
return (size_t) (page - allocation) >> G.lg_pagesize;
|
}
|
}
|
|
|
/* Set and clear the in_use bit for this page in the page group. */
|
/* Set and clear the in_use bit for this page in the page group. */
|
|
|
static inline void
|
static inline void
|
set_page_group_in_use (page_group *group, char *page)
|
set_page_group_in_use (page_group *group, char *page)
|
{
|
{
|
group->in_use |= 1 << page_group_index (group->allocation, page);
|
group->in_use |= 1 << page_group_index (group->allocation, page);
|
}
|
}
|
|
|
static inline void
|
static inline void
|
clear_page_group_in_use (page_group *group, char *page)
|
clear_page_group_in_use (page_group *group, char *page)
|
{
|
{
|
group->in_use &= ~(1 << page_group_index (group->allocation, page));
|
group->in_use &= ~(1 << page_group_index (group->allocation, page));
|
}
|
}
|
#endif
|
#endif
|
|
|
/* Allocate a new page for allocating objects of size 2^ORDER,
|
/* Allocate a new page for allocating objects of size 2^ORDER,
|
and return an entry for it. The entry is not added to the
|
and return an entry for it. The entry is not added to the
|
appropriate page_table list. */
|
appropriate page_table list. */
|
|
|
static inline struct page_entry *
|
static inline struct page_entry *
|
alloc_page (unsigned order)
|
alloc_page (unsigned order)
|
{
|
{
|
struct page_entry *entry, *p, **pp;
|
struct page_entry *entry, *p, **pp;
|
char *page;
|
char *page;
|
size_t num_objects;
|
size_t num_objects;
|
size_t bitmap_size;
|
size_t bitmap_size;
|
size_t page_entry_size;
|
size_t page_entry_size;
|
size_t entry_size;
|
size_t entry_size;
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
page_group *group;
|
page_group *group;
|
#endif
|
#endif
|
|
|
num_objects = OBJECTS_PER_PAGE (order);
|
num_objects = OBJECTS_PER_PAGE (order);
|
bitmap_size = BITMAP_SIZE (num_objects + 1);
|
bitmap_size = BITMAP_SIZE (num_objects + 1);
|
page_entry_size = sizeof (page_entry) - sizeof (long) + bitmap_size;
|
page_entry_size = sizeof (page_entry) - sizeof (long) + bitmap_size;
|
entry_size = num_objects * OBJECT_SIZE (order);
|
entry_size = num_objects * OBJECT_SIZE (order);
|
if (entry_size < G.pagesize)
|
if (entry_size < G.pagesize)
|
entry_size = G.pagesize;
|
entry_size = G.pagesize;
|
|
|
entry = NULL;
|
entry = NULL;
|
page = NULL;
|
page = NULL;
|
|
|
/* Check the list of free pages for one we can use. */
|
/* Check the list of free pages for one we can use. */
|
for (pp = &G.free_pages, p = *pp; p; pp = &p->next, p = *pp)
|
for (pp = &G.free_pages, p = *pp; p; pp = &p->next, p = *pp)
|
if (p->bytes == entry_size)
|
if (p->bytes == entry_size)
|
break;
|
break;
|
|
|
if (p != NULL)
|
if (p != NULL)
|
{
|
{
|
/* Recycle the allocated memory from this page ... */
|
/* Recycle the allocated memory from this page ... */
|
*pp = p->next;
|
*pp = p->next;
|
page = p->page;
|
page = p->page;
|
|
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
group = p->group;
|
group = p->group;
|
#endif
|
#endif
|
|
|
/* ... and, if possible, the page entry itself. */
|
/* ... and, if possible, the page entry itself. */
|
if (p->order == order)
|
if (p->order == order)
|
{
|
{
|
entry = p;
|
entry = p;
|
memset (entry, 0, page_entry_size);
|
memset (entry, 0, page_entry_size);
|
}
|
}
|
else
|
else
|
free (p);
|
free (p);
|
}
|
}
|
#ifdef USING_MMAP
|
#ifdef USING_MMAP
|
else if (entry_size == G.pagesize)
|
else if (entry_size == G.pagesize)
|
{
|
{
|
/* We want just one page. Allocate a bunch of them and put the
|
/* We want just one page. Allocate a bunch of them and put the
|
extras on the freelist. (Can only do this optimization with
|
extras on the freelist. (Can only do this optimization with
|
mmap for backing store.) */
|
mmap for backing store.) */
|
struct page_entry *e, *f = G.free_pages;
|
struct page_entry *e, *f = G.free_pages;
|
int i;
|
int i;
|
|
|
page = alloc_anon (NULL, G.pagesize * GGC_QUIRE_SIZE);
|
page = alloc_anon (NULL, G.pagesize * GGC_QUIRE_SIZE);
|
|
|
/* This loop counts down so that the chain will be in ascending
|
/* This loop counts down so that the chain will be in ascending
|
memory order. */
|
memory order. */
|
for (i = GGC_QUIRE_SIZE - 1; i >= 1; i--)
|
for (i = GGC_QUIRE_SIZE - 1; i >= 1; i--)
|
{
|
{
|
e = xcalloc (1, page_entry_size);
|
e = xcalloc (1, page_entry_size);
|
e->order = order;
|
e->order = order;
|
e->bytes = G.pagesize;
|
e->bytes = G.pagesize;
|
e->page = page + (i << G.lg_pagesize);
|
e->page = page + (i << G.lg_pagesize);
|
e->next = f;
|
e->next = f;
|
f = e;
|
f = e;
|
}
|
}
|
|
|
G.free_pages = f;
|
G.free_pages = f;
|
}
|
}
|
else
|
else
|
page = alloc_anon (NULL, entry_size);
|
page = alloc_anon (NULL, entry_size);
|
#endif
|
#endif
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
else
|
else
|
{
|
{
|
/* Allocate a large block of memory and serve out the aligned
|
/* Allocate a large block of memory and serve out the aligned
|
pages therein. This results in much less memory wastage
|
pages therein. This results in much less memory wastage
|
than the traditional implementation of valloc. */
|
than the traditional implementation of valloc. */
|
|
|
char *allocation, *a, *enda;
|
char *allocation, *a, *enda;
|
size_t alloc_size, head_slop, tail_slop;
|
size_t alloc_size, head_slop, tail_slop;
|
int multiple_pages = (entry_size == G.pagesize);
|
int multiple_pages = (entry_size == G.pagesize);
|
|
|
if (multiple_pages)
|
if (multiple_pages)
|
alloc_size = GGC_QUIRE_SIZE * G.pagesize;
|
alloc_size = GGC_QUIRE_SIZE * G.pagesize;
|
else
|
else
|
alloc_size = entry_size + G.pagesize - 1;
|
alloc_size = entry_size + G.pagesize - 1;
|
allocation = xmalloc (alloc_size);
|
allocation = xmalloc (alloc_size);
|
|
|
page = (char *) (((size_t) allocation + G.pagesize - 1) & -G.pagesize);
|
page = (char *) (((size_t) allocation + G.pagesize - 1) & -G.pagesize);
|
head_slop = page - allocation;
|
head_slop = page - allocation;
|
if (multiple_pages)
|
if (multiple_pages)
|
tail_slop = ((size_t) allocation + alloc_size) & (G.pagesize - 1);
|
tail_slop = ((size_t) allocation + alloc_size) & (G.pagesize - 1);
|
else
|
else
|
tail_slop = alloc_size - entry_size - head_slop;
|
tail_slop = alloc_size - entry_size - head_slop;
|
enda = allocation + alloc_size - tail_slop;
|
enda = allocation + alloc_size - tail_slop;
|
|
|
/* We allocated N pages, which are likely not aligned, leaving
|
/* We allocated N pages, which are likely not aligned, leaving
|
us with N-1 usable pages. We plan to place the page_group
|
us with N-1 usable pages. We plan to place the page_group
|
structure somewhere in the slop. */
|
structure somewhere in the slop. */
|
if (head_slop >= sizeof (page_group))
|
if (head_slop >= sizeof (page_group))
|
group = (page_group *)page - 1;
|
group = (page_group *)page - 1;
|
else
|
else
|
{
|
{
|
/* We magically got an aligned allocation. Too bad, we have
|
/* We magically got an aligned allocation. Too bad, we have
|
to waste a page anyway. */
|
to waste a page anyway. */
|
if (tail_slop == 0)
|
if (tail_slop == 0)
|
{
|
{
|
enda -= G.pagesize;
|
enda -= G.pagesize;
|
tail_slop += G.pagesize;
|
tail_slop += G.pagesize;
|
}
|
}
|
gcc_assert (tail_slop >= sizeof (page_group));
|
gcc_assert (tail_slop >= sizeof (page_group));
|
group = (page_group *)enda;
|
group = (page_group *)enda;
|
tail_slop -= sizeof (page_group);
|
tail_slop -= sizeof (page_group);
|
}
|
}
|
|
|
/* Remember that we allocated this memory. */
|
/* Remember that we allocated this memory. */
|
group->next = G.page_groups;
|
group->next = G.page_groups;
|
group->allocation = allocation;
|
group->allocation = allocation;
|
group->alloc_size = alloc_size;
|
group->alloc_size = alloc_size;
|
group->in_use = 0;
|
group->in_use = 0;
|
G.page_groups = group;
|
G.page_groups = group;
|
G.bytes_mapped += alloc_size;
|
G.bytes_mapped += alloc_size;
|
|
|
/* If we allocated multiple pages, put the rest on the free list. */
|
/* If we allocated multiple pages, put the rest on the free list. */
|
if (multiple_pages)
|
if (multiple_pages)
|
{
|
{
|
struct page_entry *e, *f = G.free_pages;
|
struct page_entry *e, *f = G.free_pages;
|
for (a = enda - G.pagesize; a != page; a -= G.pagesize)
|
for (a = enda - G.pagesize; a != page; a -= G.pagesize)
|
{
|
{
|
e = xcalloc (1, page_entry_size);
|
e = xcalloc (1, page_entry_size);
|
e->order = order;
|
e->order = order;
|
e->bytes = G.pagesize;
|
e->bytes = G.pagesize;
|
e->page = a;
|
e->page = a;
|
e->group = group;
|
e->group = group;
|
e->next = f;
|
e->next = f;
|
f = e;
|
f = e;
|
}
|
}
|
G.free_pages = f;
|
G.free_pages = f;
|
}
|
}
|
}
|
}
|
#endif
|
#endif
|
|
|
if (entry == NULL)
|
if (entry == NULL)
|
entry = xcalloc (1, page_entry_size);
|
entry = xcalloc (1, page_entry_size);
|
|
|
entry->bytes = entry_size;
|
entry->bytes = entry_size;
|
entry->page = page;
|
entry->page = page;
|
entry->context_depth = G.context_depth;
|
entry->context_depth = G.context_depth;
|
entry->order = order;
|
entry->order = order;
|
entry->num_free_objects = num_objects;
|
entry->num_free_objects = num_objects;
|
entry->next_bit_hint = 1;
|
entry->next_bit_hint = 1;
|
|
|
G.context_depth_allocations |= (unsigned long)1 << G.context_depth;
|
G.context_depth_allocations |= (unsigned long)1 << G.context_depth;
|
|
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
entry->group = group;
|
entry->group = group;
|
set_page_group_in_use (group, page);
|
set_page_group_in_use (group, page);
|
#endif
|
#endif
|
|
|
/* Set the one-past-the-end in-use bit. This acts as a sentry as we
|
/* Set the one-past-the-end in-use bit. This acts as a sentry as we
|
increment the hint. */
|
increment the hint. */
|
entry->in_use_p[num_objects / HOST_BITS_PER_LONG]
|
entry->in_use_p[num_objects / HOST_BITS_PER_LONG]
|
= (unsigned long) 1 << (num_objects % HOST_BITS_PER_LONG);
|
= (unsigned long) 1 << (num_objects % HOST_BITS_PER_LONG);
|
|
|
set_page_table_entry (page, entry);
|
set_page_table_entry (page, entry);
|
|
|
if (GGC_DEBUG_LEVEL >= 2)
|
if (GGC_DEBUG_LEVEL >= 2)
|
fprintf (G.debug_file,
|
fprintf (G.debug_file,
|
"Allocating page at %p, object size=%lu, data %p-%p\n",
|
"Allocating page at %p, object size=%lu, data %p-%p\n",
|
(void *) entry, (unsigned long) OBJECT_SIZE (order), page,
|
(void *) entry, (unsigned long) OBJECT_SIZE (order), page,
|
page + entry_size - 1);
|
page + entry_size - 1);
|
|
|
return entry;
|
return entry;
|
}
|
}
|
|
|
/* Adjust the size of G.depth so that no index greater than the one
|
/* Adjust the size of G.depth so that no index greater than the one
|
used by the top of the G.by_depth is used. */
|
used by the top of the G.by_depth is used. */
|
|
|
static inline void
|
static inline void
|
adjust_depth (void)
|
adjust_depth (void)
|
{
|
{
|
page_entry *top;
|
page_entry *top;
|
|
|
if (G.by_depth_in_use)
|
if (G.by_depth_in_use)
|
{
|
{
|
top = G.by_depth[G.by_depth_in_use-1];
|
top = G.by_depth[G.by_depth_in_use-1];
|
|
|
/* Peel back indices in depth that index into by_depth, so that
|
/* Peel back indices in depth that index into by_depth, so that
|
as new elements are added to by_depth, we note the indices
|
as new elements are added to by_depth, we note the indices
|
of those elements, if they are for new context depths. */
|
of those elements, if they are for new context depths. */
|
while (G.depth_in_use > (size_t)top->context_depth+1)
|
while (G.depth_in_use > (size_t)top->context_depth+1)
|
--G.depth_in_use;
|
--G.depth_in_use;
|
}
|
}
|
}
|
}
|
|
|
/* For a page that is no longer needed, put it on the free page list. */
|
/* For a page that is no longer needed, put it on the free page list. */
|
|
|
static void
|
static void
|
free_page (page_entry *entry)
|
free_page (page_entry *entry)
|
{
|
{
|
if (GGC_DEBUG_LEVEL >= 2)
|
if (GGC_DEBUG_LEVEL >= 2)
|
fprintf (G.debug_file,
|
fprintf (G.debug_file,
|
"Deallocating page at %p, data %p-%p\n", (void *) entry,
|
"Deallocating page at %p, data %p-%p\n", (void *) entry,
|
entry->page, entry->page + entry->bytes - 1);
|
entry->page, entry->page + entry->bytes - 1);
|
|
|
/* Mark the page as inaccessible. Discard the handle to avoid handle
|
/* Mark the page as inaccessible. Discard the handle to avoid handle
|
leak. */
|
leak. */
|
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (entry->page, entry->bytes));
|
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (entry->page, entry->bytes));
|
|
|
set_page_table_entry (entry->page, NULL);
|
set_page_table_entry (entry->page, NULL);
|
|
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
clear_page_group_in_use (entry->group, entry->page);
|
clear_page_group_in_use (entry->group, entry->page);
|
#endif
|
#endif
|
|
|
if (G.by_depth_in_use > 1)
|
if (G.by_depth_in_use > 1)
|
{
|
{
|
page_entry *top = G.by_depth[G.by_depth_in_use-1];
|
page_entry *top = G.by_depth[G.by_depth_in_use-1];
|
int i = entry->index_by_depth;
|
int i = entry->index_by_depth;
|
|
|
/* We cannot free a page from a context deeper than the current
|
/* We cannot free a page from a context deeper than the current
|
one. */
|
one. */
|
gcc_assert (entry->context_depth == top->context_depth);
|
gcc_assert (entry->context_depth == top->context_depth);
|
|
|
/* Put top element into freed slot. */
|
/* Put top element into freed slot. */
|
G.by_depth[i] = top;
|
G.by_depth[i] = top;
|
G.save_in_use[i] = G.save_in_use[G.by_depth_in_use-1];
|
G.save_in_use[i] = G.save_in_use[G.by_depth_in_use-1];
|
top->index_by_depth = i;
|
top->index_by_depth = i;
|
}
|
}
|
--G.by_depth_in_use;
|
--G.by_depth_in_use;
|
|
|
adjust_depth ();
|
adjust_depth ();
|
|
|
entry->next = G.free_pages;
|
entry->next = G.free_pages;
|
G.free_pages = entry;
|
G.free_pages = entry;
|
}
|
}
|
|
|
/* Release the free page cache to the system. */
|
/* Release the free page cache to the system. */
|
|
|
static void
|
static void
|
release_pages (void)
|
release_pages (void)
|
{
|
{
|
#ifdef USING_MMAP
|
#ifdef USING_MMAP
|
page_entry *p, *next;
|
page_entry *p, *next;
|
char *start;
|
char *start;
|
size_t len;
|
size_t len;
|
|
|
/* Gather up adjacent pages so they are unmapped together. */
|
/* Gather up adjacent pages so they are unmapped together. */
|
p = G.free_pages;
|
p = G.free_pages;
|
|
|
while (p)
|
while (p)
|
{
|
{
|
start = p->page;
|
start = p->page;
|
next = p->next;
|
next = p->next;
|
len = p->bytes;
|
len = p->bytes;
|
free (p);
|
free (p);
|
p = next;
|
p = next;
|
|
|
while (p && p->page == start + len)
|
while (p && p->page == start + len)
|
{
|
{
|
next = p->next;
|
next = p->next;
|
len += p->bytes;
|
len += p->bytes;
|
free (p);
|
free (p);
|
p = next;
|
p = next;
|
}
|
}
|
|
|
munmap (start, len);
|
munmap (start, len);
|
G.bytes_mapped -= len;
|
G.bytes_mapped -= len;
|
}
|
}
|
|
|
G.free_pages = NULL;
|
G.free_pages = NULL;
|
#endif
|
#endif
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
#ifdef USING_MALLOC_PAGE_GROUPS
|
page_entry **pp, *p;
|
page_entry **pp, *p;
|
page_group **gp, *g;
|
page_group **gp, *g;
|
|
|
/* Remove all pages from free page groups from the list. */
|
/* Remove all pages from free page groups from the list. */
|
pp = &G.free_pages;
|
pp = &G.free_pages;
|
while ((p = *pp) != NULL)
|
while ((p = *pp) != NULL)
|
if (p->group->in_use == 0)
|
if (p->group->in_use == 0)
|
{
|
{
|
*pp = p->next;
|
*pp = p->next;
|
free (p);
|
free (p);
|
}
|
}
|
else
|
else
|
pp = &p->next;
|
pp = &p->next;
|
|
|
/* Remove all free page groups, and release the storage. */
|
/* Remove all free page groups, and release the storage. */
|
gp = &G.page_groups;
|
gp = &G.page_groups;
|
while ((g = *gp) != NULL)
|
while ((g = *gp) != NULL)
|
if (g->in_use == 0)
|
if (g->in_use == 0)
|
{
|
{
|
*gp = g->next;
|
*gp = g->next;
|
G.bytes_mapped -= g->alloc_size;
|
G.bytes_mapped -= g->alloc_size;
|
free (g->allocation);
|
free (g->allocation);
|
}
|
}
|
else
|
else
|
gp = &g->next;
|
gp = &g->next;
|
#endif
|
#endif
|
}
|
}
|
|
|
/* This table provides a fast way to determine ceil(log_2(size)) for
|
/* This table provides a fast way to determine ceil(log_2(size)) for
|
allocation requests. The minimum allocation size is eight bytes. */
|
allocation requests. The minimum allocation size is eight bytes. */
|
#define NUM_SIZE_LOOKUP 512
|
#define NUM_SIZE_LOOKUP 512
|
static unsigned char size_lookup[NUM_SIZE_LOOKUP] =
|
static unsigned char size_lookup[NUM_SIZE_LOOKUP] =
|
{
|
{
|
3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4,
|
3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4,
|
4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
|
5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
|
5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
|
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
|
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
|
6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
|
7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
7, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
|
8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
|
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9
|
};
|
};
|
|
|
/* Typed allocation function. Does nothing special in this collector. */
|
/* Typed allocation function. Does nothing special in this collector. */
|
|
|
void *
|
void *
|
ggc_alloc_typed_stat (enum gt_types_enum type ATTRIBUTE_UNUSED, size_t size
|
ggc_alloc_typed_stat (enum gt_types_enum type ATTRIBUTE_UNUSED, size_t size
|
MEM_STAT_DECL)
|
MEM_STAT_DECL)
|
{
|
{
|
return ggc_alloc_stat (size PASS_MEM_STAT);
|
return ggc_alloc_stat (size PASS_MEM_STAT);
|
}
|
}
|
|
|
/* Allocate a chunk of memory of SIZE bytes. Its contents are undefined. */
|
/* Allocate a chunk of memory of SIZE bytes. Its contents are undefined. */
|
|
|
void *
|
void *
|
ggc_alloc_stat (size_t size MEM_STAT_DECL)
|
ggc_alloc_stat (size_t size MEM_STAT_DECL)
|
{
|
{
|
size_t order, word, bit, object_offset, object_size;
|
size_t order, word, bit, object_offset, object_size;
|
struct page_entry *entry;
|
struct page_entry *entry;
|
void *result;
|
void *result;
|
|
|
if (size < NUM_SIZE_LOOKUP)
|
if (size < NUM_SIZE_LOOKUP)
|
{
|
{
|
order = size_lookup[size];
|
order = size_lookup[size];
|
object_size = OBJECT_SIZE (order);
|
object_size = OBJECT_SIZE (order);
|
}
|
}
|
else
|
else
|
{
|
{
|
order = 10;
|
order = 10;
|
while (size > (object_size = OBJECT_SIZE (order)))
|
while (size > (object_size = OBJECT_SIZE (order)))
|
order++;
|
order++;
|
}
|
}
|
|
|
/* If there are non-full pages for this size allocation, they are at
|
/* If there are non-full pages for this size allocation, they are at
|
the head of the list. */
|
the head of the list. */
|
entry = G.pages[order];
|
entry = G.pages[order];
|
|
|
/* If there is no page for this object size, or all pages in this
|
/* If there is no page for this object size, or all pages in this
|
context are full, allocate a new page. */
|
context are full, allocate a new page. */
|
if (entry == NULL || entry->num_free_objects == 0)
|
if (entry == NULL || entry->num_free_objects == 0)
|
{
|
{
|
struct page_entry *new_entry;
|
struct page_entry *new_entry;
|
new_entry = alloc_page (order);
|
new_entry = alloc_page (order);
|
|
|
new_entry->index_by_depth = G.by_depth_in_use;
|
new_entry->index_by_depth = G.by_depth_in_use;
|
push_by_depth (new_entry, 0);
|
push_by_depth (new_entry, 0);
|
|
|
/* We can skip context depths, if we do, make sure we go all the
|
/* We can skip context depths, if we do, make sure we go all the
|
way to the new depth. */
|
way to the new depth. */
|
while (new_entry->context_depth >= G.depth_in_use)
|
while (new_entry->context_depth >= G.depth_in_use)
|
push_depth (G.by_depth_in_use-1);
|
push_depth (G.by_depth_in_use-1);
|
|
|
/* If this is the only entry, it's also the tail. If it is not
|
/* If this is the only entry, it's also the tail. If it is not
|
the only entry, then we must update the PREV pointer of the
|
the only entry, then we must update the PREV pointer of the
|
ENTRY (G.pages[order]) to point to our new page entry. */
|
ENTRY (G.pages[order]) to point to our new page entry. */
|
if (entry == NULL)
|
if (entry == NULL)
|
G.page_tails[order] = new_entry;
|
G.page_tails[order] = new_entry;
|
else
|
else
|
entry->prev = new_entry;
|
entry->prev = new_entry;
|
|
|
/* Put new pages at the head of the page list. By definition the
|
/* Put new pages at the head of the page list. By definition the
|
entry at the head of the list always has a NULL pointer. */
|
entry at the head of the list always has a NULL pointer. */
|
new_entry->next = entry;
|
new_entry->next = entry;
|
new_entry->prev = NULL;
|
new_entry->prev = NULL;
|
entry = new_entry;
|
entry = new_entry;
|
G.pages[order] = new_entry;
|
G.pages[order] = new_entry;
|
|
|
/* For a new page, we know the word and bit positions (in the
|
/* For a new page, we know the word and bit positions (in the
|
in_use bitmap) of the first available object -- they're zero. */
|
in_use bitmap) of the first available object -- they're zero. */
|
new_entry->next_bit_hint = 1;
|
new_entry->next_bit_hint = 1;
|
word = 0;
|
word = 0;
|
bit = 0;
|
bit = 0;
|
object_offset = 0;
|
object_offset = 0;
|
}
|
}
|
else
|
else
|
{
|
{
|
/* First try to use the hint left from the previous allocation
|
/* First try to use the hint left from the previous allocation
|
to locate a clear bit in the in-use bitmap. We've made sure
|
to locate a clear bit in the in-use bitmap. We've made sure
|
that the one-past-the-end bit is always set, so if the hint
|
that the one-past-the-end bit is always set, so if the hint
|
has run over, this test will fail. */
|
has run over, this test will fail. */
|
unsigned hint = entry->next_bit_hint;
|
unsigned hint = entry->next_bit_hint;
|
word = hint / HOST_BITS_PER_LONG;
|
word = hint / HOST_BITS_PER_LONG;
|
bit = hint % HOST_BITS_PER_LONG;
|
bit = hint % HOST_BITS_PER_LONG;
|
|
|
/* If the hint didn't work, scan the bitmap from the beginning. */
|
/* If the hint didn't work, scan the bitmap from the beginning. */
|
if ((entry->in_use_p[word] >> bit) & 1)
|
if ((entry->in_use_p[word] >> bit) & 1)
|
{
|
{
|
word = bit = 0;
|
word = bit = 0;
|
while (~entry->in_use_p[word] == 0)
|
while (~entry->in_use_p[word] == 0)
|
++word;
|
++word;
|
|
|
#if GCC_VERSION >= 3004
|
#if GCC_VERSION >= 3004
|
bit = __builtin_ctzl (~entry->in_use_p[word]);
|
bit = __builtin_ctzl (~entry->in_use_p[word]);
|
#else
|
#else
|
while ((entry->in_use_p[word] >> bit) & 1)
|
while ((entry->in_use_p[word] >> bit) & 1)
|
++bit;
|
++bit;
|
#endif
|
#endif
|
|
|
hint = word * HOST_BITS_PER_LONG + bit;
|
hint = word * HOST_BITS_PER_LONG + bit;
|
}
|
}
|
|
|
/* Next time, try the next bit. */
|
/* Next time, try the next bit. */
|
entry->next_bit_hint = hint + 1;
|
entry->next_bit_hint = hint + 1;
|
|
|
object_offset = hint * object_size;
|
object_offset = hint * object_size;
|
}
|
}
|
|
|
/* Set the in-use bit. */
|
/* Set the in-use bit. */
|
entry->in_use_p[word] |= ((unsigned long) 1 << bit);
|
entry->in_use_p[word] |= ((unsigned long) 1 << bit);
|
|
|
/* Keep a running total of the number of free objects. If this page
|
/* Keep a running total of the number of free objects. If this page
|
fills up, we may have to move it to the end of the list if the
|
fills up, we may have to move it to the end of the list if the
|
next page isn't full. If the next page is full, all subsequent
|
next page isn't full. If the next page is full, all subsequent
|
pages are full, so there's no need to move it. */
|
pages are full, so there's no need to move it. */
|
if (--entry->num_free_objects == 0
|
if (--entry->num_free_objects == 0
|
&& entry->next != NULL
|
&& entry->next != NULL
|
&& entry->next->num_free_objects > 0)
|
&& entry->next->num_free_objects > 0)
|
{
|
{
|
/* We have a new head for the list. */
|
/* We have a new head for the list. */
|
G.pages[order] = entry->next;
|
G.pages[order] = entry->next;
|
|
|
/* We are moving ENTRY to the end of the page table list.
|
/* We are moving ENTRY to the end of the page table list.
|
The new page at the head of the list will have NULL in
|
The new page at the head of the list will have NULL in
|
its PREV field and ENTRY will have NULL in its NEXT field. */
|
its PREV field and ENTRY will have NULL in its NEXT field. */
|
entry->next->prev = NULL;
|
entry->next->prev = NULL;
|
entry->next = NULL;
|
entry->next = NULL;
|
|
|
/* Append ENTRY to the tail of the list. */
|
/* Append ENTRY to the tail of the list. */
|
entry->prev = G.page_tails[order];
|
entry->prev = G.page_tails[order];
|
G.page_tails[order]->next = entry;
|
G.page_tails[order]->next = entry;
|
G.page_tails[order] = entry;
|
G.page_tails[order] = entry;
|
}
|
}
|
|
|
/* Calculate the object's address. */
|
/* Calculate the object's address. */
|
result = entry->page + object_offset;
|
result = entry->page + object_offset;
|
#ifdef GATHER_STATISTICS
|
#ifdef GATHER_STATISTICS
|
ggc_record_overhead (OBJECT_SIZE (order), OBJECT_SIZE (order) - size,
|
ggc_record_overhead (OBJECT_SIZE (order), OBJECT_SIZE (order) - size,
|
result PASS_MEM_STAT);
|
result PASS_MEM_STAT);
|
#endif
|
#endif
|
|
|
#ifdef ENABLE_GC_CHECKING
|
#ifdef ENABLE_GC_CHECKING
|
/* Keep poisoning-by-writing-0xaf the object, in an attempt to keep the
|
/* Keep poisoning-by-writing-0xaf the object, in an attempt to keep the
|
exact same semantics in presence of memory bugs, regardless of
|
exact same semantics in presence of memory bugs, regardless of
|
ENABLE_VALGRIND_CHECKING. We override this request below. Drop the
|
ENABLE_VALGRIND_CHECKING. We override this request below. Drop the
|
handle to avoid handle leak. */
|
handle to avoid handle leak. */
|
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (result, object_size));
|
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (result, object_size));
|
|
|
/* `Poison' the entire allocated object, including any padding at
|
/* `Poison' the entire allocated object, including any padding at
|
the end. */
|
the end. */
|
memset (result, 0xaf, object_size);
|
memset (result, 0xaf, object_size);
|
|
|
/* Make the bytes after the end of the object unaccessible. Discard the
|
/* Make the bytes after the end of the object unaccessible. Discard the
|
handle to avoid handle leak. */
|
handle to avoid handle leak. */
|
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS ((char *) result + size,
|
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS ((char *) result + size,
|
object_size - size));
|
object_size - size));
|
#endif
|
#endif
|
|
|
/* Tell Valgrind that the memory is there, but its content isn't
|
/* Tell Valgrind that the memory is there, but its content isn't
|
defined. The bytes at the end of the object are still marked
|
defined. The bytes at the end of the object are still marked
|
unaccessible. */
|
unaccessible. */
|
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (result, size));
|
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (result, size));
|
|
|
/* Keep track of how many bytes are being allocated. This
|
/* Keep track of how many bytes are being allocated. This
|
information is used in deciding when to collect. */
|
information is used in deciding when to collect. */
|
G.allocated += object_size;
|
G.allocated += object_size;
|
|
|
/* For timevar statistics. */
|
/* For timevar statistics. */
|
timevar_ggc_mem_total += object_size;
|
timevar_ggc_mem_total += object_size;
|
|
|
#ifdef GATHER_STATISTICS
|
#ifdef GATHER_STATISTICS
|
{
|
{
|
size_t overhead = object_size - size;
|
size_t overhead = object_size - size;
|
|
|
G.stats.total_overhead += overhead;
|
G.stats.total_overhead += overhead;
|
G.stats.total_allocated += object_size;
|
G.stats.total_allocated += object_size;
|
G.stats.total_overhead_per_order[order] += overhead;
|
G.stats.total_overhead_per_order[order] += overhead;
|
G.stats.total_allocated_per_order[order] += object_size;
|
G.stats.total_allocated_per_order[order] += object_size;
|
|
|
if (size <= 32)
|
if (size <= 32)
|
{
|
{
|
G.stats.total_overhead_under32 += overhead;
|
G.stats.total_overhead_under32 += overhead;
|
G.stats.total_allocated_under32 += object_size;
|
G.stats.total_allocated_under32 += object_size;
|
}
|
}
|
if (size <= 64)
|
if (size <= 64)
|
{
|
{
|
G.stats.total_overhead_under64 += overhead;
|
G.stats.total_overhead_under64 += overhead;
|
G.stats.total_allocated_under64 += object_size;
|
G.stats.total_allocated_under64 += object_size;
|
}
|
}
|
if (size <= 128)
|
if (size <= 128)
|
{
|
{
|
G.stats.total_overhead_under128 += overhead;
|
G.stats.total_overhead_under128 += overhead;
|
G.stats.total_allocated_under128 += object_size;
|
G.stats.total_allocated_under128 += object_size;
|
}
|
}
|
}
|
}
|
#endif
|
#endif
|
|
|
if (GGC_DEBUG_LEVEL >= 3)
|
if (GGC_DEBUG_LEVEL >= 3)
|
fprintf (G.debug_file,
|
fprintf (G.debug_file,
|
"Allocating object, requested size=%lu, actual=%lu at %p on %p\n",
|
"Allocating object, requested size=%lu, actual=%lu at %p on %p\n",
|
(unsigned long) size, (unsigned long) object_size, result,
|
(unsigned long) size, (unsigned long) object_size, result,
|
(void *) entry);
|
(void *) entry);
|
|
|
return result;
|
return result;
|
}
|
}
|
|
|
/* If P is not marked, marks it and return false. Otherwise return true.
|
/* If P is not marked, marks it and return false. Otherwise return true.
|
P must have been allocated by the GC allocator; it mustn't point to
|
P must have been allocated by the GC allocator; it mustn't point to
|
static objects, stack variables, or memory allocated with malloc. */
|
static objects, stack variables, or memory allocated with malloc. */
|
|
|
int
|
int
|
ggc_set_mark (const void *p)
|
ggc_set_mark (const void *p)
|
{
|
{
|
page_entry *entry;
|
page_entry *entry;
|
unsigned bit, word;
|
unsigned bit, word;
|
unsigned long mask;
|
unsigned long mask;
|
|
|
/* Look up the page on which the object is alloced. If the object
|
/* Look up the page on which the object is alloced. If the object
|
wasn't allocated by the collector, we'll probably die. */
|
wasn't allocated by the collector, we'll probably die. */
|
entry = lookup_page_table_entry (p);
|
entry = lookup_page_table_entry (p);
|
gcc_assert (entry);
|
gcc_assert (entry);
|
|
|
/* Calculate the index of the object on the page; this is its bit
|
/* Calculate the index of the object on the page; this is its bit
|
position in the in_use_p bitmap. */
|
position in the in_use_p bitmap. */
|
bit = OFFSET_TO_BIT (((const char *) p) - entry->page, entry->order);
|
bit = OFFSET_TO_BIT (((const char *) p) - entry->page, entry->order);
|
word = bit / HOST_BITS_PER_LONG;
|
word = bit / HOST_BITS_PER_LONG;
|
mask = (unsigned long) 1 << (bit % HOST_BITS_PER_LONG);
|
mask = (unsigned long) 1 << (bit % HOST_BITS_PER_LONG);
|
|
|
/* If the bit was previously set, skip it. */
|
/* If the bit was previously set, skip it. */
|
if (entry->in_use_p[word] & mask)
|
if (entry->in_use_p[word] & mask)
|
return 1;
|
return 1;
|
|
|
/* Otherwise set it, and decrement the free object count. */
|
/* Otherwise set it, and decrement the free object count. */
|
entry->in_use_p[word] |= mask;
|
entry->in_use_p[word] |= mask;
|
entry->num_free_objects -= 1;
|
entry->num_free_objects -= 1;
|
|
|
if (GGC_DEBUG_LEVEL >= 4)
|
if (GGC_DEBUG_LEVEL >= 4)
|
fprintf (G.debug_file, "Marking %p\n", p);
|
fprintf (G.debug_file, "Marking %p\n", p);
|
|
|
return 0;
|
return 0;
|
}
|
}
|
|
|
/* Return 1 if P has been marked, zero otherwise.
|
/* Return 1 if P has been marked, zero otherwise.
|
P must have been allocated by the GC allocator; it mustn't point to
|
P must have been allocated by the GC allocator; it mustn't point to
|
static objects, stack variables, or memory allocated with malloc. */
|
static objects, stack variables, or memory allocated with malloc. */
|
|
|
int
|
int
|
ggc_marked_p (const void *p)
|
ggc_marked_p (const void *p)
|
{
|
{
|
page_entry *entry;
|
page_entry *entry;
|
unsigned bit, word;
|
unsigned bit, word;
|
unsigned long mask;
|
unsigned long mask;
|
|
|
/* Look up the page on which the object is alloced. If the object
|
/* Look up the page on which the object is alloced. If the object
|
wasn't allocated by the collector, we'll probably die. */
|
wasn't allocated by the collector, we'll probably die. */
|
entry = lookup_page_table_entry (p);
|
entry = lookup_page_table_entry (p);
|
gcc_assert (entry);
|
gcc_assert (entry);
|
|
|
/* Calculate the index of the object on the page; this is its bit
|
/* Calculate the index of the object on the page; this is its bit
|
position in the in_use_p bitmap. */
|
position in the in_use_p bitmap. */
|
bit = OFFSET_TO_BIT (((const char *) p) - entry->page, entry->order);
|
bit = OFFSET_TO_BIT (((const char *) p) - entry->page, entry->order);
|
word = bit / HOST_BITS_PER_LONG;
|
word = bit / HOST_BITS_PER_LONG;
|
mask = (unsigned long) 1 << (bit % HOST_BITS_PER_LONG);
|
mask = (unsigned long) 1 << (bit % HOST_BITS_PER_LONG);
|
|
|
return (entry->in_use_p[word] & mask) != 0;
|
return (entry->in_use_p[word] & mask) != 0;
|
}
|
}
|
|
|
/* Return the size of the gc-able object P. */
|
/* Return the size of the gc-able object P. */
|
|
|
size_t
|
size_t
|
ggc_get_size (const void *p)
|
ggc_get_size (const void *p)
|
{
|
{
|
page_entry *pe = lookup_page_table_entry (p);
|
page_entry *pe = lookup_page_table_entry (p);
|
return OBJECT_SIZE (pe->order);
|
return OBJECT_SIZE (pe->order);
|
}
|
}
|
|
|
/* Release the memory for object P. */
|
/* Release the memory for object P. */
|
|
|
void
|
void
|
ggc_free (void *p)
|
ggc_free (void *p)
|
{
|
{
|
page_entry *pe = lookup_page_table_entry (p);
|
page_entry *pe = lookup_page_table_entry (p);
|
size_t order = pe->order;
|
size_t order = pe->order;
|
size_t size = OBJECT_SIZE (order);
|
size_t size = OBJECT_SIZE (order);
|
|
|
#ifdef GATHER_STATISTICS
|
#ifdef GATHER_STATISTICS
|
ggc_free_overhead (p);
|
ggc_free_overhead (p);
|
#endif
|
#endif
|
|
|
if (GGC_DEBUG_LEVEL >= 3)
|
if (GGC_DEBUG_LEVEL >= 3)
|
fprintf (G.debug_file,
|
fprintf (G.debug_file,
|
"Freeing object, actual size=%lu, at %p on %p\n",
|
"Freeing object, actual size=%lu, at %p on %p\n",
|
(unsigned long) size, p, (void *) pe);
|
(unsigned long) size, p, (void *) pe);
|
|
|
#ifdef ENABLE_GC_CHECKING
|
#ifdef ENABLE_GC_CHECKING
|
/* Poison the data, to indicate the data is garbage. */
|
/* Poison the data, to indicate the data is garbage. */
|
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (p, size));
|
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (p, size));
|
memset (p, 0xa5, size);
|
memset (p, 0xa5, size);
|
#endif
|
#endif
|
/* Let valgrind know the object is free. */
|
/* Let valgrind know the object is free. */
|
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (p, size));
|
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (p, size));
|
|
|
#ifdef ENABLE_GC_ALWAYS_COLLECT
|
#ifdef ENABLE_GC_ALWAYS_COLLECT
|
/* In the completely-anal-checking mode, we do *not* immediately free
|
/* In the completely-anal-checking mode, we do *not* immediately free
|
the data, but instead verify that the data is *actually* not
|
the data, but instead verify that the data is *actually* not
|
reachable the next time we collect. */
|
reachable the next time we collect. */
|
{
|
{
|
struct free_object *fo = XNEW (struct free_object);
|
struct free_object *fo = XNEW (struct free_object);
|
fo->object = p;
|
fo->object = p;
|
fo->next = G.free_object_list;
|
fo->next = G.free_object_list;
|
G.free_object_list = fo;
|
G.free_object_list = fo;
|
}
|
}
|
#else
|
#else
|
{
|
{
|
unsigned int bit_offset, word, bit;
|
unsigned int bit_offset, word, bit;
|
|
|
G.allocated -= size;
|
G.allocated -= size;
|
|
|
/* Mark the object not-in-use. */
|
/* Mark the object not-in-use. */
|
bit_offset = OFFSET_TO_BIT (((const char *) p) - pe->page, order);
|
bit_offset = OFFSET_TO_BIT (((const char *) p) - pe->page, order);
|
word = bit_offset / HOST_BITS_PER_LONG;
|
word = bit_offset / HOST_BITS_PER_LONG;
|
bit = bit_offset % HOST_BITS_PER_LONG;
|
bit = bit_offset % HOST_BITS_PER_LONG;
|
pe->in_use_p[word] &= ~(1UL << bit);
|
pe->in_use_p[word] &= ~(1UL << bit);
|
|
|
if (pe->num_free_objects++ == 0)
|
if (pe->num_free_objects++ == 0)
|
{
|
{
|
page_entry *p, *q;
|
page_entry *p, *q;
|
|
|
/* If the page is completely full, then it's supposed to
|
/* If the page is completely full, then it's supposed to
|
be after all pages that aren't. Since we've freed one
|
be after all pages that aren't. Since we've freed one
|
object from a page that was full, we need to move the
|
object from a page that was full, we need to move the
|
page to the head of the list.
|
page to the head of the list.
|
|
|
PE is the node we want to move. Q is the previous node
|
PE is the node we want to move. Q is the previous node
|
and P is the next node in the list. */
|
and P is the next node in the list. */
|
q = pe->prev;
|
q = pe->prev;
|
if (q && q->num_free_objects == 0)
|
if (q && q->num_free_objects == 0)
|
{
|
{
|
p = pe->next;
|
p = pe->next;
|
|
|
q->next = p;
|
q->next = p;
|
|
|
/* If PE was at the end of the list, then Q becomes the
|
/* If PE was at the end of the list, then Q becomes the
|
new end of the list. If PE was not the end of the
|
new end of the list. If PE was not the end of the
|
list, then we need to update the PREV field for P. */
|
list, then we need to update the PREV field for P. */
|
if (!p)
|
if (!p)
|
G.page_tails[order] = q;
|
G.page_tails[order] = q;
|
else
|
else
|
p->prev = q;
|
p->prev = q;
|
|
|
/* Move PE to the head of the list. */
|
/* Move PE to the head of the list. */
|
pe->next = G.pages[order];
|
pe->next = G.pages[order];
|
pe->prev = NULL;
|
pe->prev = NULL;
|
G.pages[order]->prev = pe;
|
G.pages[order]->prev = pe;
|
G.pages[order] = pe;
|
G.pages[order] = pe;
|
}
|
}
|
|
|
/* Reset the hint bit to point to the only free object. */
|
/* Reset the hint bit to point to the only free object. */
|
pe->next_bit_hint = bit_offset;
|
pe->next_bit_hint = bit_offset;
|
}
|
}
|
}
|
}
|
#endif
|
#endif
|
}
|
}
|
|
|
/* Subroutine of init_ggc which computes the pair of numbers used to
|
/* Subroutine of init_ggc which computes the pair of numbers used to
|
perform division by OBJECT_SIZE (order) and fills in inverse_table[].
|
perform division by OBJECT_SIZE (order) and fills in inverse_table[].
|
|
|
This algorithm is taken from Granlund and Montgomery's paper
|
This algorithm is taken from Granlund and Montgomery's paper
|
"Division by Invariant Integers using Multiplication"
|
"Division by Invariant Integers using Multiplication"
|
(Proc. SIGPLAN PLDI, 1994), section 9 (Exact division by
|
(Proc. SIGPLAN PLDI, 1994), section 9 (Exact division by
|
constants). */
|
constants). */
|
|
|
static void
|
static void
|
compute_inverse (unsigned order)
|
compute_inverse (unsigned order)
|
{
|
{
|
size_t size, inv;
|
size_t size, inv;
|
unsigned int e;
|
unsigned int e;
|
|
|
size = OBJECT_SIZE (order);
|
size = OBJECT_SIZE (order);
|
e = 0;
|
e = 0;
|
while (size % 2 == 0)
|
while (size % 2 == 0)
|
{
|
{
|
e++;
|
e++;
|
size >>= 1;
|
size >>= 1;
|
}
|
}
|
|
|
inv = size;
|
inv = size;
|
while (inv * size != 1)
|
while (inv * size != 1)
|
inv = inv * (2 - inv*size);
|
inv = inv * (2 - inv*size);
|
|
|
DIV_MULT (order) = inv;
|
DIV_MULT (order) = inv;
|
DIV_SHIFT (order) = e;
|
DIV_SHIFT (order) = e;
|
}
|
}
|
|
|
/* Initialize the ggc-mmap allocator. */
|
/* Initialize the ggc-mmap allocator. */
|
void
|
void
|
init_ggc (void)
|
init_ggc (void)
|
{
|
{
|
unsigned order;
|
unsigned order;
|
|
|
G.pagesize = getpagesize();
|
G.pagesize = getpagesize();
|
G.lg_pagesize = exact_log2 (G.pagesize);
|
G.lg_pagesize = exact_log2 (G.pagesize);
|
|
|
#ifdef HAVE_MMAP_DEV_ZERO
|
#ifdef HAVE_MMAP_DEV_ZERO
|
G.dev_zero_fd = open ("/dev/zero", O_RDONLY);
|
G.dev_zero_fd = open ("/dev/zero", O_RDONLY);
|
if (G.dev_zero_fd == -1)
|
if (G.dev_zero_fd == -1)
|
internal_error ("open /dev/zero: %m");
|
internal_error ("open /dev/zero: %m");
|
#endif
|
#endif
|
|
|
#if 0
|
#if 0
|
G.debug_file = fopen ("ggc-mmap.debug", "w");
|
G.debug_file = fopen ("ggc-mmap.debug", "w");
|
#else
|
#else
|
G.debug_file = stdout;
|
G.debug_file = stdout;
|
#endif
|
#endif
|
|
|
#ifdef USING_MMAP
|
#ifdef USING_MMAP
|
/* StunOS has an amazing off-by-one error for the first mmap allocation
|
/* StunOS has an amazing off-by-one error for the first mmap allocation
|
after fiddling with RLIMIT_STACK. The result, as hard as it is to
|
after fiddling with RLIMIT_STACK. The result, as hard as it is to
|
believe, is an unaligned page allocation, which would cause us to
|
believe, is an unaligned page allocation, which would cause us to
|
hork badly if we tried to use it. */
|
hork badly if we tried to use it. */
|
{
|
{
|
char *p = alloc_anon (NULL, G.pagesize);
|
char *p = alloc_anon (NULL, G.pagesize);
|
struct page_entry *e;
|
struct page_entry *e;
|
if ((size_t)p & (G.pagesize - 1))
|
if ((size_t)p & (G.pagesize - 1))
|
{
|
{
|
/* How losing. Discard this one and try another. If we still
|
/* How losing. Discard this one and try another. If we still
|
can't get something useful, give up. */
|
can't get something useful, give up. */
|
|
|
p = alloc_anon (NULL, G.pagesize);
|
p = alloc_anon (NULL, G.pagesize);
|
gcc_assert (!((size_t)p & (G.pagesize - 1)));
|
gcc_assert (!((size_t)p & (G.pagesize - 1)));
|
}
|
}
|
|
|
/* We have a good page, might as well hold onto it... */
|
/* We have a good page, might as well hold onto it... */
|
e = XCNEW (struct page_entry);
|
e = XCNEW (struct page_entry);
|
e->bytes = G.pagesize;
|
e->bytes = G.pagesize;
|
e->page = p;
|
e->page = p;
|
e->next = G.free_pages;
|
e->next = G.free_pages;
|
G.free_pages = e;
|
G.free_pages = e;
|
}
|
}
|
#endif
|
#endif
|
|
|
/* Initialize the object size table. */
|
/* Initialize the object size table. */
|
for (order = 0; order < HOST_BITS_PER_PTR; ++order)
|
for (order = 0; order < HOST_BITS_PER_PTR; ++order)
|
object_size_table[order] = (size_t) 1 << order;
|
object_size_table[order] = (size_t) 1 << order;
|
for (order = HOST_BITS_PER_PTR; order < NUM_ORDERS; ++order)
|
for (order = HOST_BITS_PER_PTR; order < NUM_ORDERS; ++order)
|
{
|
{
|
size_t s = extra_order_size_table[order - HOST_BITS_PER_PTR];
|
size_t s = extra_order_size_table[order - HOST_BITS_PER_PTR];
|
|
|
/* If S is not a multiple of the MAX_ALIGNMENT, then round it up
|
/* If S is not a multiple of the MAX_ALIGNMENT, then round it up
|
so that we're sure of getting aligned memory. */
|
so that we're sure of getting aligned memory. */
|
s = ROUND_UP (s, MAX_ALIGNMENT);
|
s = ROUND_UP (s, MAX_ALIGNMENT);
|
object_size_table[order] = s;
|
object_size_table[order] = s;
|
}
|
}
|
|
|
/* Initialize the objects-per-page and inverse tables. */
|
/* Initialize the objects-per-page and inverse tables. */
|
for (order = 0; order < NUM_ORDERS; ++order)
|
for (order = 0; order < NUM_ORDERS; ++order)
|
{
|
{
|
objects_per_page_table[order] = G.pagesize / OBJECT_SIZE (order);
|
objects_per_page_table[order] = G.pagesize / OBJECT_SIZE (order);
|
if (objects_per_page_table[order] == 0)
|
if (objects_per_page_table[order] == 0)
|
objects_per_page_table[order] = 1;
|
objects_per_page_table[order] = 1;
|
compute_inverse (order);
|
compute_inverse (order);
|
}
|
}
|
|
|
/* Reset the size_lookup array to put appropriately sized objects in
|
/* Reset the size_lookup array to put appropriately sized objects in
|
the special orders. All objects bigger than the previous power
|
the special orders. All objects bigger than the previous power
|
of two, but no greater than the special size, should go in the
|
of two, but no greater than the special size, should go in the
|
new order. */
|
new order. */
|
for (order = HOST_BITS_PER_PTR; order < NUM_ORDERS; ++order)
|
for (order = HOST_BITS_PER_PTR; order < NUM_ORDERS; ++order)
|
{
|
{
|
int o;
|
int o;
|
int i;
|
int i;
|
|
|
i = OBJECT_SIZE (order);
|
i = OBJECT_SIZE (order);
|
if (i >= NUM_SIZE_LOOKUP)
|
if (i >= NUM_SIZE_LOOKUP)
|
continue;
|
continue;
|
|
|
for (o = size_lookup[i]; o == size_lookup [i]; --i)
|
for (o = size_lookup[i]; o == size_lookup [i]; --i)
|
size_lookup[i] = order;
|
size_lookup[i] = order;
|
}
|
}
|
|
|
G.depth_in_use = 0;
|
G.depth_in_use = 0;
|
G.depth_max = 10;
|
G.depth_max = 10;
|
G.depth = XNEWVEC (unsigned int, G.depth_max);
|
G.depth = XNEWVEC (unsigned int, G.depth_max);
|
|
|
G.by_depth_in_use = 0;
|
G.by_depth_in_use = 0;
|
G.by_depth_max = INITIAL_PTE_COUNT;
|
G.by_depth_max = INITIAL_PTE_COUNT;
|
G.by_depth = XNEWVEC (page_entry *, G.by_depth_max);
|
G.by_depth = XNEWVEC (page_entry *, G.by_depth_max);
|
G.save_in_use = XNEWVEC (unsigned long *, G.by_depth_max);
|
G.save_in_use = XNEWVEC (unsigned long *, G.by_depth_max);
|
}
|
}
|
|
|
/* Start a new GGC zone. */
|
/* Start a new GGC zone. */
|
|
|
struct alloc_zone *
|
struct alloc_zone *
|
new_ggc_zone (const char *name ATTRIBUTE_UNUSED)
|
new_ggc_zone (const char *name ATTRIBUTE_UNUSED)
|
{
|
{
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Destroy a GGC zone. */
|
/* Destroy a GGC zone. */
|
void
|
void
|
destroy_ggc_zone (struct alloc_zone *zone ATTRIBUTE_UNUSED)
|
destroy_ggc_zone (struct alloc_zone *zone ATTRIBUTE_UNUSED)
|
{
|
{
|
}
|
}
|
|
|
/* Merge the SAVE_IN_USE_P and IN_USE_P arrays in P so that IN_USE_P
|
/* Merge the SAVE_IN_USE_P and IN_USE_P arrays in P so that IN_USE_P
|
reflects reality. Recalculate NUM_FREE_OBJECTS as well. */
|
reflects reality. Recalculate NUM_FREE_OBJECTS as well. */
|
|
|
static void
|
static void
|
ggc_recalculate_in_use_p (page_entry *p)
|
ggc_recalculate_in_use_p (page_entry *p)
|
{
|
{
|
unsigned int i;
|
unsigned int i;
|
size_t num_objects;
|
size_t num_objects;
|
|
|
/* Because the past-the-end bit in in_use_p is always set, we
|
/* Because the past-the-end bit in in_use_p is always set, we
|
pretend there is one additional object. */
|
pretend there is one additional object. */
|
num_objects = OBJECTS_IN_PAGE (p) + 1;
|
num_objects = OBJECTS_IN_PAGE (p) + 1;
|
|
|
/* Reset the free object count. */
|
/* Reset the free object count. */
|
p->num_free_objects = num_objects;
|
p->num_free_objects = num_objects;
|
|
|
/* Combine the IN_USE_P and SAVE_IN_USE_P arrays. */
|
/* Combine the IN_USE_P and SAVE_IN_USE_P arrays. */
|
for (i = 0;
|
for (i = 0;
|
i < CEIL (BITMAP_SIZE (num_objects),
|
i < CEIL (BITMAP_SIZE (num_objects),
|
sizeof (*p->in_use_p));
|
sizeof (*p->in_use_p));
|
++i)
|
++i)
|
{
|
{
|
unsigned long j;
|
unsigned long j;
|
|
|
/* Something is in use if it is marked, or if it was in use in a
|
/* Something is in use if it is marked, or if it was in use in a
|
context further down the context stack. */
|
context further down the context stack. */
|
p->in_use_p[i] |= save_in_use_p (p)[i];
|
p->in_use_p[i] |= save_in_use_p (p)[i];
|
|
|
/* Decrement the free object count for every object allocated. */
|
/* Decrement the free object count for every object allocated. */
|
for (j = p->in_use_p[i]; j; j >>= 1)
|
for (j = p->in_use_p[i]; j; j >>= 1)
|
p->num_free_objects -= (j & 1);
|
p->num_free_objects -= (j & 1);
|
}
|
}
|
|
|
gcc_assert (p->num_free_objects < num_objects);
|
gcc_assert (p->num_free_objects < num_objects);
|
}
|
}
|
|
|
/* Unmark all objects. */
|
/* Unmark all objects. */
|
|
|
static void
|
static void
|
clear_marks (void)
|
clear_marks (void)
|
{
|
{
|
unsigned order;
|
unsigned order;
|
|
|
for (order = 2; order < NUM_ORDERS; order++)
|
for (order = 2; order < NUM_ORDERS; order++)
|
{
|
{
|
page_entry *p;
|
page_entry *p;
|
|
|
for (p = G.pages[order]; p != NULL; p = p->next)
|
for (p = G.pages[order]; p != NULL; p = p->next)
|
{
|
{
|
size_t num_objects = OBJECTS_IN_PAGE (p);
|
size_t num_objects = OBJECTS_IN_PAGE (p);
|
size_t bitmap_size = BITMAP_SIZE (num_objects + 1);
|
size_t bitmap_size = BITMAP_SIZE (num_objects + 1);
|
|
|
/* The data should be page-aligned. */
|
/* The data should be page-aligned. */
|
gcc_assert (!((size_t) p->page & (G.pagesize - 1)));
|
gcc_assert (!((size_t) p->page & (G.pagesize - 1)));
|
|
|
/* Pages that aren't in the topmost context are not collected;
|
/* Pages that aren't in the topmost context are not collected;
|
nevertheless, we need their in-use bit vectors to store GC
|
nevertheless, we need their in-use bit vectors to store GC
|
marks. So, back them up first. */
|
marks. So, back them up first. */
|
if (p->context_depth < G.context_depth)
|
if (p->context_depth < G.context_depth)
|
{
|
{
|
if (! save_in_use_p (p))
|
if (! save_in_use_p (p))
|
save_in_use_p (p) = xmalloc (bitmap_size);
|
save_in_use_p (p) = xmalloc (bitmap_size);
|
memcpy (save_in_use_p (p), p->in_use_p, bitmap_size);
|
memcpy (save_in_use_p (p), p->in_use_p, bitmap_size);
|
}
|
}
|
|
|
/* Reset reset the number of free objects and clear the
|
/* Reset reset the number of free objects and clear the
|
in-use bits. These will be adjusted by mark_obj. */
|
in-use bits. These will be adjusted by mark_obj. */
|
p->num_free_objects = num_objects;
|
p->num_free_objects = num_objects;
|
memset (p->in_use_p, 0, bitmap_size);
|
memset (p->in_use_p, 0, bitmap_size);
|
|
|
/* Make sure the one-past-the-end bit is always set. */
|
/* Make sure the one-past-the-end bit is always set. */
|
p->in_use_p[num_objects / HOST_BITS_PER_LONG]
|
p->in_use_p[num_objects / HOST_BITS_PER_LONG]
|
= ((unsigned long) 1 << (num_objects % HOST_BITS_PER_LONG));
|
= ((unsigned long) 1 << (num_objects % HOST_BITS_PER_LONG));
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
/* Free all empty pages. Partially empty pages need no attention
|
/* Free all empty pages. Partially empty pages need no attention
|
because the `mark' bit doubles as an `unused' bit. */
|
because the `mark' bit doubles as an `unused' bit. */
|
|
|
static void
|
static void
|
sweep_pages (void)
|
sweep_pages (void)
|
{
|
{
|
unsigned order;
|
unsigned order;
|
|
|
for (order = 2; order < NUM_ORDERS; order++)
|
for (order = 2; order < NUM_ORDERS; order++)
|
{
|
{
|
/* The last page-entry to consider, regardless of entries
|
/* The last page-entry to consider, regardless of entries
|
placed at the end of the list. */
|
placed at the end of the list. */
|
page_entry * const last = G.page_tails[order];
|
page_entry * const last = G.page_tails[order];
|
|
|
size_t num_objects;
|
size_t num_objects;
|
size_t live_objects;
|
size_t live_objects;
|
page_entry *p, *previous;
|
page_entry *p, *previous;
|
int done;
|
int done;
|
|
|
p = G.pages[order];
|
p = G.pages[order];
|
if (p == NULL)
|
if (p == NULL)
|
continue;
|
continue;
|
|
|
previous = NULL;
|
previous = NULL;
|
do
|
do
|
{
|
{
|
page_entry *next = p->next;
|
page_entry *next = p->next;
|
|
|
/* Loop until all entries have been examined. */
|
/* Loop until all entries have been examined. */
|
done = (p == last);
|
done = (p == last);
|
|
|
num_objects = OBJECTS_IN_PAGE (p);
|
num_objects = OBJECTS_IN_PAGE (p);
|
|
|
/* Add all live objects on this page to the count of
|
/* Add all live objects on this page to the count of
|
allocated memory. */
|
allocated memory. */
|
live_objects = num_objects - p->num_free_objects;
|
live_objects = num_objects - p->num_free_objects;
|
|
|
G.allocated += OBJECT_SIZE (order) * live_objects;
|
G.allocated += OBJECT_SIZE (order) * live_objects;
|
|
|
/* Only objects on pages in the topmost context should get
|
/* Only objects on pages in the topmost context should get
|
collected. */
|
collected. */
|
if (p->context_depth < G.context_depth)
|
if (p->context_depth < G.context_depth)
|
;
|
;
|
|
|
/* Remove the page if it's empty. */
|
/* Remove the page if it's empty. */
|
else if (live_objects == 0)
|
else if (live_objects == 0)
|
{
|
{
|
/* If P was the first page in the list, then NEXT
|
/* If P was the first page in the list, then NEXT
|
becomes the new first page in the list, otherwise
|
becomes the new first page in the list, otherwise
|
splice P out of the forward pointers. */
|
splice P out of the forward pointers. */
|
if (! previous)
|
if (! previous)
|
G.pages[order] = next;
|
G.pages[order] = next;
|
else
|
else
|
previous->next = next;
|
previous->next = next;
|
|
|
/* Splice P out of the back pointers too. */
|
/* Splice P out of the back pointers too. */
|
if (next)
|
if (next)
|
next->prev = previous;
|
next->prev = previous;
|
|
|
/* Are we removing the last element? */
|
/* Are we removing the last element? */
|
if (p == G.page_tails[order])
|
if (p == G.page_tails[order])
|
G.page_tails[order] = previous;
|
G.page_tails[order] = previous;
|
free_page (p);
|
free_page (p);
|
p = previous;
|
p = previous;
|
}
|
}
|
|
|
/* If the page is full, move it to the end. */
|
/* If the page is full, move it to the end. */
|
else if (p->num_free_objects == 0)
|
else if (p->num_free_objects == 0)
|
{
|
{
|
/* Don't move it if it's already at the end. */
|
/* Don't move it if it's already at the end. */
|
if (p != G.page_tails[order])
|
if (p != G.page_tails[order])
|
{
|
{
|
/* Move p to the end of the list. */
|
/* Move p to the end of the list. */
|
p->next = NULL;
|
p->next = NULL;
|
p->prev = G.page_tails[order];
|
p->prev = G.page_tails[order];
|
G.page_tails[order]->next = p;
|
G.page_tails[order]->next = p;
|
|
|
/* Update the tail pointer... */
|
/* Update the tail pointer... */
|
G.page_tails[order] = p;
|
G.page_tails[order] = p;
|
|
|
/* ... and the head pointer, if necessary. */
|
/* ... and the head pointer, if necessary. */
|
if (! previous)
|
if (! previous)
|
G.pages[order] = next;
|
G.pages[order] = next;
|
else
|
else
|
previous->next = next;
|
previous->next = next;
|
|
|
/* And update the backpointer in NEXT if necessary. */
|
/* And update the backpointer in NEXT if necessary. */
|
if (next)
|
if (next)
|
next->prev = previous;
|
next->prev = previous;
|
|
|
p = previous;
|
p = previous;
|
}
|
}
|
}
|
}
|
|
|
/* If we've fallen through to here, it's a page in the
|
/* If we've fallen through to here, it's a page in the
|
topmost context that is neither full nor empty. Such a
|
topmost context that is neither full nor empty. Such a
|
page must precede pages at lesser context depth in the
|
page must precede pages at lesser context depth in the
|
list, so move it to the head. */
|
list, so move it to the head. */
|
else if (p != G.pages[order])
|
else if (p != G.pages[order])
|
{
|
{
|
previous->next = p->next;
|
previous->next = p->next;
|
|
|
/* Update the backchain in the next node if it exists. */
|
/* Update the backchain in the next node if it exists. */
|
if (p->next)
|
if (p->next)
|
p->next->prev = previous;
|
p->next->prev = previous;
|
|
|
/* Move P to the head of the list. */
|
/* Move P to the head of the list. */
|
p->next = G.pages[order];
|
p->next = G.pages[order];
|
p->prev = NULL;
|
p->prev = NULL;
|
G.pages[order]->prev = p;
|
G.pages[order]->prev = p;
|
|
|
/* Update the head pointer. */
|
/* Update the head pointer. */
|
G.pages[order] = p;
|
G.pages[order] = p;
|
|
|
/* Are we moving the last element? */
|
/* Are we moving the last element? */
|
if (G.page_tails[order] == p)
|
if (G.page_tails[order] == p)
|
G.page_tails[order] = previous;
|
G.page_tails[order] = previous;
|
p = previous;
|
p = previous;
|
}
|
}
|
|
|
previous = p;
|
previous = p;
|
p = next;
|
p = next;
|
}
|
}
|
while (! done);
|
while (! done);
|
|
|
/* Now, restore the in_use_p vectors for any pages from contexts
|
/* Now, restore the in_use_p vectors for any pages from contexts
|
other than the current one. */
|
other than the current one. */
|
for (p = G.pages[order]; p; p = p->next)
|
for (p = G.pages[order]; p; p = p->next)
|
if (p->context_depth != G.context_depth)
|
if (p->context_depth != G.context_depth)
|
ggc_recalculate_in_use_p (p);
|
ggc_recalculate_in_use_p (p);
|
}
|
}
|
}
|
}
|
|
|
#ifdef ENABLE_GC_CHECKING
|
#ifdef ENABLE_GC_CHECKING
|
/* Clobber all free objects. */
|
/* Clobber all free objects. */
|
|
|
static void
|
static void
|
poison_pages (void)
|
poison_pages (void)
|
{
|
{
|
unsigned order;
|
unsigned order;
|
|
|
for (order = 2; order < NUM_ORDERS; order++)
|
for (order = 2; order < NUM_ORDERS; order++)
|
{
|
{
|
size_t size = OBJECT_SIZE (order);
|
size_t size = OBJECT_SIZE (order);
|
page_entry *p;
|
page_entry *p;
|
|
|
for (p = G.pages[order]; p != NULL; p = p->next)
|
for (p = G.pages[order]; p != NULL; p = p->next)
|
{
|
{
|
size_t num_objects;
|
size_t num_objects;
|
size_t i;
|
size_t i;
|
|
|
if (p->context_depth != G.context_depth)
|
if (p->context_depth != G.context_depth)
|
/* Since we don't do any collection for pages in pushed
|
/* Since we don't do any collection for pages in pushed
|
contexts, there's no need to do any poisoning. And
|
contexts, there's no need to do any poisoning. And
|
besides, the IN_USE_P array isn't valid until we pop
|
besides, the IN_USE_P array isn't valid until we pop
|
contexts. */
|
contexts. */
|
continue;
|
continue;
|
|
|
num_objects = OBJECTS_IN_PAGE (p);
|
num_objects = OBJECTS_IN_PAGE (p);
|
for (i = 0; i < num_objects; i++)
|
for (i = 0; i < num_objects; i++)
|
{
|
{
|
size_t word, bit;
|
size_t word, bit;
|
word = i / HOST_BITS_PER_LONG;
|
word = i / HOST_BITS_PER_LONG;
|
bit = i % HOST_BITS_PER_LONG;
|
bit = i % HOST_BITS_PER_LONG;
|
if (((p->in_use_p[word] >> bit) & 1) == 0)
|
if (((p->in_use_p[word] >> bit) & 1) == 0)
|
{
|
{
|
char *object = p->page + i * size;
|
char *object = p->page + i * size;
|
|
|
/* Keep poison-by-write when we expect to use Valgrind,
|
/* Keep poison-by-write when we expect to use Valgrind,
|
so the exact same memory semantics is kept, in case
|
so the exact same memory semantics is kept, in case
|
there are memory errors. We override this request
|
there are memory errors. We override this request
|
below. */
|
below. */
|
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (object, size));
|
VALGRIND_DISCARD (VALGRIND_MAKE_WRITABLE (object, size));
|
memset (object, 0xa5, size);
|
memset (object, 0xa5, size);
|
|
|
/* Drop the handle to avoid handle leak. */
|
/* Drop the handle to avoid handle leak. */
|
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (object, size));
|
VALGRIND_DISCARD (VALGRIND_MAKE_NOACCESS (object, size));
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
#else
|
#else
|
#define poison_pages()
|
#define poison_pages()
|
#endif
|
#endif
|
|
|
#ifdef ENABLE_GC_ALWAYS_COLLECT
|
#ifdef ENABLE_GC_ALWAYS_COLLECT
|
/* Validate that the reportedly free objects actually are. */
|
/* Validate that the reportedly free objects actually are. */
|
|
|
static void
|
static void
|
validate_free_objects (void)
|
validate_free_objects (void)
|
{
|
{
|
struct free_object *f, *next, *still_free = NULL;
|
struct free_object *f, *next, *still_free = NULL;
|
|
|
for (f = G.free_object_list; f ; f = next)
|
for (f = G.free_object_list; f ; f = next)
|
{
|
{
|
page_entry *pe = lookup_page_table_entry (f->object);
|
page_entry *pe = lookup_page_table_entry (f->object);
|
size_t bit, word;
|
size_t bit, word;
|
|
|
bit = OFFSET_TO_BIT ((char *)f->object - pe->page, pe->order);
|
bit = OFFSET_TO_BIT ((char *)f->object - pe->page, pe->order);
|
word = bit / HOST_BITS_PER_LONG;
|
word = bit / HOST_BITS_PER_LONG;
|
bit = bit % HOST_BITS_PER_LONG;
|
bit = bit % HOST_BITS_PER_LONG;
|
next = f->next;
|
next = f->next;
|
|
|
/* Make certain it isn't visible from any root. Notice that we
|
/* Make certain it isn't visible from any root. Notice that we
|
do this check before sweep_pages merges save_in_use_p. */
|
do this check before sweep_pages merges save_in_use_p. */
|
gcc_assert (!(pe->in_use_p[word] & (1UL << bit)));
|
gcc_assert (!(pe->in_use_p[word] & (1UL << bit)));
|
|
|
/* If the object comes from an outer context, then retain the
|
/* If the object comes from an outer context, then retain the
|
free_object entry, so that we can verify that the address
|
free_object entry, so that we can verify that the address
|
isn't live on the stack in some outer context. */
|
isn't live on the stack in some outer context. */
|
if (pe->context_depth != G.context_depth)
|
if (pe->context_depth != G.context_depth)
|
{
|
{
|
f->next = still_free;
|
f->next = still_free;
|
still_free = f;
|
still_free = f;
|
}
|
}
|
else
|
else
|
free (f);
|
free (f);
|
}
|
}
|
|
|
G.free_object_list = still_free;
|
G.free_object_list = still_free;
|
}
|
}
|
#else
|
#else
|
#define validate_free_objects()
|
#define validate_free_objects()
|
#endif
|
#endif
|
|
|
/* Top level mark-and-sweep routine. */
|
/* Top level mark-and-sweep routine. */
|
|
|
void
|
void
|
ggc_collect (void)
|
ggc_collect (void)
|
{
|
{
|
/* Avoid frequent unnecessary work by skipping collection if the
|
/* Avoid frequent unnecessary work by skipping collection if the
|
total allocations haven't expanded much since the last
|
total allocations haven't expanded much since the last
|
collection. */
|
collection. */
|
float allocated_last_gc =
|
float allocated_last_gc =
|
MAX (G.allocated_last_gc, (size_t)PARAM_VALUE (GGC_MIN_HEAPSIZE) * 1024);
|
MAX (G.allocated_last_gc, (size_t)PARAM_VALUE (GGC_MIN_HEAPSIZE) * 1024);
|
|
|
float min_expand = allocated_last_gc * PARAM_VALUE (GGC_MIN_EXPAND) / 100;
|
float min_expand = allocated_last_gc * PARAM_VALUE (GGC_MIN_EXPAND) / 100;
|
|
|
if (G.allocated < allocated_last_gc + min_expand && !ggc_force_collect)
|
if (G.allocated < allocated_last_gc + min_expand && !ggc_force_collect)
|
return;
|
return;
|
|
|
timevar_push (TV_GC);
|
timevar_push (TV_GC);
|
if (!quiet_flag)
|
if (!quiet_flag)
|
fprintf (stderr, " {GC %luk -> ", (unsigned long) G.allocated / 1024);
|
fprintf (stderr, " {GC %luk -> ", (unsigned long) G.allocated / 1024);
|
if (GGC_DEBUG_LEVEL >= 2)
|
if (GGC_DEBUG_LEVEL >= 2)
|
fprintf (G.debug_file, "BEGIN COLLECTING\n");
|
fprintf (G.debug_file, "BEGIN COLLECTING\n");
|
|
|
/* Zero the total allocated bytes. This will be recalculated in the
|
/* Zero the total allocated bytes. This will be recalculated in the
|
sweep phase. */
|
sweep phase. */
|
G.allocated = 0;
|
G.allocated = 0;
|
|
|
/* Release the pages we freed the last time we collected, but didn't
|
/* Release the pages we freed the last time we collected, but didn't
|
reuse in the interim. */
|
reuse in the interim. */
|
release_pages ();
|
release_pages ();
|
|
|
/* Indicate that we've seen collections at this context depth. */
|
/* Indicate that we've seen collections at this context depth. */
|
G.context_depth_collections = ((unsigned long)1 << (G.context_depth + 1)) - 1;
|
G.context_depth_collections = ((unsigned long)1 << (G.context_depth + 1)) - 1;
|
|
|
clear_marks ();
|
clear_marks ();
|
ggc_mark_roots ();
|
ggc_mark_roots ();
|
#ifdef GATHER_STATISTICS
|
#ifdef GATHER_STATISTICS
|
ggc_prune_overhead_list ();
|
ggc_prune_overhead_list ();
|
#endif
|
#endif
|
poison_pages ();
|
poison_pages ();
|
validate_free_objects ();
|
validate_free_objects ();
|
sweep_pages ();
|
sweep_pages ();
|
|
|
G.allocated_last_gc = G.allocated;
|
G.allocated_last_gc = G.allocated;
|
|
|
timevar_pop (TV_GC);
|
timevar_pop (TV_GC);
|
|
|
if (!quiet_flag)
|
if (!quiet_flag)
|
fprintf (stderr, "%luk}", (unsigned long) G.allocated / 1024);
|
fprintf (stderr, "%luk}", (unsigned long) G.allocated / 1024);
|
if (GGC_DEBUG_LEVEL >= 2)
|
if (GGC_DEBUG_LEVEL >= 2)
|
fprintf (G.debug_file, "END COLLECTING\n");
|
fprintf (G.debug_file, "END COLLECTING\n");
|
}
|
}
|
|
|
/* Print allocation statistics. */
|
/* Print allocation statistics. */
|
#define SCALE(x) ((unsigned long) ((x) < 1024*10 \
|
#define SCALE(x) ((unsigned long) ((x) < 1024*10 \
|
? (x) \
|
? (x) \
|
: ((x) < 1024*1024*10 \
|
: ((x) < 1024*1024*10 \
|
? (x) / 1024 \
|
? (x) / 1024 \
|
: (x) / (1024*1024))))
|
: (x) / (1024*1024))))
|
#define STAT_LABEL(x) ((x) < 1024*10 ? ' ' : ((x) < 1024*1024*10 ? 'k' : 'M'))
|
#define STAT_LABEL(x) ((x) < 1024*10 ? ' ' : ((x) < 1024*1024*10 ? 'k' : 'M'))
|
|
|
void
|
void
|
ggc_print_statistics (void)
|
ggc_print_statistics (void)
|
{
|
{
|
struct ggc_statistics stats;
|
struct ggc_statistics stats;
|
unsigned int i;
|
unsigned int i;
|
size_t total_overhead = 0;
|
size_t total_overhead = 0;
|
|
|
/* Clear the statistics. */
|
/* Clear the statistics. */
|
memset (&stats, 0, sizeof (stats));
|
memset (&stats, 0, sizeof (stats));
|
|
|
/* Make sure collection will really occur. */
|
/* Make sure collection will really occur. */
|
G.allocated_last_gc = 0;
|
G.allocated_last_gc = 0;
|
|
|
/* Collect and print the statistics common across collectors. */
|
/* Collect and print the statistics common across collectors. */
|
ggc_print_common_statistics (stderr, &stats);
|
ggc_print_common_statistics (stderr, &stats);
|
|
|
/* Release free pages so that we will not count the bytes allocated
|
/* Release free pages so that we will not count the bytes allocated
|
there as part of the total allocated memory. */
|
there as part of the total allocated memory. */
|
release_pages ();
|
release_pages ();
|
|
|
/* Collect some information about the various sizes of
|
/* Collect some information about the various sizes of
|
allocation. */
|
allocation. */
|
fprintf (stderr,
|
fprintf (stderr,
|
"Memory still allocated at the end of the compilation process\n");
|
"Memory still allocated at the end of the compilation process\n");
|
fprintf (stderr, "%-5s %10s %10s %10s\n",
|
fprintf (stderr, "%-5s %10s %10s %10s\n",
|
"Size", "Allocated", "Used", "Overhead");
|
"Size", "Allocated", "Used", "Overhead");
|
for (i = 0; i < NUM_ORDERS; ++i)
|
for (i = 0; i < NUM_ORDERS; ++i)
|
{
|
{
|
page_entry *p;
|
page_entry *p;
|
size_t allocated;
|
size_t allocated;
|
size_t in_use;
|
size_t in_use;
|
size_t overhead;
|
size_t overhead;
|
|
|
/* Skip empty entries. */
|
/* Skip empty entries. */
|
if (!G.pages[i])
|
if (!G.pages[i])
|
continue;
|
continue;
|
|
|
overhead = allocated = in_use = 0;
|
overhead = allocated = in_use = 0;
|
|
|
/* Figure out the total number of bytes allocated for objects of
|
/* Figure out the total number of bytes allocated for objects of
|
this size, and how many of them are actually in use. Also figure
|
this size, and how many of them are actually in use. Also figure
|
out how much memory the page table is using. */
|
out how much memory the page table is using. */
|
for (p = G.pages[i]; p; p = p->next)
|
for (p = G.pages[i]; p; p = p->next)
|
{
|
{
|
allocated += p->bytes;
|
allocated += p->bytes;
|
in_use +=
|
in_use +=
|
(OBJECTS_IN_PAGE (p) - p->num_free_objects) * OBJECT_SIZE (i);
|
(OBJECTS_IN_PAGE (p) - p->num_free_objects) * OBJECT_SIZE (i);
|
|
|
overhead += (sizeof (page_entry) - sizeof (long)
|
overhead += (sizeof (page_entry) - sizeof (long)
|
+ BITMAP_SIZE (OBJECTS_IN_PAGE (p) + 1));
|
+ BITMAP_SIZE (OBJECTS_IN_PAGE (p) + 1));
|
}
|
}
|
fprintf (stderr, "%-5lu %10lu%c %10lu%c %10lu%c\n",
|
fprintf (stderr, "%-5lu %10lu%c %10lu%c %10lu%c\n",
|
(unsigned long) OBJECT_SIZE (i),
|
(unsigned long) OBJECT_SIZE (i),
|
SCALE (allocated), STAT_LABEL (allocated),
|
SCALE (allocated), STAT_LABEL (allocated),
|
SCALE (in_use), STAT_LABEL (in_use),
|
SCALE (in_use), STAT_LABEL (in_use),
|
SCALE (overhead), STAT_LABEL (overhead));
|
SCALE (overhead), STAT_LABEL (overhead));
|
total_overhead += overhead;
|
total_overhead += overhead;
|
}
|
}
|
fprintf (stderr, "%-5s %10lu%c %10lu%c %10lu%c\n", "Total",
|
fprintf (stderr, "%-5s %10lu%c %10lu%c %10lu%c\n", "Total",
|
SCALE (G.bytes_mapped), STAT_LABEL (G.bytes_mapped),
|
SCALE (G.bytes_mapped), STAT_LABEL (G.bytes_mapped),
|
SCALE (G.allocated), STAT_LABEL(G.allocated),
|
SCALE (G.allocated), STAT_LABEL(G.allocated),
|
SCALE (total_overhead), STAT_LABEL (total_overhead));
|
SCALE (total_overhead), STAT_LABEL (total_overhead));
|
|
|
#ifdef GATHER_STATISTICS
|
#ifdef GATHER_STATISTICS
|
{
|
{
|
fprintf (stderr, "\nTotal allocations and overheads during the compilation process\n");
|
fprintf (stderr, "\nTotal allocations and overheads during the compilation process\n");
|
|
|
fprintf (stderr, "Total Overhead: %10lld\n",
|
fprintf (stderr, "Total Overhead: %10lld\n",
|
G.stats.total_overhead);
|
G.stats.total_overhead);
|
fprintf (stderr, "Total Allocated: %10lld\n",
|
fprintf (stderr, "Total Allocated: %10lld\n",
|
G.stats.total_allocated);
|
G.stats.total_allocated);
|
|
|
fprintf (stderr, "Total Overhead under 32B: %10lld\n",
|
fprintf (stderr, "Total Overhead under 32B: %10lld\n",
|
G.stats.total_overhead_under32);
|
G.stats.total_overhead_under32);
|
fprintf (stderr, "Total Allocated under 32B: %10lld\n",
|
fprintf (stderr, "Total Allocated under 32B: %10lld\n",
|
G.stats.total_allocated_under32);
|
G.stats.total_allocated_under32);
|
fprintf (stderr, "Total Overhead under 64B: %10lld\n",
|
fprintf (stderr, "Total Overhead under 64B: %10lld\n",
|
G.stats.total_overhead_under64);
|
G.stats.total_overhead_under64);
|
fprintf (stderr, "Total Allocated under 64B: %10lld\n",
|
fprintf (stderr, "Total Allocated under 64B: %10lld\n",
|
G.stats.total_allocated_under64);
|
G.stats.total_allocated_under64);
|
fprintf (stderr, "Total Overhead under 128B: %10lld\n",
|
fprintf (stderr, "Total Overhead under 128B: %10lld\n",
|
G.stats.total_overhead_under128);
|
G.stats.total_overhead_under128);
|
fprintf (stderr, "Total Allocated under 128B: %10lld\n",
|
fprintf (stderr, "Total Allocated under 128B: %10lld\n",
|
G.stats.total_allocated_under128);
|
G.stats.total_allocated_under128);
|
|
|
for (i = 0; i < NUM_ORDERS; i++)
|
for (i = 0; i < NUM_ORDERS; i++)
|
if (G.stats.total_allocated_per_order[i])
|
if (G.stats.total_allocated_per_order[i])
|
{
|
{
|
fprintf (stderr, "Total Overhead page size %7d: %10lld\n",
|
fprintf (stderr, "Total Overhead page size %7d: %10lld\n",
|
OBJECT_SIZE (i), G.stats.total_overhead_per_order[i]);
|
OBJECT_SIZE (i), G.stats.total_overhead_per_order[i]);
|
fprintf (stderr, "Total Allocated page size %7d: %10lld\n",
|
fprintf (stderr, "Total Allocated page size %7d: %10lld\n",
|
OBJECT_SIZE (i), G.stats.total_allocated_per_order[i]);
|
OBJECT_SIZE (i), G.stats.total_allocated_per_order[i]);
|
}
|
}
|
}
|
}
|
#endif
|
#endif
|
}
|
}
|
|
|
struct ggc_pch_data
|
struct ggc_pch_data
|
{
|
{
|
struct ggc_pch_ondisk
|
struct ggc_pch_ondisk
|
{
|
{
|
unsigned totals[NUM_ORDERS];
|
unsigned totals[NUM_ORDERS];
|
} d;
|
} d;
|
size_t base[NUM_ORDERS];
|
size_t base[NUM_ORDERS];
|
size_t written[NUM_ORDERS];
|
size_t written[NUM_ORDERS];
|
};
|
};
|
|
|
struct ggc_pch_data *
|
struct ggc_pch_data *
|
init_ggc_pch (void)
|
init_ggc_pch (void)
|
{
|
{
|
return XCNEW (struct ggc_pch_data);
|
return XCNEW (struct ggc_pch_data);
|
}
|
}
|
|
|
void
|
void
|
ggc_pch_count_object (struct ggc_pch_data *d, void *x ATTRIBUTE_UNUSED,
|
ggc_pch_count_object (struct ggc_pch_data *d, void *x ATTRIBUTE_UNUSED,
|
size_t size, bool is_string ATTRIBUTE_UNUSED,
|
size_t size, bool is_string ATTRIBUTE_UNUSED,
|
enum gt_types_enum type ATTRIBUTE_UNUSED)
|
enum gt_types_enum type ATTRIBUTE_UNUSED)
|
{
|
{
|
unsigned order;
|
unsigned order;
|
|
|
if (size < NUM_SIZE_LOOKUP)
|
if (size < NUM_SIZE_LOOKUP)
|
order = size_lookup[size];
|
order = size_lookup[size];
|
else
|
else
|
{
|
{
|
order = 10;
|
order = 10;
|
while (size > OBJECT_SIZE (order))
|
while (size > OBJECT_SIZE (order))
|
order++;
|
order++;
|
}
|
}
|
|
|
d->d.totals[order]++;
|
d->d.totals[order]++;
|
}
|
}
|
|
|
size_t
|
size_t
|
ggc_pch_total_size (struct ggc_pch_data *d)
|
ggc_pch_total_size (struct ggc_pch_data *d)
|
{
|
{
|
size_t a = 0;
|
size_t a = 0;
|
unsigned i;
|
unsigned i;
|
|
|
for (i = 0; i < NUM_ORDERS; i++)
|
for (i = 0; i < NUM_ORDERS; i++)
|
a += ROUND_UP (d->d.totals[i] * OBJECT_SIZE (i), G.pagesize);
|
a += ROUND_UP (d->d.totals[i] * OBJECT_SIZE (i), G.pagesize);
|
return a;
|
return a;
|
}
|
}
|
|
|
void
|
void
|
ggc_pch_this_base (struct ggc_pch_data *d, void *base)
|
ggc_pch_this_base (struct ggc_pch_data *d, void *base)
|
{
|
{
|
size_t a = (size_t) base;
|
size_t a = (size_t) base;
|
unsigned i;
|
unsigned i;
|
|
|
for (i = 0; i < NUM_ORDERS; i++)
|
for (i = 0; i < NUM_ORDERS; i++)
|
{
|
{
|
d->base[i] = a;
|
d->base[i] = a;
|
a += ROUND_UP (d->d.totals[i] * OBJECT_SIZE (i), G.pagesize);
|
a += ROUND_UP (d->d.totals[i] * OBJECT_SIZE (i), G.pagesize);
|
}
|
}
|
}
|
}
|
|
|
|
|
char *
|
char *
|
ggc_pch_alloc_object (struct ggc_pch_data *d, void *x ATTRIBUTE_UNUSED,
|
ggc_pch_alloc_object (struct ggc_pch_data *d, void *x ATTRIBUTE_UNUSED,
|
size_t size, bool is_string ATTRIBUTE_UNUSED,
|
size_t size, bool is_string ATTRIBUTE_UNUSED,
|
enum gt_types_enum type ATTRIBUTE_UNUSED)
|
enum gt_types_enum type ATTRIBUTE_UNUSED)
|
{
|
{
|
unsigned order;
|
unsigned order;
|
char *result;
|
char *result;
|
|
|
if (size < NUM_SIZE_LOOKUP)
|
if (size < NUM_SIZE_LOOKUP)
|
order = size_lookup[size];
|
order = size_lookup[size];
|
else
|
else
|
{
|
{
|
order = 10;
|
order = 10;
|
while (size > OBJECT_SIZE (order))
|
while (size > OBJECT_SIZE (order))
|
order++;
|
order++;
|
}
|
}
|
|
|
result = (char *) d->base[order];
|
result = (char *) d->base[order];
|
d->base[order] += OBJECT_SIZE (order);
|
d->base[order] += OBJECT_SIZE (order);
|
return result;
|
return result;
|
}
|
}
|
|
|
void
|
void
|
ggc_pch_prepare_write (struct ggc_pch_data *d ATTRIBUTE_UNUSED,
|
ggc_pch_prepare_write (struct ggc_pch_data *d ATTRIBUTE_UNUSED,
|
FILE *f ATTRIBUTE_UNUSED)
|
FILE *f ATTRIBUTE_UNUSED)
|
{
|
{
|
/* Nothing to do. */
|
/* Nothing to do. */
|
}
|
}
|
|
|
void
|
void
|
ggc_pch_write_object (struct ggc_pch_data *d ATTRIBUTE_UNUSED,
|
ggc_pch_write_object (struct ggc_pch_data *d ATTRIBUTE_UNUSED,
|
FILE *f, void *x, void *newx ATTRIBUTE_UNUSED,
|
FILE *f, void *x, void *newx ATTRIBUTE_UNUSED,
|
size_t size, bool is_string ATTRIBUTE_UNUSED)
|
size_t size, bool is_string ATTRIBUTE_UNUSED)
|
{
|
{
|
unsigned order;
|
unsigned order;
|
static const char emptyBytes[256];
|
static const char emptyBytes[256];
|
|
|
if (size < NUM_SIZE_LOOKUP)
|
if (size < NUM_SIZE_LOOKUP)
|
order = size_lookup[size];
|
order = size_lookup[size];
|
else
|
else
|
{
|
{
|
order = 10;
|
order = 10;
|
while (size > OBJECT_SIZE (order))
|
while (size > OBJECT_SIZE (order))
|
order++;
|
order++;
|
}
|
}
|
|
|
if (fwrite (x, size, 1, f) != 1)
|
if (fwrite (x, size, 1, f) != 1)
|
fatal_error ("can't write PCH file: %m");
|
fatal_error ("can't write PCH file: %m");
|
|
|
/* If SIZE is not the same as OBJECT_SIZE(order), then we need to pad the
|
/* If SIZE is not the same as OBJECT_SIZE(order), then we need to pad the
|
object out to OBJECT_SIZE(order). This happens for strings. */
|
object out to OBJECT_SIZE(order). This happens for strings. */
|
|
|
if (size != OBJECT_SIZE (order))
|
if (size != OBJECT_SIZE (order))
|
{
|
{
|
unsigned padding = OBJECT_SIZE(order) - size;
|
unsigned padding = OBJECT_SIZE(order) - size;
|
|
|
/* To speed small writes, we use a nulled-out array that's larger
|
/* To speed small writes, we use a nulled-out array that's larger
|
than most padding requests as the source for our null bytes. This
|
than most padding requests as the source for our null bytes. This
|
permits us to do the padding with fwrite() rather than fseek(), and
|
permits us to do the padding with fwrite() rather than fseek(), and
|
limits the chance the OS may try to flush any outstanding writes. */
|
limits the chance the OS may try to flush any outstanding writes. */
|
if (padding <= sizeof(emptyBytes))
|
if (padding <= sizeof(emptyBytes))
|
{
|
{
|
if (fwrite (emptyBytes, 1, padding, f) != padding)
|
if (fwrite (emptyBytes, 1, padding, f) != padding)
|
fatal_error ("can't write PCH file");
|
fatal_error ("can't write PCH file");
|
}
|
}
|
else
|
else
|
{
|
{
|
/* Larger than our buffer? Just default to fseek. */
|
/* Larger than our buffer? Just default to fseek. */
|
if (fseek (f, padding, SEEK_CUR) != 0)
|
if (fseek (f, padding, SEEK_CUR) != 0)
|
fatal_error ("can't write PCH file");
|
fatal_error ("can't write PCH file");
|
}
|
}
|
}
|
}
|
|
|
d->written[order]++;
|
d->written[order]++;
|
if (d->written[order] == d->d.totals[order]
|
if (d->written[order] == d->d.totals[order]
|
&& fseek (f, ROUND_UP_VALUE (d->d.totals[order] * OBJECT_SIZE (order),
|
&& fseek (f, ROUND_UP_VALUE (d->d.totals[order] * OBJECT_SIZE (order),
|
G.pagesize),
|
G.pagesize),
|
SEEK_CUR) != 0)
|
SEEK_CUR) != 0)
|
fatal_error ("can't write PCH file: %m");
|
fatal_error ("can't write PCH file: %m");
|
}
|
}
|
|
|
void
|
void
|
ggc_pch_finish (struct ggc_pch_data *d, FILE *f)
|
ggc_pch_finish (struct ggc_pch_data *d, FILE *f)
|
{
|
{
|
if (fwrite (&d->d, sizeof (d->d), 1, f) != 1)
|
if (fwrite (&d->d, sizeof (d->d), 1, f) != 1)
|
fatal_error ("can't write PCH file: %m");
|
fatal_error ("can't write PCH file: %m");
|
free (d);
|
free (d);
|
}
|
}
|
|
|
/* Move the PCH PTE entries just added to the end of by_depth, to the
|
/* Move the PCH PTE entries just added to the end of by_depth, to the
|
front. */
|
front. */
|
|
|
static void
|
static void
|
move_ptes_to_front (int count_old_page_tables, int count_new_page_tables)
|
move_ptes_to_front (int count_old_page_tables, int count_new_page_tables)
|
{
|
{
|
unsigned i;
|
unsigned i;
|
|
|
/* First, we swap the new entries to the front of the varrays. */
|
/* First, we swap the new entries to the front of the varrays. */
|
page_entry **new_by_depth;
|
page_entry **new_by_depth;
|
unsigned long **new_save_in_use;
|
unsigned long **new_save_in_use;
|
|
|
new_by_depth = XNEWVEC (page_entry *, G.by_depth_max);
|
new_by_depth = XNEWVEC (page_entry *, G.by_depth_max);
|
new_save_in_use = XNEWVEC (unsigned long *, G.by_depth_max);
|
new_save_in_use = XNEWVEC (unsigned long *, G.by_depth_max);
|
|
|
memcpy (&new_by_depth[0],
|
memcpy (&new_by_depth[0],
|
&G.by_depth[count_old_page_tables],
|
&G.by_depth[count_old_page_tables],
|
count_new_page_tables * sizeof (void *));
|
count_new_page_tables * sizeof (void *));
|
memcpy (&new_by_depth[count_new_page_tables],
|
memcpy (&new_by_depth[count_new_page_tables],
|
&G.by_depth[0],
|
&G.by_depth[0],
|
count_old_page_tables * sizeof (void *));
|
count_old_page_tables * sizeof (void *));
|
memcpy (&new_save_in_use[0],
|
memcpy (&new_save_in_use[0],
|
&G.save_in_use[count_old_page_tables],
|
&G.save_in_use[count_old_page_tables],
|
count_new_page_tables * sizeof (void *));
|
count_new_page_tables * sizeof (void *));
|
memcpy (&new_save_in_use[count_new_page_tables],
|
memcpy (&new_save_in_use[count_new_page_tables],
|
&G.save_in_use[0],
|
&G.save_in_use[0],
|
count_old_page_tables * sizeof (void *));
|
count_old_page_tables * sizeof (void *));
|
|
|
free (G.by_depth);
|
free (G.by_depth);
|
free (G.save_in_use);
|
free (G.save_in_use);
|
|
|
G.by_depth = new_by_depth;
|
G.by_depth = new_by_depth;
|
G.save_in_use = new_save_in_use;
|
G.save_in_use = new_save_in_use;
|
|
|
/* Now update all the index_by_depth fields. */
|
/* Now update all the index_by_depth fields. */
|
for (i = G.by_depth_in_use; i > 0; --i)
|
for (i = G.by_depth_in_use; i > 0; --i)
|
{
|
{
|
page_entry *p = G.by_depth[i-1];
|
page_entry *p = G.by_depth[i-1];
|
p->index_by_depth = i-1;
|
p->index_by_depth = i-1;
|
}
|
}
|
|
|
/* And last, we update the depth pointers in G.depth. The first
|
/* And last, we update the depth pointers in G.depth. The first
|
entry is already 0, and context 0 entries always start at index
|
entry is already 0, and context 0 entries always start at index
|
0, so there is nothing to update in the first slot. We need a
|
0, so there is nothing to update in the first slot. We need a
|
second slot, only if we have old ptes, and if we do, they start
|
second slot, only if we have old ptes, and if we do, they start
|
at index count_new_page_tables. */
|
at index count_new_page_tables. */
|
if (count_old_page_tables)
|
if (count_old_page_tables)
|
push_depth (count_new_page_tables);
|
push_depth (count_new_page_tables);
|
}
|
}
|
|
|
void
|
void
|
ggc_pch_read (FILE *f, void *addr)
|
ggc_pch_read (FILE *f, void *addr)
|
{
|
{
|
struct ggc_pch_ondisk d;
|
struct ggc_pch_ondisk d;
|
unsigned i;
|
unsigned i;
|
char *offs = addr;
|
char *offs = addr;
|
unsigned long count_old_page_tables;
|
unsigned long count_old_page_tables;
|
unsigned long count_new_page_tables;
|
unsigned long count_new_page_tables;
|
|
|
count_old_page_tables = G.by_depth_in_use;
|
count_old_page_tables = G.by_depth_in_use;
|
|
|
/* We've just read in a PCH file. So, every object that used to be
|
/* We've just read in a PCH file. So, every object that used to be
|
allocated is now free. */
|
allocated is now free. */
|
clear_marks ();
|
clear_marks ();
|
#ifdef ENABLE_GC_CHECKING
|
#ifdef ENABLE_GC_CHECKING
|
poison_pages ();
|
poison_pages ();
|
#endif
|
#endif
|
|
|
/* No object read from a PCH file should ever be freed. So, set the
|
/* No object read from a PCH file should ever be freed. So, set the
|
context depth to 1, and set the depth of all the currently-allocated
|
context depth to 1, and set the depth of all the currently-allocated
|
pages to be 1 too. PCH pages will have depth 0. */
|
pages to be 1 too. PCH pages will have depth 0. */
|
gcc_assert (!G.context_depth);
|
gcc_assert (!G.context_depth);
|
G.context_depth = 1;
|
G.context_depth = 1;
|
for (i = 0; i < NUM_ORDERS; i++)
|
for (i = 0; i < NUM_ORDERS; i++)
|
{
|
{
|
page_entry *p;
|
page_entry *p;
|
for (p = G.pages[i]; p != NULL; p = p->next)
|
for (p = G.pages[i]; p != NULL; p = p->next)
|
p->context_depth = G.context_depth;
|
p->context_depth = G.context_depth;
|
}
|
}
|
|
|
/* Allocate the appropriate page-table entries for the pages read from
|
/* Allocate the appropriate page-table entries for the pages read from
|
the PCH file. */
|
the PCH file. */
|
if (fread (&d, sizeof (d), 1, f) != 1)
|
if (fread (&d, sizeof (d), 1, f) != 1)
|
fatal_error ("can't read PCH file: %m");
|
fatal_error ("can't read PCH file: %m");
|
|
|
for (i = 0; i < NUM_ORDERS; i++)
|
for (i = 0; i < NUM_ORDERS; i++)
|
{
|
{
|
struct page_entry *entry;
|
struct page_entry *entry;
|
char *pte;
|
char *pte;
|
size_t bytes;
|
size_t bytes;
|
size_t num_objs;
|
size_t num_objs;
|
size_t j;
|
size_t j;
|
|
|
if (d.totals[i] == 0)
|
if (d.totals[i] == 0)
|
continue;
|
continue;
|
|
|
bytes = ROUND_UP (d.totals[i] * OBJECT_SIZE (i), G.pagesize);
|
bytes = ROUND_UP (d.totals[i] * OBJECT_SIZE (i), G.pagesize);
|
num_objs = bytes / OBJECT_SIZE (i);
|
num_objs = bytes / OBJECT_SIZE (i);
|
entry = xcalloc (1, (sizeof (struct page_entry)
|
entry = xcalloc (1, (sizeof (struct page_entry)
|
- sizeof (long)
|
- sizeof (long)
|
+ BITMAP_SIZE (num_objs + 1)));
|
+ BITMAP_SIZE (num_objs + 1)));
|
entry->bytes = bytes;
|
entry->bytes = bytes;
|
entry->page = offs;
|
entry->page = offs;
|
entry->context_depth = 0;
|
entry->context_depth = 0;
|
offs += bytes;
|
offs += bytes;
|
entry->num_free_objects = 0;
|
entry->num_free_objects = 0;
|
entry->order = i;
|
entry->order = i;
|
|
|
for (j = 0;
|
for (j = 0;
|
j + HOST_BITS_PER_LONG <= num_objs + 1;
|
j + HOST_BITS_PER_LONG <= num_objs + 1;
|
j += HOST_BITS_PER_LONG)
|
j += HOST_BITS_PER_LONG)
|
entry->in_use_p[j / HOST_BITS_PER_LONG] = -1;
|
entry->in_use_p[j / HOST_BITS_PER_LONG] = -1;
|
for (; j < num_objs + 1; j++)
|
for (; j < num_objs + 1; j++)
|
entry->in_use_p[j / HOST_BITS_PER_LONG]
|
entry->in_use_p[j / HOST_BITS_PER_LONG]
|
|= 1L << (j % HOST_BITS_PER_LONG);
|
|= 1L << (j % HOST_BITS_PER_LONG);
|
|
|
for (pte = entry->page;
|
for (pte = entry->page;
|
pte < entry->page + entry->bytes;
|
pte < entry->page + entry->bytes;
|
pte += G.pagesize)
|
pte += G.pagesize)
|
set_page_table_entry (pte, entry);
|
set_page_table_entry (pte, entry);
|
|
|
if (G.page_tails[i] != NULL)
|
if (G.page_tails[i] != NULL)
|
G.page_tails[i]->next = entry;
|
G.page_tails[i]->next = entry;
|
else
|
else
|
G.pages[i] = entry;
|
G.pages[i] = entry;
|
G.page_tails[i] = entry;
|
G.page_tails[i] = entry;
|
|
|
/* We start off by just adding all the new information to the
|
/* We start off by just adding all the new information to the
|
end of the varrays, later, we will move the new information
|
end of the varrays, later, we will move the new information
|
to the front of the varrays, as the PCH page tables are at
|
to the front of the varrays, as the PCH page tables are at
|
context 0. */
|
context 0. */
|
push_by_depth (entry, 0);
|
push_by_depth (entry, 0);
|
}
|
}
|
|
|
/* Now, we update the various data structures that speed page table
|
/* Now, we update the various data structures that speed page table
|
handling. */
|
handling. */
|
count_new_page_tables = G.by_depth_in_use - count_old_page_tables;
|
count_new_page_tables = G.by_depth_in_use - count_old_page_tables;
|
|
|
move_ptes_to_front (count_old_page_tables, count_new_page_tables);
|
move_ptes_to_front (count_old_page_tables, count_new_page_tables);
|
|
|
/* Update the statistics. */
|
/* Update the statistics. */
|
G.allocated = G.allocated_last_gc = offs - (char *)addr;
|
G.allocated = G.allocated_last_gc = offs - (char *)addr;
|
}
|
}
|
|
|
