107#elif defined RUBY_EXPORT
109#include "internal/bits.h"
110#include "internal/hash.h"
111#include "internal/sanitizers.h"
112#include "internal/set_table.h"
113#include "internal/static_assert.h"
114#include "internal/st.h"
115#include "ruby_assert.h"
125#define PREFETCH(addr, write_p) __builtin_prefetch(addr, write_p)
126#define EXPECT(expr, val) __builtin_expect(expr, val)
127#define ATTRIBUTE_UNUSED __attribute__((unused))
129#define PREFETCH(addr, write_p)
130#define EXPECT(expr, val) (expr)
131#define ATTRIBUTE_UNUSED
135typedef st_index_t st_hash_t;
143#define type_numhash st_hashtype_num
144static const struct st_hash_type st_hashtype_num = {
149static int st_strcmp(st_data_t, st_data_t);
150static st_index_t strhash(st_data_t);
151static const struct st_hash_type type_strhash = {
156static int st_locale_insensitive_strcasecmp_i(st_data_t lhs, st_data_t rhs);
157static st_index_t strcasehash(st_data_t);
158static const struct st_hash_type type_strcasehash = {
159 st_locale_insensitive_strcasecmp_i,
166#define ST_INIT_VAL 0xafafafafafafafaf
167#define ST_INIT_VAL_BYTE 0xafa
174#define malloc ruby_xmalloc
175#define calloc ruby_xcalloc
176#define realloc ruby_xrealloc
177#define free ruby_xfree
180#define EQUAL(tab,x,y) ((x) == (y) || (*(tab)->type->compare)((x),(y)) == 0)
181#define PTR_EQUAL(tab, ptr, hash_val, key_) \
182 ((ptr)->hash == (hash_val) && EQUAL((tab), (key_), (ptr)->key))
186#define DO_PTR_EQUAL_CHECK(tab, ptr, hash_val, key, res, rebuilt_p) \
188 unsigned int _old_rebuilds_num = (tab)->rebuilds_num; \
189 res = PTR_EQUAL(tab, ptr, hash_val, key); \
190 rebuilt_p = _old_rebuilds_num != (tab)->rebuilds_num; \
196 unsigned char entry_power;
200 unsigned char bin_power;
202 unsigned char size_ind;
205 st_index_t bins_words;
209#if SIZEOF_ST_INDEX_T == 8
228 {16, 17, 2, 0x10000},
229 {17, 18, 2, 0x20000},
230 {18, 19, 2, 0x40000},
231 {19, 20, 2, 0x80000},
232 {20, 21, 2, 0x100000},
233 {21, 22, 2, 0x200000},
234 {22, 23, 2, 0x400000},
235 {23, 24, 2, 0x800000},
236 {24, 25, 2, 0x1000000},
237 {25, 26, 2, 0x2000000},
238 {26, 27, 2, 0x4000000},
239 {27, 28, 2, 0x8000000},
240 {28, 29, 2, 0x10000000},
241 {29, 30, 2, 0x20000000},
242 {30, 31, 2, 0x40000000},
243 {31, 32, 2, 0x80000000},
244 {32, 33, 3, 0x200000000},
245 {33, 34, 3, 0x400000000},
246 {34, 35, 3, 0x800000000},
247 {35, 36, 3, 0x1000000000},
248 {36, 37, 3, 0x2000000000},
249 {37, 38, 3, 0x4000000000},
250 {38, 39, 3, 0x8000000000},
251 {39, 40, 3, 0x10000000000},
252 {40, 41, 3, 0x20000000000},
253 {41, 42, 3, 0x40000000000},
254 {42, 43, 3, 0x80000000000},
255 {43, 44, 3, 0x100000000000},
256 {44, 45, 3, 0x200000000000},
257 {45, 46, 3, 0x400000000000},
258 {46, 47, 3, 0x800000000000},
259 {47, 48, 3, 0x1000000000000},
260 {48, 49, 3, 0x2000000000000},
261 {49, 50, 3, 0x4000000000000},
262 {50, 51, 3, 0x8000000000000},
263 {51, 52, 3, 0x10000000000000},
264 {52, 53, 3, 0x20000000000000},
265 {53, 54, 3, 0x40000000000000},
266 {54, 55, 3, 0x80000000000000},
267 {55, 56, 3, 0x100000000000000},
268 {56, 57, 3, 0x200000000000000},
269 {57, 58, 3, 0x400000000000000},
270#define MAX_BINS_WORDS 0x800000000000000
271 {58, 59, 3, MAX_BINS_WORDS},
294 {16, 17, 2, 0x20000},
295 {17, 18, 2, 0x40000},
296 {18, 19, 2, 0x80000},
297 {19, 20, 2, 0x100000},
298 {20, 21, 2, 0x200000},
299 {21, 22, 2, 0x400000},
300 {22, 23, 2, 0x800000},
301 {23, 24, 2, 0x1000000},
302 {24, 25, 2, 0x2000000},
303 {25, 26, 2, 0x4000000},
304 {26, 27, 2, 0x8000000},
305#define MAX_BINS_WORDS 0x10000000
306 {27, 28, 2, MAX_BINS_WORDS},
313#define MAX_ADDRESSABLE_ENTRIES (SIZE_MAX / sizeof(st_table_entry))
314#define MAX_ENTRIES (((st_index_t) 1)<<MAX_POWER2)
315STATIC_ASSERT(max_power_entries_overflow, MAX_ENTRIES < MAX_ADDRESSABLE_ENTRIES);
316#define MAX_BINS_SIZE (SIZE_MAX - (MAX_ENTRIES * sizeof(st_table_entry)))
317STATIC_ASSERT(max_power_bins_overflow, (MAX_BINS_SIZE /
sizeof(st_index_t)) > MAX_BINS_WORDS);
321#define RESERVED_HASH_VAL (~(st_hash_t) 0)
322#define RESERVED_HASH_SUBSTITUTION_VAL ((st_hash_t) 0)
324static inline st_hash_t
325normalize_hash_value(st_hash_t hash)
329 return hash == RESERVED_HASH_VAL ? RESERVED_HASH_SUBSTITUTION_VAL : hash;
333static inline st_hash_t
334do_hash(st_data_t key, st_table *tab)
336 st_hash_t hash = (st_hash_t)(tab->type->hash)(key);
337 return normalize_hash_value(hash);
341#define MINIMAL_POWER2 2
343#if MINIMAL_POWER2 < 2
344#error "MINIMAL_POWER2 should be >= 2"
349#define MAX_POWER2_FOR_TABLES_WITHOUT_BINS 4
353get_power2(st_index_t size)
355 unsigned int n = ST_INDEX_BITS - nlz_intptr(size);
357 return n < MINIMAL_POWER2 ? MINIMAL_POWER2 : n;
368static inline st_index_t
369get_bin(st_index_t *bins,
int s, st_index_t n)
371 return (s == 0 ? ((
unsigned char *) bins)[n]
372 : s == 1 ? ((
unsigned short *) bins)[n]
373 : s == 2 ? ((
unsigned int *) bins)[n]
374 : ((st_index_t *) bins)[n]);
380set_bin(st_index_t *bins,
int s, st_index_t n, st_index_t v)
382 if (s == 0) ((
unsigned char *) bins)[n] = (
unsigned char) v;
383 else if (s == 1) ((
unsigned short *) bins)[n] = (
unsigned short) v;
384 else if (s == 2) ((
unsigned int *) bins)[n] = (
unsigned int) v;
385 else ((st_index_t *) bins)[n] = v;
398#define MARK_BIN_EMPTY(tab, i) (set_bin((tab)->bins, get_size_ind(tab), i, EMPTY_BIN))
402#define UNDEFINED_ENTRY_IND (~(st_index_t) 0)
403#define UNDEFINED_BIN_IND (~(st_index_t) 0)
407#define REBUILT_TABLE_ENTRY_IND (~(st_index_t) 1)
408#define REBUILT_TABLE_BIN_IND (~(st_index_t) 1)
413#define MARK_BIN_DELETED(tab, i) \
415 set_bin((tab)->bins, get_size_ind(tab), i, DELETED_BIN); \
420#define EMPTY_BIN_P(b) ((b) == EMPTY_BIN)
421#define DELETED_BIN_P(b) ((b) == DELETED_BIN)
422#define EMPTY_OR_DELETED_BIN_P(b) ((b) <= DELETED_BIN)
426#define IND_EMPTY_BIN_P(tab, i) (EMPTY_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
427#define IND_DELETED_BIN_P(tab, i) (DELETED_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
428#define IND_EMPTY_OR_DELETED_BIN_P(tab, i) (EMPTY_OR_DELETED_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
432#define MARK_ENTRY_DELETED(e_ptr) ((e_ptr)->hash = RESERVED_HASH_VAL)
433#define DELETED_ENTRY_P(e_ptr) ((e_ptr)->hash == RESERVED_HASH_VAL)
436static inline unsigned int
437get_size_ind(
const st_table *tab)
439 return tab->size_ind;
443static inline st_index_t
444get_bins_num(
const st_table *tab)
446 return ((st_index_t) 1)<<tab->bin_power;
450static inline st_index_t
451bins_mask(
const st_table *tab)
453 return get_bins_num(tab) - 1;
458static inline st_index_t
459hash_bin(st_hash_t hash_value, st_table *tab)
461 return hash_value & bins_mask(tab);
465static inline st_index_t
466get_allocated_entries(
const st_table *tab)
468 return ((st_index_t) 1)<<tab->entry_power;
472static inline st_index_t
473bins_size(
const st_table *tab)
475 return features[tab->entry_power].bins_words *
sizeof (st_index_t);
480initialize_bins(st_table *tab)
482 memset(tab->bins, 0, bins_size(tab));
487make_tab_empty(st_table *tab)
489 tab->num_entries = 0;
490 tab->entries_start = tab->entries_bound = 0;
491 if (tab->bins != NULL)
492 initialize_bins(tab);
500 int all, total, num, str, strcase;
505static int init_st = 0;
512 char fname[10+
sizeof(long)*3];
514 if (!collision.total)
return;
515 f = fopen((snprintf(fname,
sizeof(fname),
"/tmp/col%ld", (
long)getpid()), fname),
"w");
518 fprintf(f,
"collision: %d / %d (%6.2f)\n", collision.all, collision.total,
519 ((
double)collision.all / (collision.total)) * 100);
520 fprintf(f,
"num: %d, str: %d, strcase: %d\n", collision.num, collision.str, collision.strcase);
526st_init_existing_table_with_size(st_table *tab,
const struct st_hash_type *type, st_index_t size)
533 const char *e = getenv(
"ST_HASH_LOG");
534 if (!e || !*e) init_st = 1;
543 n = get_power2(size);
550 tab->entry_power = n;
551 tab->bin_power = features[n].bin_power;
552 tab->size_ind = features[n].size_ind;
553 if (n <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
556 tab->bins = (st_index_t *) malloc(bins_size(tab));
558 if (tab->bins == NULL) {
564 tab->entries = (st_table_entry *) malloc(get_allocated_entries(tab)
565 *
sizeof(st_table_entry));
567 if (tab->entries == NULL) {
573 tab->rebuilds_num = 0;
581st_init_table_with_size(
const struct st_hash_type *type, st_index_t size)
583 st_table *tab = malloc(
sizeof(st_table));
590 st_init_existing_table_with_size(tab, type, size);
592 if (st_init_existing_table_with_size(tab, type, size) == NULL) {
602st_table_size(
const struct st_table *tbl)
604 return tbl->num_entries;
610st_init_table(
const struct st_hash_type *type)
612 return st_init_table_with_size(type, 0);
618st_init_numtable(
void)
620 return st_init_table(&type_numhash);
625st_init_numtable_with_size(st_index_t size)
627 return st_init_table_with_size(&type_numhash, size);
633st_init_strtable(
void)
635 return st_init_table(&type_strhash);
640st_init_strtable_with_size(st_index_t size)
642 return st_init_table_with_size(&type_strhash, size);
648st_init_strcasetable(
void)
650 return st_init_table(&type_strcasehash);
656st_init_strcasetable_with_size(st_index_t size)
658 return st_init_table_with_size(&type_strcasehash, size);
663st_clear(st_table *tab)
671st_free_table(st_table *tab)
680st_memsize(
const st_table *tab)
683 return(
sizeof(st_table)
684 + (tab->bins == NULL ? 0 : bins_size(tab))
685 + get_allocated_entries(tab) *
sizeof(st_table_entry));
689find_table_entry_ind(st_table *tab, st_hash_t hash_value, st_data_t key);
692find_table_bin_ind(st_table *tab, st_hash_t hash_value, st_data_t key);
695find_table_bin_ind_direct(st_table *table, st_hash_t hash_value, st_data_t key);
698find_table_bin_ptr_and_reserve(st_table *tab, st_hash_t *hash_value,
699 st_data_t key, st_index_t *bin_ind);
703count_collision(
const struct st_hash_type *type)
706 if (type == &type_numhash) {
709 else if (type == &type_strhash) {
712 else if (type == &type_strcasehash) {
717#define COLLISION (collision_check ? count_collision(tab->type) : (void)0)
718#define FOUND_BIN (collision_check ? collision.total++ : (void)0)
719#define collision_check 0
728#define REBUILD_THRESHOLD 4
730#if REBUILD_THRESHOLD < 2
731#error "REBUILD_THRESHOLD should be >= 2"
734static void rebuild_table_with(st_table *
const new_tab, st_table *
const tab);
735static void rebuild_move_table(st_table *
const new_tab, st_table *
const tab);
736static void rebuild_cleanup(st_table *
const tab);
743rebuild_table(st_table *tab)
745 if ((2 * tab->num_entries <= get_allocated_entries(tab)
746 && REBUILD_THRESHOLD * tab->num_entries > get_allocated_entries(tab))
747 || tab->num_entries < (1 << MINIMAL_POWER2)) {
749 tab->num_entries = 0;
750 if (tab->bins != NULL)
751 initialize_bins(tab);
752 rebuild_table_with(tab, tab);
759 new_tab = st_init_table_with_size(tab->type,
760 2 * tab->num_entries - 1);
761 rebuild_table_with(new_tab, tab);
762 rebuild_move_table(new_tab, tab);
764 rebuild_cleanup(tab);
768rebuild_table_with(st_table *
const new_tab, st_table *
const tab)
771 unsigned int size_ind;
772 st_table_entry *new_entries;
773 st_table_entry *curr_entry_ptr;
777 new_entries = new_tab->entries;
780 bins = new_tab->bins;
781 size_ind = get_size_ind(new_tab);
782 st_index_t bound = tab->entries_bound;
783 st_table_entry *entries = tab->entries;
785 for (i = tab->entries_start; i < bound; i++) {
786 curr_entry_ptr = &entries[i];
787 PREFETCH(entries + i + 1, 0);
788 if (EXPECT(DELETED_ENTRY_P(curr_entry_ptr), 0))
790 if (&new_entries[ni] != curr_entry_ptr)
791 new_entries[ni] = *curr_entry_ptr;
792 if (EXPECT(bins != NULL, 1)) {
793 bin_ind = find_table_bin_ind_direct(new_tab, curr_entry_ptr->hash,
794 curr_entry_ptr->key);
795 set_bin(bins, size_ind, bin_ind, ni + ENTRY_BASE);
797 new_tab->num_entries++;
801 assert(new_tab->num_entries == tab->num_entries);
805rebuild_move_table(st_table *
const new_tab, st_table *
const tab)
807 tab->entry_power = new_tab->entry_power;
808 tab->bin_power = new_tab->bin_power;
809 tab->size_ind = new_tab->size_ind;
811 tab->bins = new_tab->bins;
813 tab->entries = new_tab->entries;
818rebuild_cleanup(st_table *
const tab)
820 tab->entries_start = 0;
821 tab->entries_bound = tab->num_entries;
837static inline st_index_t
838secondary_hash(st_index_t ind, st_table *tab, st_index_t *perturb)
841 ind = (ind << 2) + ind + *perturb + 1;
842 return hash_bin(ind, tab);
849static inline st_index_t
850find_entry(st_table *tab, st_hash_t hash_value, st_data_t key)
854 st_table_entry *entries;
856 bound = tab->entries_bound;
857 entries = tab->entries;
858 for (i = tab->entries_start; i < bound; i++) {
859 DO_PTR_EQUAL_CHECK(tab, &entries[i], hash_value, key, eq_p, rebuilt_p);
860 if (EXPECT(rebuilt_p, 0))
861 return REBUILT_TABLE_ENTRY_IND;
865 return UNDEFINED_ENTRY_IND;
877find_table_entry_ind(st_table *tab, st_hash_t hash_value, st_data_t key)
881#ifdef QUADRATIC_PROBE
887 st_table_entry *entries = tab->entries;
889 ind = hash_bin(hash_value, tab);
890#ifdef QUADRATIC_PROBE
893 perturb = hash_value;
897 bin = get_bin(tab->bins, get_size_ind(tab), ind);
898 if (! EMPTY_OR_DELETED_BIN_P(bin)) {
899 DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
900 if (EXPECT(rebuilt_p, 0))
901 return REBUILT_TABLE_ENTRY_IND;
905 else if (EMPTY_BIN_P(bin))
906 return UNDEFINED_ENTRY_IND;
907#ifdef QUADRATIC_PROBE
908 ind = hash_bin(ind + d, tab);
911 ind = secondary_hash(ind, tab, &perturb);
923find_table_bin_ind(st_table *tab, st_hash_t hash_value, st_data_t key)
927#ifdef QUADRATIC_PROBE
933 st_table_entry *entries = tab->entries;
935 ind = hash_bin(hash_value, tab);
936#ifdef QUADRATIC_PROBE
939 perturb = hash_value;
943 bin = get_bin(tab->bins, get_size_ind(tab), ind);
944 if (! EMPTY_OR_DELETED_BIN_P(bin)) {
945 DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
946 if (EXPECT(rebuilt_p, 0))
947 return REBUILT_TABLE_BIN_IND;
951 else if (EMPTY_BIN_P(bin))
952 return UNDEFINED_BIN_IND;
953#ifdef QUADRATIC_PROBE
954 ind = hash_bin(ind + d, tab);
957 ind = secondary_hash(ind, tab, &perturb);
968find_table_bin_ind_direct(st_table *tab, st_hash_t hash_value, st_data_t key)
971#ifdef QUADRATIC_PROBE
978 ind = hash_bin(hash_value, tab);
979#ifdef QUADRATIC_PROBE
982 perturb = hash_value;
986 bin = get_bin(tab->bins, get_size_ind(tab), ind);
987 if (EMPTY_OR_DELETED_BIN_P(bin))
989#ifdef QUADRATIC_PROBE
990 ind = hash_bin(ind + d, tab);
993 ind = secondary_hash(ind, tab, &perturb);
1009find_table_bin_ptr_and_reserve(st_table *tab, st_hash_t *hash_value,
1010 st_data_t key, st_index_t *bin_ind)
1012 int eq_p, rebuilt_p;
1014 st_hash_t curr_hash_value = *hash_value;
1015#ifdef QUADRATIC_PROBE
1020 st_index_t entry_index;
1021 st_index_t first_deleted_bin_ind;
1022 st_table_entry *entries;
1024 ind = hash_bin(curr_hash_value, tab);
1025#ifdef QUADRATIC_PROBE
1028 perturb = curr_hash_value;
1031 first_deleted_bin_ind = UNDEFINED_BIN_IND;
1032 entries = tab->entries;
1034 entry_index = get_bin(tab->bins, get_size_ind(tab), ind);
1035 if (EMPTY_BIN_P(entry_index)) {
1037 entry_index = UNDEFINED_ENTRY_IND;
1038 if (first_deleted_bin_ind != UNDEFINED_BIN_IND) {
1040 ind = first_deleted_bin_ind;
1041 MARK_BIN_EMPTY(tab, ind);
1045 else if (! DELETED_BIN_P(entry_index)) {
1046 DO_PTR_EQUAL_CHECK(tab, &entries[entry_index - ENTRY_BASE], curr_hash_value, key, eq_p, rebuilt_p);
1047 if (EXPECT(rebuilt_p, 0))
1048 return REBUILT_TABLE_ENTRY_IND;
1052 else if (first_deleted_bin_ind == UNDEFINED_BIN_IND)
1053 first_deleted_bin_ind = ind;
1054#ifdef QUADRATIC_PROBE
1055 ind = hash_bin(ind + d, tab);
1058 ind = secondary_hash(ind, tab, &perturb);
1069st_lookup(st_table *tab, st_data_t key, st_data_t *value)
1072 st_hash_t hash = do_hash(key, tab);
1075 if (tab->bins == NULL) {
1076 bin = find_entry(tab, hash, key);
1077 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1079 if (bin == UNDEFINED_ENTRY_IND)
1083 bin = find_table_entry_ind(tab, hash, key);
1084 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1086 if (bin == UNDEFINED_ENTRY_IND)
1091 *value = tab->entries[bin].record;
1098st_get_key(st_table *tab, st_data_t key, st_data_t *result)
1101 st_hash_t hash = do_hash(key, tab);
1104 if (tab->bins == NULL) {
1105 bin = find_entry(tab, hash, key);
1106 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1108 if (bin == UNDEFINED_ENTRY_IND)
1112 bin = find_table_entry_ind(tab, hash, key);
1113 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1115 if (bin == UNDEFINED_ENTRY_IND)
1120 *result = tab->entries[bin].key;
1126rebuild_table_if_necessary (st_table *tab)
1128 st_index_t bound = tab->entries_bound;
1130 if (bound == get_allocated_entries(tab))
1138st_insert(st_table *tab, st_data_t key, st_data_t value)
1140 st_table_entry *entry;
1143 st_hash_t hash_value;
1147 hash_value = do_hash(key, tab);
1149 rebuild_table_if_necessary(tab);
1150 if (tab->bins == NULL) {
1151 bin = find_entry(tab, hash_value, key);
1152 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1154 new_p = bin == UNDEFINED_ENTRY_IND;
1157 bin_ind = UNDEFINED_BIN_IND;
1160 bin = find_table_bin_ptr_and_reserve(tab, &hash_value,
1162 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1164 new_p = bin == UNDEFINED_ENTRY_IND;
1168 ind = tab->entries_bound++;
1169 entry = &tab->entries[ind];
1170 entry->hash = hash_value;
1172 entry->record = value;
1173 if (bin_ind != UNDEFINED_BIN_IND)
1174 set_bin(tab->bins, get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
1177 tab->entries[bin].record = value;
1184st_add_direct_with_hash(st_table *tab,
1185 st_data_t key, st_data_t value, st_hash_t hash)
1187 st_table_entry *entry;
1191 assert(hash != RESERVED_HASH_VAL);
1193 rebuild_table_if_necessary(tab);
1194 ind = tab->entries_bound++;
1195 entry = &tab->entries[ind];
1198 entry->record = value;
1200 if (tab->bins != NULL) {
1201 bin_ind = find_table_bin_ind_direct(tab, hash, key);
1202 set_bin(tab->bins, get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
1207rb_st_add_direct_with_hash(st_table *tab,
1208 st_data_t key, st_data_t value, st_hash_t hash)
1210 st_add_direct_with_hash(tab, key, value, normalize_hash_value(hash));
1216st_add_direct(st_table *tab, st_data_t key, st_data_t value)
1218 st_hash_t hash_value;
1220 hash_value = do_hash(key, tab);
1221 st_add_direct_with_hash(tab, key, value, hash_value);
1228st_insert2(st_table *tab, st_data_t key, st_data_t value,
1229 st_data_t (*func)(st_data_t))
1231 st_table_entry *entry;
1234 st_hash_t hash_value;
1238 hash_value = do_hash(key, tab);
1240 rebuild_table_if_necessary (tab);
1241 if (tab->bins == NULL) {
1242 bin = find_entry(tab, hash_value, key);
1243 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1245 new_p = bin == UNDEFINED_ENTRY_IND;
1248 bin_ind = UNDEFINED_BIN_IND;
1251 bin = find_table_bin_ptr_and_reserve(tab, &hash_value,
1253 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1255 new_p = bin == UNDEFINED_ENTRY_IND;
1260 ind = tab->entries_bound++;
1261 entry = &tab->entries[ind];
1262 entry->hash = hash_value;
1264 entry->record = value;
1265 if (bin_ind != UNDEFINED_BIN_IND)
1266 set_bin(tab->bins, get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
1269 tab->entries[bin].record = value;
1275st_replace(st_table *new_tab, st_table *old_tab)
1277 *new_tab = *old_tab;
1278 if (old_tab->bins == NULL)
1279 new_tab->bins = NULL;
1281 new_tab->bins = (st_index_t *) malloc(bins_size(old_tab));
1283 if (new_tab->bins == NULL) {
1288 new_tab->entries = (st_table_entry *) malloc(get_allocated_entries(old_tab)
1289 *
sizeof(st_table_entry));
1291 if (new_tab->entries == NULL) {
1295 MEMCPY(new_tab->entries, old_tab->entries, st_table_entry,
1296 get_allocated_entries(old_tab));
1297 if (old_tab->bins != NULL)
1298 MEMCPY(new_tab->bins, old_tab->bins,
char, bins_size(old_tab));
1305st_copy(st_table *old_tab)
1309 new_tab = (st_table *) malloc(
sizeof(st_table));
1311 if (new_tab == NULL)
1315 if (st_replace(new_tab, old_tab) == NULL) {
1316 st_free_table(new_tab);
1326update_range_for_deleted(st_table *tab, st_index_t n)
1330 if (tab->entries_start == n) {
1331 st_index_t start = n + 1;
1332 st_index_t bound = tab->entries_bound;
1333 st_table_entry *entries = tab->entries;
1334 while (start < bound && DELETED_ENTRY_P(&entries[start])) start++;
1335 tab->entries_start = start;
1344st_general_delete(st_table *tab, st_data_t *key, st_data_t *value)
1346 st_table_entry *entry;
1351 hash = do_hash(*key, tab);
1353 if (tab->bins == NULL) {
1354 bin = find_entry(tab, hash, *key);
1355 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1357 if (bin == UNDEFINED_ENTRY_IND) {
1358 if (value != 0) *value = 0;
1363 bin_ind = find_table_bin_ind(tab, hash, *key);
1364 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
1366 if (bin_ind == UNDEFINED_BIN_IND) {
1367 if (value != 0) *value = 0;
1370 bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
1371 MARK_BIN_DELETED(tab, bin_ind);
1373 entry = &tab->entries[bin];
1375 if (value != 0) *value = entry->record;
1376 MARK_ENTRY_DELETED(entry);
1378 update_range_for_deleted(tab, bin);
1383st_delete(st_table *tab, st_data_t *key, st_data_t *value)
1385 return st_general_delete(tab, key, value);
1394st_delete_safe(st_table *tab, st_data_t *key, st_data_t *value,
1395 st_data_t never ATTRIBUTE_UNUSED)
1397 return st_general_delete(tab, key, value);
1405st_shift(st_table *tab, st_data_t *key, st_data_t *value)
1407 st_index_t i, bound;
1409 st_table_entry *entries, *curr_entry_ptr;
1412 entries = tab->entries;
1413 bound = tab->entries_bound;
1414 for (i = tab->entries_start; i < bound; i++) {
1415 curr_entry_ptr = &entries[i];
1416 if (! DELETED_ENTRY_P(curr_entry_ptr)) {
1417 st_hash_t entry_hash = curr_entry_ptr->hash;
1418 st_data_t entry_key = curr_entry_ptr->key;
1420 if (value != 0) *value = curr_entry_ptr->record;
1423 if (tab->bins == NULL) {
1424 bin = find_entry(tab, entry_hash, entry_key);
1425 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0)) {
1426 entries = tab->entries;
1429 curr_entry_ptr = &entries[bin];
1432 bin_ind = find_table_bin_ind(tab, entry_hash, entry_key);
1433 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0)) {
1434 entries = tab->entries;
1437 curr_entry_ptr = &entries[get_bin(tab->bins, get_size_ind(tab), bin_ind)
1439 MARK_BIN_DELETED(tab, bin_ind);
1441 MARK_ENTRY_DELETED(curr_entry_ptr);
1443 update_range_for_deleted(tab, i);
1447 if (value != 0) *value = 0;
1453st_cleanup_safe(st_table *tab ATTRIBUTE_UNUSED,
1454 st_data_t never ATTRIBUTE_UNUSED)
1468st_update(st_table *tab, st_data_t key,
1469 st_update_callback_func *func, st_data_t arg)
1471 st_table_entry *entry = NULL;
1473 st_table_entry *entries;
1475 st_data_t value = 0, old_key;
1476 int retval, existing;
1477 st_hash_t hash = do_hash(key, tab);
1480 entries = tab->entries;
1481 if (tab->bins == NULL) {
1482 bin = find_entry(tab, hash, key);
1483 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1485 existing = bin != UNDEFINED_ENTRY_IND;
1486 entry = &entries[bin];
1487 bin_ind = UNDEFINED_BIN_IND;
1490 bin_ind = find_table_bin_ind(tab, hash, key);
1491 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
1493 existing = bin_ind != UNDEFINED_BIN_IND;
1495 bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
1496 entry = &entries[bin];
1501 value = entry->record;
1505 unsigned int rebuilds_num = tab->rebuilds_num;
1507 retval = (*func)(&key, &value, arg, existing);
1511 assert(rebuilds_num == tab->rebuilds_num);
1517 st_add_direct_with_hash(tab, key, value, hash);
1520 if (old_key != key) {
1523 entry->record = value;
1527 if (bin_ind != UNDEFINED_BIN_IND)
1528 MARK_BIN_DELETED(tab, bin_ind);
1529 MARK_ENTRY_DELETED(entry);
1531 update_range_for_deleted(tab, bin);
1547st_general_foreach(st_table *tab, st_foreach_check_callback_func *func, st_update_callback_func *replace, st_data_t arg,
1552 st_table_entry *entries, *curr_entry_ptr;
1553 enum st_retval retval;
1554 st_index_t i, rebuilds_num;
1557 int error_p, packed_p = tab->bins == NULL;
1559 entries = tab->entries;
1562 for (i = tab->entries_start; i < tab->entries_bound; i++) {
1563 curr_entry_ptr = &entries[i];
1564 if (EXPECT(DELETED_ENTRY_P(curr_entry_ptr), 0))
1566 key = curr_entry_ptr->key;
1567 rebuilds_num = tab->rebuilds_num;
1568 hash = curr_entry_ptr->hash;
1569 retval = (*func)(key, curr_entry_ptr->record, arg, 0);
1571 if (retval == ST_REPLACE && replace) {
1573 value = curr_entry_ptr->record;
1574 retval = (*replace)(&key, &value, arg, TRUE);
1575 curr_entry_ptr->key = key;
1576 curr_entry_ptr->record = value;
1579 if (rebuilds_num != tab->rebuilds_num) {
1581 entries = tab->entries;
1582 packed_p = tab->bins == NULL;
1584 i = find_entry(tab, hash, key);
1585 if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
1587 error_p = i == UNDEFINED_ENTRY_IND;
1590 i = find_table_entry_ind(tab, hash, key);
1591 if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
1593 error_p = i == UNDEFINED_ENTRY_IND;
1596 if (error_p && check_p) {
1598 retval = (*func)(0, 0, arg, 1);
1601 curr_entry_ptr = &entries[i];
1614 st_data_t key = curr_entry_ptr->key;
1618 bin = find_entry(tab, hash, key);
1619 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
1621 if (bin == UNDEFINED_ENTRY_IND)
1625 bin_ind = find_table_bin_ind(tab, hash, key);
1626 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
1628 if (bin_ind == UNDEFINED_BIN_IND)
1630 bin = get_bin(tab->bins, get_size_ind(tab), bin_ind) - ENTRY_BASE;
1631 MARK_BIN_DELETED(tab, bin_ind);
1633 curr_entry_ptr = &entries[bin];
1634 MARK_ENTRY_DELETED(curr_entry_ptr);
1636 update_range_for_deleted(tab, bin);
1645st_foreach_with_replace(st_table *tab, st_foreach_check_callback_func *func, st_update_callback_func *replace, st_data_t arg)
1647 return st_general_foreach(tab, func, replace, arg, TRUE);
1651 st_foreach_callback_func *func;
1656apply_functor(st_data_t k, st_data_t v, st_data_t d,
int _)
1658 const struct functor *f = (
void *)d;
1659 return f->func(k, v, f->arg);
1663st_foreach(st_table *tab, st_foreach_callback_func *func, st_data_t arg)
1665 const struct functor f = { func, arg };
1666 return st_general_foreach(tab, apply_functor, 0, (st_data_t)&f, FALSE);
1671st_foreach_check(st_table *tab, st_foreach_check_callback_func *func, st_data_t arg,
1672 st_data_t never ATTRIBUTE_UNUSED)
1674 return st_general_foreach(tab, func, 0, arg, TRUE);
1679static inline st_index_t
1680st_general_keys(st_table *tab, st_data_t *keys, st_index_t size)
1682 st_index_t i, bound;
1683 st_data_t key, *keys_start, *keys_end;
1684 st_table_entry *curr_entry_ptr, *entries = tab->entries;
1686 bound = tab->entries_bound;
1688 keys_end = keys + size;
1689 for (i = tab->entries_start; i < bound; i++) {
1690 if (keys == keys_end)
1692 curr_entry_ptr = &entries[i];
1693 key = curr_entry_ptr->key;
1694 if (! DELETED_ENTRY_P(curr_entry_ptr))
1698 return keys - keys_start;
1702st_keys(st_table *tab, st_data_t *keys, st_index_t size)
1704 return st_general_keys(tab, keys, size);
1709st_keys_check(st_table *tab, st_data_t *keys, st_index_t size,
1710 st_data_t never ATTRIBUTE_UNUSED)
1712 return st_general_keys(tab, keys, size);
1717static inline st_index_t
1718st_general_values(st_table *tab, st_data_t *values, st_index_t size)
1720 st_index_t i, bound;
1721 st_data_t *values_start, *values_end;
1722 st_table_entry *curr_entry_ptr, *entries = tab->entries;
1724 values_start = values;
1725 values_end = values + size;
1726 bound = tab->entries_bound;
1727 for (i = tab->entries_start; i < bound; i++) {
1728 if (values == values_end)
1730 curr_entry_ptr = &entries[i];
1731 if (! DELETED_ENTRY_P(curr_entry_ptr))
1732 *values++ = curr_entry_ptr->record;
1735 return values - values_start;
1739st_values(st_table *tab, st_data_t *values, st_index_t size)
1741 return st_general_values(tab, values, size);
1746st_values_check(st_table *tab, st_data_t *values, st_index_t size,
1747 st_data_t never ATTRIBUTE_UNUSED)
1749 return st_general_values(tab, values, size);
1752#define FNV1_32A_INIT 0x811c9dc5
1757#define FNV_32_PRIME 0x01000193
1760#ifndef UNALIGNED_WORD_ACCESS
1761# if defined(__i386) || defined(__i386__) || defined(_M_IX86) || \
1762 defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || \
1763 defined(__powerpc64__) || defined(__POWERPC__) || defined(__aarch64__) || \
1764 defined(__mc68020__)
1765# define UNALIGNED_WORD_ACCESS 1
1768#ifndef UNALIGNED_WORD_ACCESS
1769# define UNALIGNED_WORD_ACCESS 0
1775#define BIG_CONSTANT(x,y) ((st_index_t)(x)<<32|(st_index_t)(y))
1776#define ROTL(x,n) ((x)<<(n)|(x)>>(SIZEOF_ST_INDEX_T*CHAR_BIT-(n)))
1778#if ST_INDEX_BITS <= 32
1779#define C1 (st_index_t)0xcc9e2d51
1780#define C2 (st_index_t)0x1b873593
1782#define C1 BIG_CONSTANT(0x87c37b91,0x114253d5);
1783#define C2 BIG_CONSTANT(0x4cf5ad43,0x2745937f);
1785NO_SANITIZE(
"unsigned-integer-overflow",
static inline st_index_t murmur_step(st_index_t h, st_index_t k));
1786NO_SANITIZE(
"unsigned-integer-overflow",
static inline st_index_t murmur_finish(st_index_t h));
1787NO_SANITIZE(
"unsigned-integer-overflow",
extern st_index_t st_hash(
const void *
ptr,
size_t len, st_index_t h));
1789static inline st_index_t
1790murmur_step(st_index_t h, st_index_t k)
1792#if ST_INDEX_BITS <= 32
1808static inline st_index_t
1809murmur_finish(st_index_t h)
1811#if ST_INDEX_BITS <= 32
1815 const st_index_t c1 = 0x85ebca6b;
1816 const st_index_t c2 = 0xc2b2ae35;
1822 const st_index_t c1 = BIG_CONSTANT(0xbf58476d,0x1ce4e5b9);
1823 const st_index_t c2 = BIG_CONSTANT(0x94d049bb,0x133111eb);
1825#if ST_INDEX_BITS > 64
1842st_hash(
const void *
ptr,
size_t len, st_index_t h)
1844 const char *data =
ptr;
1848#define data_at(n) (st_index_t)((unsigned char)data[(n)])
1849#define UNALIGNED_ADD_4 UNALIGNED_ADD(2); UNALIGNED_ADD(1); UNALIGNED_ADD(0)
1850#if SIZEOF_ST_INDEX_T > 4
1851#define UNALIGNED_ADD_8 UNALIGNED_ADD(6); UNALIGNED_ADD(5); UNALIGNED_ADD(4); UNALIGNED_ADD(3); UNALIGNED_ADD_4
1852#if SIZEOF_ST_INDEX_T > 8
1853#define UNALIGNED_ADD_16 UNALIGNED_ADD(14); UNALIGNED_ADD(13); UNALIGNED_ADD(12); UNALIGNED_ADD(11); \
1854 UNALIGNED_ADD(10); UNALIGNED_ADD(9); UNALIGNED_ADD(8); UNALIGNED_ADD(7); UNALIGNED_ADD_8
1855#define UNALIGNED_ADD_ALL UNALIGNED_ADD_16
1857#define UNALIGNED_ADD_ALL UNALIGNED_ADD_8
1859#define UNALIGNED_ADD_ALL UNALIGNED_ADD_4
1862 if (
len >=
sizeof(st_index_t)) {
1863#if !UNALIGNED_WORD_ACCESS
1864 int align = (int)((st_data_t)data %
sizeof(st_index_t));
1870#ifdef WORDS_BIGENDIAN
1871# define UNALIGNED_ADD(n) case SIZEOF_ST_INDEX_T - (n) - 1: \
1872 t |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 2)
1874# define UNALIGNED_ADD(n) case SIZEOF_ST_INDEX_T - (n) - 1: \
1875 t |= data_at(n) << CHAR_BIT*(n)
1881#ifdef WORDS_BIGENDIAN
1882 t >>= (CHAR_BIT * align) - CHAR_BIT;
1884 t <<= (CHAR_BIT * align);
1887 data +=
sizeof(st_index_t)-align;
1888 len -=
sizeof(st_index_t)-align;
1890 sl = CHAR_BIT * (SIZEOF_ST_INDEX_T-align);
1891 sr = CHAR_BIT * align;
1893 while (
len >=
sizeof(st_index_t)) {
1894 d = *(st_index_t *)data;
1895#ifdef WORDS_BIGENDIAN
1896 t = (t << sr) | (d >> sl);
1898 t = (t >> sr) | (d << sl);
1900 h = murmur_step(h, t);
1902 data +=
sizeof(st_index_t);
1903 len -=
sizeof(st_index_t);
1906 pack =
len < (size_t)align ? (
int)
len : align;
1909#ifdef WORDS_BIGENDIAN
1910# define UNALIGNED_ADD(n) case (n) + 1: \
1911 d |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 1)
1913# define UNALIGNED_ADD(n) case (n) + 1: \
1914 d |= data_at(n) << CHAR_BIT*(n)
1919#ifdef WORDS_BIGENDIAN
1920 t = (t << sr) | (d >> sl);
1922 t = (t >> sr) | (d << sl);
1925 if (
len < (
size_t)align)
goto skip_tail;
1927 h = murmur_step(h, t);
1933#ifdef HAVE_BUILTIN___BUILTIN_ASSUME_ALIGNED
1934#define aligned_data __builtin_assume_aligned(data, sizeof(st_index_t))
1936#define aligned_data data
1940 h = murmur_step(h, *(st_index_t *)aligned_data);
1941 data +=
sizeof(st_index_t);
1942 len -=
sizeof(st_index_t);
1943 }
while (
len >=
sizeof(st_index_t));
1949#if UNALIGNED_WORD_ACCESS && SIZEOF_ST_INDEX_T <= 8 && CHAR_BIT == 8
1951#if SIZEOF_ST_INDEX_T > 4
1952 case 7: t |= data_at(6) << 48;
1953 case 6: t |= data_at(5) << 40;
1954 case 5: t |= data_at(4) << 32;
1956 t |= (st_index_t)*(uint32_t*)aligned_data;
1960 case 3: t |= data_at(2) << 16;
1961 case 2: t |= data_at(1) << 8;
1962 case 1: t |= data_at(0);
1964#ifdef WORDS_BIGENDIAN
1965# define UNALIGNED_ADD(n) case (n) + 1: \
1966 t |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 1)
1968# define UNALIGNED_ADD(n) case (n) + 1: \
1969 t |= data_at(n) << CHAR_BIT*(n)
1977 h ^= t; h -= ROTL(t, 7);
1983 return murmur_finish(h);
1987st_hash_uint32(st_index_t h, uint32_t i)
1989 return murmur_step(h, i);
1992NO_SANITIZE(
"unsigned-integer-overflow",
extern st_index_t st_hash_uint(st_index_t h, st_index_t i));
1994st_hash_uint(st_index_t h, st_index_t i)
1999#if SIZEOF_ST_INDEX_T*CHAR_BIT > 8*8
2000 h = murmur_step(h, i >> 8*8);
2002 h = murmur_step(h, i);
2007st_hash_end(st_index_t h)
2009 h = murmur_finish(h);
2015rb_st_hash_start(st_index_t h)
2021strhash(st_data_t arg)
2023 register const char *
string = (
const char *)arg;
2024 return st_hash(
string, strlen(
string), FNV1_32A_INIT);
2028st_locale_insensitive_strcasecmp(
const char *s1,
const char *s2)
2035 if (c1 ==
'\0' || c2 ==
'\0') {
2036 if (c1 !=
'\0')
return 1;
2037 if (c2 !=
'\0')
return -1;
2040 if ((
'A' <= c1) && (c1 <=
'Z')) c1 +=
'a' -
'A';
2041 if ((
'A' <= c2) && (c2 <=
'Z')) c2 +=
'a' -
'A';
2052st_locale_insensitive_strncasecmp(
const char *s1,
const char *s2,
size_t n)
2057 for (i = 0; i < n; i++) {
2060 if (c1 ==
'\0' || c2 ==
'\0') {
2061 if (c1 !=
'\0')
return 1;
2062 if (c2 !=
'\0')
return -1;
2065 if ((
'A' <= c1) && (c1 <=
'Z')) c1 +=
'a' -
'A';
2066 if ((
'A' <= c2) && (c2 <=
'Z')) c2 +=
'a' -
'A';
2078st_strcmp(st_data_t lhs, st_data_t rhs)
2080 const char *s1 = (
char *)lhs;
2081 const char *s2 = (
char *)rhs;
2082 return strcmp(s1, s2);
2086st_locale_insensitive_strcasecmp_i(st_data_t lhs, st_data_t rhs)
2088 const char *s1 = (
char *)lhs;
2089 const char *s2 = (
char *)rhs;
2090 return st_locale_insensitive_strcasecmp(s1, s2);
2093NO_SANITIZE(
"unsigned-integer-overflow", PUREFUNC(
static st_index_t strcasehash(st_data_t)));
2095strcasehash(st_data_t arg)
2097 register const char *
string = (
const char *)arg;
2098 register st_index_t hval = FNV1_32A_INIT;
2104 unsigned int c = (
unsigned char)*
string++;
2105 if ((
unsigned int)(c -
'A') <= (
'Z' -
'A')) c +=
'a' -
'A';
2109 hval *= FNV_32_PRIME;
2115st_numcmp(st_data_t x, st_data_t y)
2121st_numhash(st_data_t n)
2123 enum {s1 = 11, s2 = 3};
2124 return (st_index_t)((n>>s1|(n<<s2)) ^ (n>>s2));
2132st_expand_table(st_table *tab, st_index_t siz)
2137 if (siz <= get_allocated_entries(tab))
2140 tmp = st_init_table_with_size(tab->type, siz);
2141 n = get_allocated_entries(tab);
2142 MEMCPY(tmp->entries, tab->entries, st_table_entry, n);
2146 tab->entry_power = tmp->entry_power;
2147 tab->bin_power = tmp->bin_power;
2148 tab->size_ind = tmp->size_ind;
2149 tab->entries = tmp->entries;
2151 tab->rebuilds_num++;
2158st_rehash_linear(st_table *tab)
2160 int eq_p, rebuilt_p;
2162 st_table_entry *p, *q;
2167 for (i = tab->entries_start; i < tab->entries_bound; i++) {
2168 p = &tab->entries[i];
2169 if (DELETED_ENTRY_P(p))
2171 for (j = i + 1; j < tab->entries_bound; j++) {
2172 q = &tab->entries[j];
2173 if (DELETED_ENTRY_P(q))
2175 DO_PTR_EQUAL_CHECK(tab, p, q->hash, q->key, eq_p, rebuilt_p);
2176 if (EXPECT(rebuilt_p, 0))
2180 MARK_ENTRY_DELETED(q);
2182 update_range_for_deleted(tab, j);
2192st_rehash_indexed(st_table *tab)
2194 int eq_p, rebuilt_p;
2196 st_index_t
const n = bins_size(tab);
2197 unsigned int const size_ind = get_size_ind(tab);
2198 st_index_t *bins = realloc(tab->bins, n);
2200 initialize_bins(tab);
2201 for (i = tab->entries_start; i < tab->entries_bound; i++) {
2202 st_table_entry *p = &tab->entries[i];
2204#ifdef QUADRATIC_PROBE
2207 st_index_t perturb = p->hash;
2210 if (DELETED_ENTRY_P(p))
2213 ind = hash_bin(p->hash, tab);
2215 st_index_t bin = get_bin(bins, size_ind, ind);
2216 if (EMPTY_OR_DELETED_BIN_P(bin)) {
2218 set_bin(bins, size_ind, ind, i + ENTRY_BASE);
2222 st_table_entry *q = &tab->entries[bin - ENTRY_BASE];
2223 DO_PTR_EQUAL_CHECK(tab, q, p->hash, p->key, eq_p, rebuilt_p);
2224 if (EXPECT(rebuilt_p, 0))
2228 q->record = p->record;
2229 MARK_ENTRY_DELETED(p);
2231 update_range_for_deleted(tab, bin);
2236#ifdef QUADRATIC_PROBE
2237 ind = hash_bin(ind + d, tab);
2240 ind = secondary_hash(ind, tab, &perturb);
2253st_rehash(st_table *tab)
2258 if (tab->bin_power <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
2259 rebuilt_p = st_rehash_linear(tab);
2261 rebuilt_p = st_rehash_indexed(tab);
2262 }
while (rebuilt_p);
2266st_stringify(
VALUE key)
2269 rb_hash_key_str(key) : key;
2275 st_data_t k = st_stringify(key);
2277 e.hash = do_hash(k, tab);
2281 tab->entries[tab->entries_bound++] = e;
2288st_insert_linear(st_table *tab,
long argc,
const VALUE *argv,
VALUE hash)
2292 for (i = 0; i < argc; ) {
2293 st_data_t k = st_stringify(argv[i++]);
2294 st_data_t v = argv[i++];
2295 st_insert(tab, k, v);
2302st_insert_generic(st_table *tab,
long argc,
const VALUE *argv,
VALUE hash)
2307 for (i = 0; i < argc; ) {
2308 VALUE key = argv[i++];
2309 VALUE val = argv[i++];
2310 st_insert_single(tab, hash, key, val);
2320rb_hash_bulk_insert_into_st_table(
long argc,
const VALUE *argv,
VALUE hash)
2322 st_index_t n, size = argc / 2;
2323 st_table *tab = RHASH_ST_TABLE(hash);
2325 tab = RHASH_TBL_RAW(hash);
2326 n = tab->entries_bound + size;
2327 st_expand_table(tab, n);
2328 if (UNLIKELY(tab->num_entries))
2329 st_insert_generic(tab, argc, argv, hash);
2331 st_insert_single(tab, hash, argv[0], argv[1]);
2332 else if (tab->bin_power <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS)
2333 st_insert_linear(tab, argc, argv, hash);
2335 st_insert_generic(tab, argc, argv, hash);
2339rb_st_compact_table(st_table *tab)
2341 st_index_t num = tab->num_entries;
2342 if (REBUILD_THRESHOLD * num <= get_allocated_entries(tab)) {
2344 st_table *new_tab = st_init_table_with_size(tab->type, 2 * num);
2345 rebuild_table_with(new_tab, tab);
2346 rebuild_move_table(new_tab, tab);
2347 rebuild_cleanup(tab);
2355struct set_table_entry {
2361static inline st_hash_t
2362set_do_hash(st_data_t key,
set_table *tab)
2364 st_hash_t hash = (st_hash_t)(tab->type->hash)(key);
2365 return normalize_hash_value(hash);
2369static inline unsigned int
2372 return tab->size_ind;
2376static inline st_index_t
2379 return ((st_index_t) 1)<<tab->bin_power;
2383static inline st_index_t
2386 return set_get_bins_num(tab) - 1;
2391static inline st_index_t
2392set_hash_bin(st_hash_t hash_value,
set_table *tab)
2394 return hash_value & set_bins_mask(tab);
2398static inline st_index_t
2399set_get_allocated_entries(
const set_table *tab)
2401 return ((st_index_t) 1)<<tab->entry_power;
2405set_allocated_entries_size(
const set_table *tab)
2407 return set_get_allocated_entries(tab) *
sizeof(set_table_entry);
2413 return tab->entry_power > MAX_POWER2_FOR_TABLES_WITHOUT_BINS;
2417static inline st_index_t
2420 if (set_has_bins(tab)) {
2421 return features[tab->entry_power].bins_words *
sizeof (st_index_t);
2427static inline st_index_t *
2430 if (set_has_bins(tab)) {
2431 return (st_index_t *)(((
char *)tab->
entries) + set_allocated_entries_size(tab));
2441 memset(set_bins_ptr(tab), 0, set_bins_size(tab));
2448 tab->num_entries = 0;
2449 tab->entries_start = tab->entries_bound = 0;
2450 if (set_bins_ptr(tab) != NULL)
2451 set_initialize_bins(tab);
2455set_init_existing_table_with_size(
set_table *tab,
const struct st_hash_type *type, st_index_t size)
2462 const char *e = getenv(
"ST_HASH_LOG");
2463 if (!e || !*e) init_st = 1;
2472 n = get_power2(size);
2475 tab->entry_power = n;
2476 tab->bin_power = features[n].bin_power;
2477 tab->size_ind = features[n].size_ind;
2480 if (set_has_bins(tab)) {
2481 memsize += set_bins_size(tab);
2483 memsize += set_get_allocated_entries(tab) *
sizeof(set_table_entry);
2484 tab->
entries = (set_table_entry *)malloc(memsize);
2485 set_make_tab_empty(tab);
2486 tab->rebuilds_num = 0;
2494set_init_table_with_size(
set_table *tab,
const struct st_hash_type *type, st_index_t size)
2496 if (tab == NULL) tab = malloc(
sizeof(
set_table));
2498 set_init_existing_table_with_size(tab, type, size);
2504set_init_numtable(
void)
2506 return set_init_table_with_size(NULL, &type_numhash, 0);
2510set_init_numtable_with_size(st_index_t size)
2512 return set_init_table_with_size(NULL, &type_numhash, size);
2516set_table_size(
const struct set_table *tbl)
2518 return tbl->num_entries;
2525 set_make_tab_empty(tab);
2526 tab->rebuilds_num++;
2543 + (tab->entry_power <= MAX_POWER2_FOR_TABLES_WITHOUT_BINS ? 0 : set_bins_size(tab))
2544 + set_get_allocated_entries(tab) *
sizeof(set_table_entry));
2548set_find_table_entry_ind(
set_table *tab, st_hash_t hash_value, st_data_t key);
2551set_find_table_bin_ind(
set_table *tab, st_hash_t hash_value, st_data_t key);
2554set_find_table_bin_ind_direct(
set_table *table, st_hash_t hash_value, st_data_t key);
2557set_find_table_bin_ptr_and_reserve(
set_table *tab, st_hash_t *hash_value,
2558 st_data_t key, st_index_t *bin_ind);
2562static void set_rebuild_cleanup(
set_table *
const tab);
2571 if ((2 * tab->num_entries <= set_get_allocated_entries(tab)
2572 && REBUILD_THRESHOLD * tab->num_entries > set_get_allocated_entries(tab))
2573 || tab->num_entries < (1 << MINIMAL_POWER2)) {
2575 tab->num_entries = 0;
2576 if (set_has_bins(tab))
2577 set_initialize_bins(tab);
2578 set_rebuild_table_with(tab, tab);
2585 new_tab = set_init_table_with_size(NULL, tab->type,
2586 2 * tab->num_entries - 1);
2587 set_rebuild_table_with(new_tab, tab);
2588 set_rebuild_move_table(new_tab, tab);
2590 set_rebuild_cleanup(tab);
2597 unsigned int size_ind;
2598 set_table_entry *new_entries;
2599 set_table_entry *curr_entry_ptr;
2603 new_entries = new_tab->
entries;
2606 bins = set_bins_ptr(new_tab);
2607 size_ind = set_get_size_ind(new_tab);
2608 st_index_t bound = tab->entries_bound;
2609 set_table_entry *entries = tab->
entries;
2611 for (i = tab->entries_start; i < bound; i++) {
2612 curr_entry_ptr = &entries[i];
2613 PREFETCH(entries + i + 1, 0);
2614 if (EXPECT(DELETED_ENTRY_P(curr_entry_ptr), 0))
2616 if (&new_entries[ni] != curr_entry_ptr)
2617 new_entries[ni] = *curr_entry_ptr;
2618 if (EXPECT(bins != NULL, 1)) {
2619 bin_ind = set_find_table_bin_ind_direct(new_tab, curr_entry_ptr->hash,
2620 curr_entry_ptr->key);
2621 set_bin(bins, size_ind, bin_ind, ni + ENTRY_BASE);
2623 new_tab->num_entries++;
2627 assert(new_tab->num_entries == tab->num_entries);
2633 tab->entry_power = new_tab->entry_power;
2634 tab->bin_power = new_tab->bin_power;
2635 tab->size_ind = new_tab->size_ind;
2642set_rebuild_cleanup(
set_table *
const tab)
2644 tab->entries_start = 0;
2645 tab->entries_bound = tab->num_entries;
2646 tab->rebuilds_num++;
2661static inline st_index_t
2662set_secondary_hash(st_index_t ind,
set_table *tab, st_index_t *perturb)
2665 ind = (ind << 2) + ind + *perturb + 1;
2666 return set_hash_bin(ind, tab);
2673static inline st_index_t
2674set_find_entry(
set_table *tab, st_hash_t hash_value, st_data_t key)
2676 int eq_p, rebuilt_p;
2677 st_index_t i, bound;
2678 set_table_entry *entries;
2680 bound = tab->entries_bound;
2682 for (i = tab->entries_start; i < bound; i++) {
2683 DO_PTR_EQUAL_CHECK(tab, &entries[i], hash_value, key, eq_p, rebuilt_p);
2684 if (EXPECT(rebuilt_p, 0))
2685 return REBUILT_TABLE_ENTRY_IND;
2689 return UNDEFINED_ENTRY_IND;
2701set_find_table_entry_ind(
set_table *tab, st_hash_t hash_value, st_data_t key)
2703 int eq_p, rebuilt_p;
2705#ifdef QUADRATIC_PROBE
2711 set_table_entry *entries = tab->
entries;
2713 ind = set_hash_bin(hash_value, tab);
2714#ifdef QUADRATIC_PROBE
2717 perturb = hash_value;
2720 bin = get_bin(set_bins_ptr(tab), set_get_size_ind(tab), ind);
2721 if (! EMPTY_OR_DELETED_BIN_P(bin)) {
2722 DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
2723 if (EXPECT(rebuilt_p, 0))
2724 return REBUILT_TABLE_ENTRY_IND;
2728 else if (EMPTY_BIN_P(bin))
2729 return UNDEFINED_ENTRY_IND;
2730#ifdef QUADRATIC_PROBE
2731 ind = set_hash_bin(ind + d, tab);
2734 ind = set_secondary_hash(ind, tab, &perturb);
2745set_find_table_bin_ind(
set_table *tab, st_hash_t hash_value, st_data_t key)
2747 int eq_p, rebuilt_p;
2749#ifdef QUADRATIC_PROBE
2755 set_table_entry *entries = tab->
entries;
2757 ind = set_hash_bin(hash_value, tab);
2758#ifdef QUADRATIC_PROBE
2761 perturb = hash_value;
2764 bin = get_bin(set_bins_ptr(tab), set_get_size_ind(tab), ind);
2765 if (! EMPTY_OR_DELETED_BIN_P(bin)) {
2766 DO_PTR_EQUAL_CHECK(tab, &entries[bin - ENTRY_BASE], hash_value, key, eq_p, rebuilt_p);
2767 if (EXPECT(rebuilt_p, 0))
2768 return REBUILT_TABLE_BIN_IND;
2772 else if (EMPTY_BIN_P(bin))
2773 return UNDEFINED_BIN_IND;
2774#ifdef QUADRATIC_PROBE
2775 ind = set_hash_bin(ind + d, tab);
2778 ind = set_secondary_hash(ind, tab, &perturb);
2788set_find_table_bin_ind_direct(
set_table *tab, st_hash_t hash_value, st_data_t key)
2791#ifdef QUADRATIC_PROBE
2798 ind = set_hash_bin(hash_value, tab);
2799#ifdef QUADRATIC_PROBE
2802 perturb = hash_value;
2805 bin = get_bin(set_bins_ptr(tab), set_get_size_ind(tab), ind);
2806 if (EMPTY_OR_DELETED_BIN_P(bin))
2808#ifdef QUADRATIC_PROBE
2809 ind = set_hash_bin(ind + d, tab);
2812 ind = set_secondary_hash(ind, tab, &perturb);
2819#define MARK_SET_BIN_EMPTY(tab, i) (set_bin(set_bins_ptr(tab), set_get_size_ind(tab), i, EMPTY_BIN))
2831set_find_table_bin_ptr_and_reserve(
set_table *tab, st_hash_t *hash_value,
2832 st_data_t key, st_index_t *bin_ind)
2834 int eq_p, rebuilt_p;
2836 st_hash_t curr_hash_value = *hash_value;
2837#ifdef QUADRATIC_PROBE
2842 st_index_t entry_index;
2843 st_index_t firset_deleted_bin_ind;
2844 set_table_entry *entries;
2846 ind = set_hash_bin(curr_hash_value, tab);
2847#ifdef QUADRATIC_PROBE
2850 perturb = curr_hash_value;
2852 firset_deleted_bin_ind = UNDEFINED_BIN_IND;
2855 entry_index = get_bin(set_bins_ptr(tab), set_get_size_ind(tab), ind);
2856 if (EMPTY_BIN_P(entry_index)) {
2858 entry_index = UNDEFINED_ENTRY_IND;
2859 if (firset_deleted_bin_ind != UNDEFINED_BIN_IND) {
2861 ind = firset_deleted_bin_ind;
2862 MARK_SET_BIN_EMPTY(tab, ind);
2866 else if (! DELETED_BIN_P(entry_index)) {
2867 DO_PTR_EQUAL_CHECK(tab, &entries[entry_index - ENTRY_BASE], curr_hash_value, key, eq_p, rebuilt_p);
2868 if (EXPECT(rebuilt_p, 0))
2869 return REBUILT_TABLE_ENTRY_IND;
2873 else if (firset_deleted_bin_ind == UNDEFINED_BIN_IND)
2874 firset_deleted_bin_ind = ind;
2875#ifdef QUADRATIC_PROBE
2876 ind = set_hash_bin(ind + d, tab);
2879 ind = set_secondary_hash(ind, tab, &perturb);
2889set_table_lookup(
set_table *tab, st_data_t key)
2892 st_hash_t hash = set_do_hash(key, tab);
2895 if (!set_has_bins(tab)) {
2896 bin = set_find_entry(tab, hash, key);
2897 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
2899 if (bin == UNDEFINED_ENTRY_IND)
2903 bin = set_find_table_entry_ind(tab, hash, key);
2904 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
2906 if (bin == UNDEFINED_ENTRY_IND)
2915set_rebuild_table_if_necessary (
set_table *tab)
2917 st_index_t bound = tab->entries_bound;
2919 if (bound == set_get_allocated_entries(tab))
2920 set_rebuild_table(tab);
2927set_insert(
set_table *tab, st_data_t key)
2929 set_table_entry *entry;
2932 st_hash_t hash_value;
2936 hash_value = set_do_hash(key, tab);
2938 set_rebuild_table_if_necessary(tab);
2939 if (!set_has_bins(tab)) {
2940 bin = set_find_entry(tab, hash_value, key);
2941 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
2943 new_p = bin == UNDEFINED_ENTRY_IND;
2946 bin_ind = UNDEFINED_BIN_IND;
2949 bin = set_find_table_bin_ptr_and_reserve(tab, &hash_value,
2951 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
2953 new_p = bin == UNDEFINED_ENTRY_IND;
2957 ind = tab->entries_bound++;
2959 entry->hash = hash_value;
2961 if (bin_ind != UNDEFINED_BIN_IND)
2962 set_bin(set_bins_ptr(tab), set_get_size_ind(tab), bin_ind, ind + ENTRY_BASE);
2972 *new_tab = *old_tab;
2973 size_t memsize = set_allocated_entries_size(old_tab) + set_bins_size(old_tab);
2974 new_tab->
entries = (set_table_entry *)malloc(memsize);
2985 if (set_replace(new_tab, old_tab) == NULL) {
2986 set_free_table(new_tab);
2996set_update_range_for_deleted(
set_table *tab, st_index_t n)
3000 if (tab->entries_start == n) {
3001 st_index_t start = n + 1;
3002 st_index_t bound = tab->entries_bound;
3003 set_table_entry *entries = tab->
entries;
3004 while (start < bound && DELETED_ENTRY_P(&entries[start])) start++;
3005 tab->entries_start = start;
3012#define MARK_SET_BIN_DELETED(tab, i) \
3014 set_bin(set_bins_ptr(tab), set_get_size_ind(tab), i, DELETED_BIN); \
3020set_table_delete(
set_table *tab, st_data_t *key)
3022 set_table_entry *entry;
3027 hash = set_do_hash(*key, tab);
3029 if (!set_has_bins(tab)) {
3030 bin = set_find_entry(tab, hash, *key);
3031 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
3033 if (bin == UNDEFINED_ENTRY_IND) {
3038 bin_ind = set_find_table_bin_ind(tab, hash, *key);
3039 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
3041 if (bin_ind == UNDEFINED_BIN_IND) {
3044 bin = get_bin(set_bins_ptr(tab), set_get_size_ind(tab), bin_ind) - ENTRY_BASE;
3045 MARK_SET_BIN_DELETED(tab, bin_ind);
3049 MARK_ENTRY_DELETED(entry);
3051 set_update_range_for_deleted(tab, bin);
3064set_general_foreach(
set_table *tab, set_foreach_check_callback_func *func,
3065 set_update_callback_func *replace, st_data_t arg,
3070 set_table_entry *entries, *curr_entry_ptr;
3071 enum st_retval retval;
3072 st_index_t i, rebuilds_num;
3075 int error_p, packed_p = !set_has_bins(tab);
3080 for (i = tab->entries_start; i < tab->entries_bound; i++) {
3081 curr_entry_ptr = &entries[i];
3082 if (EXPECT(DELETED_ENTRY_P(curr_entry_ptr), 0))
3084 key = curr_entry_ptr->key;
3085 rebuilds_num = tab->rebuilds_num;
3086 hash = curr_entry_ptr->hash;
3087 retval = (*func)(key, arg, 0);
3089 if (retval == ST_REPLACE && replace) {
3090 retval = (*replace)(&key, arg, TRUE);
3091 curr_entry_ptr->key = key;
3094 if (rebuilds_num != tab->rebuilds_num) {
3097 packed_p = !set_has_bins(tab);
3099 i = set_find_entry(tab, hash, key);
3100 if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
3102 error_p = i == UNDEFINED_ENTRY_IND;
3105 i = set_find_table_entry_ind(tab, hash, key);
3106 if (EXPECT(i == REBUILT_TABLE_ENTRY_IND, 0))
3108 error_p = i == UNDEFINED_ENTRY_IND;
3111 if (error_p && check_p) {
3113 retval = (*func)(0, arg, 1);
3116 curr_entry_ptr = &entries[i];
3129 st_data_t key = curr_entry_ptr->key;
3133 bin = set_find_entry(tab, hash, key);
3134 if (EXPECT(bin == REBUILT_TABLE_ENTRY_IND, 0))
3136 if (bin == UNDEFINED_ENTRY_IND)
3140 bin_ind = set_find_table_bin_ind(tab, hash, key);
3141 if (EXPECT(bin_ind == REBUILT_TABLE_BIN_IND, 0))
3143 if (bin_ind == UNDEFINED_BIN_IND)
3145 bin = get_bin(set_bins_ptr(tab), set_get_size_ind(tab), bin_ind) - ENTRY_BASE;
3146 MARK_SET_BIN_DELETED(tab, bin_ind);
3148 curr_entry_ptr = &entries[bin];
3149 MARK_ENTRY_DELETED(curr_entry_ptr);
3151 set_update_range_for_deleted(tab, bin);
3160set_foreach_with_replace(
set_table *tab, set_foreach_check_callback_func *func, set_update_callback_func *replace, st_data_t arg)
3162 return set_general_foreach(tab, func, replace, arg, TRUE);
3166 set_foreach_callback_func *func;
3171set_apply_functor(st_data_t k, st_data_t d,
int _)
3173 const struct set_functor *f = (
void *)d;
3174 return f->func(k, f->arg);
3178set_table_foreach(
set_table *tab, set_foreach_callback_func *func, st_data_t arg)
3180 const struct set_functor f = { func, arg };
3181 return set_general_foreach(tab, set_apply_functor, NULL, (st_data_t)&f, FALSE);
3186set_foreach_check(
set_table *tab, set_foreach_check_callback_func *func, st_data_t arg,
3187 st_data_t never ATTRIBUTE_UNUSED)
3189 return set_general_foreach(tab, func, NULL, arg, TRUE);
3195set_keys(
set_table *tab, st_data_t *keys, st_index_t size)
3197 st_index_t i, bound;
3198 st_data_t key, *keys_start, *keys_end;
3199 set_table_entry *curr_entry_ptr, *entries = tab->
entries;
3201 bound = tab->entries_bound;
3203 keys_end = keys + size;
3204 for (i = tab->entries_start; i < bound; i++) {
3205 if (keys == keys_end)
3207 curr_entry_ptr = &entries[i];
3208 key = curr_entry_ptr->key;
3209 if (! DELETED_ENTRY_P(curr_entry_ptr))
3213 return keys - keys_start;
3219 st_index_t num = tab->num_entries;
3220 if (REBUILD_THRESHOLD * num <= set_get_allocated_entries(tab)) {
3222 set_table *new_tab = set_init_table_with_size(NULL, tab->type, 2 * num);
3223 set_rebuild_table_with(new_tab, tab);
3224 set_rebuild_move_table(new_tab, tab);
3225 set_rebuild_cleanup(tab);
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
#define Qundef
Old name of RUBY_Qundef.
VALUE rb_eRuntimeError
RuntimeError exception.
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
VALUE rb_cString
String class.
#define RB_OBJ_WRITTEN(old, oldv, young)
Identical to RB_OBJ_WRITE(), except it doesn't write any values, but only a WB declaration.
char * ptr
Pointer to the underlying memory region, of at least capa bytes.
int len
Length of the buffer.
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
VALUE type(ANYARGS)
ANYARGS-ed function type.
#define _(args)
This was a transition path from K&R to ANSI.
set_table_entry * entries
Array of size 2^entry_power.
uintptr_t VALUE
Type that represents a Ruby object.