1 // SPDX-License-Identifier: LGPL-2.1-or-later
3 * Copyright (C) 2015-2021 Paul Cercueil <paul@crapouillou.net>
6 #include "blockcache.h"
8 #include "lightrec-private.h"
9 #include "memmanager.h"
11 #include "recompiler.h"
17 /* Must be power of two */
18 #define LUT_SIZE 0x4000
21 struct lightrec_state *state;
22 struct block * lut[LUT_SIZE];
25 u16 lightrec_get_lut_entry(const struct block *block)
27 return (kunseg(block->pc) >> 2) & (LUT_SIZE - 1);
30 struct block * lightrec_find_block(struct blockcache *cache, u32 pc)
36 for (block = cache->lut[(pc >> 2) & (LUT_SIZE - 1)];
37 block; block = block->next)
38 if (kunseg(block->pc) == pc)
44 struct block * lightrec_find_block_from_lut(struct blockcache *cache,
45 u16 lut_entry, u32 addr_in_block)
50 addr_in_block = kunseg(addr_in_block);
52 for (block = cache->lut[lut_entry]; block; block = block->next) {
53 pc = kunseg(block->pc);
54 if (addr_in_block >= pc &&
55 addr_in_block < pc + (block->nb_ops << 2))
62 void remove_from_code_lut(struct blockcache *cache, struct block *block)
64 struct lightrec_state *state = cache->state;
65 u32 offset = lut_offset(block->pc);
67 if (block->function) {
68 memset(lut_address(state, offset), 0,
69 block->nb_ops * lut_elm_size(state));
73 void lightrec_register_block(struct blockcache *cache, struct block *block)
75 u32 pc = kunseg(block->pc);
78 old = cache->lut[(pc >> 2) & (LUT_SIZE - 1)];
82 cache->lut[(pc >> 2) & (LUT_SIZE - 1)] = block;
84 remove_from_code_lut(cache, block);
87 void lightrec_unregister_block(struct blockcache *cache, struct block *block)
89 u32 pc = kunseg(block->pc);
90 struct block *old = cache->lut[(pc >> 2) & (LUT_SIZE - 1)];
93 cache->lut[(pc >> 2) & (LUT_SIZE - 1)] = old->next;
97 for (; old; old = old->next) {
98 if (old->next == block) {
99 old->next = block->next;
104 pr_err("Block at "PC_FMT" is not in cache\n", block->pc);
107 static bool lightrec_block_is_old(const struct lightrec_state *state,
108 const struct block *block)
110 u32 diff = state->current_cycle - block->precompile_date;
112 return diff > (1 << 27); /* About 4 seconds */
115 static void lightrec_free_blocks(struct blockcache *cache,
116 const struct block *except, bool all)
118 struct lightrec_state *state = cache->state;
119 struct block *block, *next;
124 for (i = 0; i < LUT_SIZE; i++) {
125 for (block = cache->lut[i]; block; block = next) {
128 if (except && block == except)
132 outdated = lightrec_block_is_old(state, block) ||
133 lightrec_block_is_outdated(state, block);
139 old_flags = block_set_flags(block, BLOCK_IS_DEAD);
141 if (!(old_flags & BLOCK_IS_DEAD)) {
142 if (ENABLE_THREADED_COMPILER)
143 lightrec_recompiler_remove(state->rec, block);
145 pr_debug("Freeing outdated block at "PC_FMT"\n", block->pc);
146 remove_from_code_lut(cache, block);
147 lightrec_unregister_block(cache, block);
148 lightrec_free_block(state, block);
154 void lightrec_remove_outdated_blocks(struct blockcache *cache,
155 const struct block *except)
157 pr_info("Running out of code space. Cleaning block cache...\n");
159 lightrec_free_blocks(cache, except, false);
162 void lightrec_free_all_blocks(struct blockcache *cache)
164 lightrec_free_blocks(cache, NULL, true);
167 void lightrec_free_block_cache(struct blockcache *cache)
169 lightrec_free_all_blocks(cache);
170 lightrec_free(cache->state, MEM_FOR_LIGHTREC, sizeof(*cache), cache);
173 struct blockcache * lightrec_blockcache_init(struct lightrec_state *state)
175 struct blockcache *cache;
177 cache = lightrec_calloc(state, MEM_FOR_LIGHTREC, sizeof(*cache));
181 cache->state = state;
186 u32 lightrec_calculate_block_hash(const struct block *block)
188 const u32 *code = block->code;
189 u32 hash = 0xffffffff;
192 /* Jenkins one-at-a-time hash algorithm */
193 for (i = 0; i < block->nb_ops; i++) {
195 hash += (hash << 10);
200 hash ^= (hash >> 11);
201 hash += (hash << 15);
206 static void lightrec_reset_lut_offset(struct lightrec_state *state, void *d)
208 u32 pc = (u32)(uintptr_t) d;
212 block = lightrec_find_block(state->block_cache, pc);
216 if (block_has_flag(block, BLOCK_IS_DEAD))
219 addr = block->function ?: state->get_next_block;
220 lut_write(state, lut_offset(pc), addr);
223 bool lightrec_block_is_outdated(struct lightrec_state *state, struct block *block)
225 u32 offset = lut_offset(block->pc);
228 if (lut_read(state, offset))
231 outdated = block->hash != lightrec_calculate_block_hash(block);
232 if (likely(!outdated)) {
233 /* The block was marked as outdated, but the content is still
236 if (ENABLE_THREADED_COMPILER) {
238 * When compiling a block that covers ours, the threaded
239 * compiler will set the LUT entries of the various
240 * entry points. Therefore we cannot write the LUT here,
241 * as we would risk overwriting the new entry points.
242 * Leave it to the reaper to re-install the LUT entries.
245 lightrec_reaper_add(state->reaper,
246 lightrec_reset_lut_offset,
247 (void *)(uintptr_t) block->pc);
248 } else if (block->function) {
249 lut_write(state, offset, block->function);
251 lut_write(state, offset, state->get_next_block);