Commit | Line | Data |
---|---|---|
98fa08a5 | 1 | // SPDX-License-Identifier: LGPL-2.1-or-later |
d16005f8 | 2 | /* |
98fa08a5 | 3 | * Copyright (C) 2019-2021 Paul Cercueil <paul@crapouillou.net> |
d16005f8 PC |
4 | */ |
5 | ||
d8b04acd | 6 | #include "blockcache.h" |
d16005f8 PC |
7 | #include "debug.h" |
8 | #include "interpreter.h" | |
9 | #include "lightrec-private.h" | |
10 | #include "memmanager.h" | |
d8b04acd | 11 | #include "reaper.h" |
a59e5536 | 12 | #include "slist.h" |
d16005f8 PC |
13 | |
14 | #include <errno.h> | |
15 | #include <stdatomic.h> | |
16 | #include <stdbool.h> | |
17 | #include <stdlib.h> | |
18 | #include <pthread.h> | |
98fa08a5 PC |
19 | #ifdef __linux__ |
20 | #include <unistd.h> | |
21 | #endif | |
d16005f8 PC |
22 | |
23 | struct block_rec { | |
24 | struct block *block; | |
a59e5536 | 25 | struct slist_elm slist; |
a5a6f7b8 | 26 | unsigned int requests; |
98fa08a5 PC |
27 | bool compiling; |
28 | }; | |
29 | ||
30 | struct recompiler_thd { | |
31 | struct lightrec_cstate *cstate; | |
32 | unsigned int tid; | |
33 | pthread_t thd; | |
d16005f8 PC |
34 | }; |
35 | ||
36 | struct recompiler { | |
37 | struct lightrec_state *state; | |
d16005f8 | 38 | pthread_cond_t cond; |
98fa08a5 | 39 | pthread_cond_t cond2; |
d16005f8 | 40 | pthread_mutex_t mutex; |
d8b04acd | 41 | bool stop, must_flush; |
a59e5536 | 42 | struct slist_elm slist; |
98fa08a5 | 43 | |
d8b04acd PC |
44 | pthread_mutex_t alloc_mutex; |
45 | ||
98fa08a5 PC |
46 | unsigned int nb_recs; |
47 | struct recompiler_thd thds[]; | |
d16005f8 PC |
48 | }; |
49 | ||
98fa08a5 PC |
50 | static unsigned int get_processors_count(void) |
51 | { | |
11357fef | 52 | unsigned int nb = 1; |
98fa08a5 PC |
53 | |
54 | #if defined(PTW32_VERSION) | |
55 | nb = pthread_num_processors_np(); | |
56 | #elif defined(__APPLE__) || defined(__FreeBSD__) | |
57 | int count; | |
58 | size_t size = sizeof(count); | |
59 | ||
60 | nb = sysctlbyname("hw.ncpu", &count, &size, NULL, 0) ? 1 : count; | |
11357fef | 61 | #elif defined(_SC_NPROCESSORS_ONLN) |
98fa08a5 PC |
62 | nb = sysconf(_SC_NPROCESSORS_ONLN); |
63 | #endif | |
64 | ||
65 | return nb < 1 ? 1 : nb; | |
66 | } | |
67 | ||
a5a6f7b8 | 68 | static struct block_rec * lightrec_get_best_elm(struct slist_elm *head) |
98fa08a5 | 69 | { |
a5a6f7b8 | 70 | struct block_rec *block_rec, *best = NULL; |
98fa08a5 PC |
71 | struct slist_elm *elm; |
72 | ||
73 | for (elm = slist_first(head); elm; elm = elm->next) { | |
74 | block_rec = container_of(elm, struct block_rec, slist); | |
75 | ||
a5a6f7b8 PC |
76 | if (!block_rec->compiling |
77 | && (!best || block_rec->requests > best->requests)) | |
78 | best = block_rec; | |
98fa08a5 PC |
79 | } |
80 | ||
a5a6f7b8 | 81 | return best; |
98fa08a5 PC |
82 | } |
83 | ||
d8b04acd PC |
84 | static bool lightrec_cancel_block_rec(struct recompiler *rec, |
85 | struct block_rec *block_rec) | |
86 | { | |
87 | if (block_rec->compiling) { | |
88 | /* Block is being recompiled - wait for | |
89 | * completion */ | |
90 | pthread_cond_wait(&rec->cond2, &rec->mutex); | |
91 | ||
92 | /* We can't guarantee the signal was for us. | |
93 | * Since block_rec may have been removed while | |
94 | * we were waiting on the condition, we cannot | |
95 | * check block_rec->compiling again. The best | |
96 | * thing is just to restart the function. */ | |
97 | return false; | |
98 | } | |
99 | ||
100 | /* Block is not yet being processed - remove it from the list */ | |
101 | slist_remove(&rec->slist, &block_rec->slist); | |
102 | lightrec_free(rec->state, MEM_FOR_LIGHTREC, | |
103 | sizeof(*block_rec), block_rec); | |
104 | ||
105 | return true; | |
106 | } | |
107 | ||
108 | static void lightrec_cancel_list(struct recompiler *rec) | |
109 | { | |
110 | struct block_rec *block_rec; | |
ba3814c1 | 111 | struct slist_elm *elm, *head = &rec->slist; |
d8b04acd | 112 | |
ba3814c1 PC |
113 | for (elm = slist_first(head); elm; elm = slist_first(head)) { |
114 | block_rec = container_of(elm, struct block_rec, slist); | |
d8b04acd PC |
115 | lightrec_cancel_block_rec(rec, block_rec); |
116 | } | |
d8b04acd PC |
117 | } |
118 | ||
119 | static void lightrec_flush_code_buffer(struct lightrec_state *state, void *d) | |
120 | { | |
121 | struct recompiler *rec = d; | |
122 | ||
ba3814c1 PC |
123 | lightrec_remove_outdated_blocks(state->block_cache, NULL); |
124 | rec->must_flush = false; | |
d8b04acd PC |
125 | } |
126 | ||
98fa08a5 PC |
127 | static void lightrec_compile_list(struct recompiler *rec, |
128 | struct recompiler_thd *thd) | |
d16005f8 | 129 | { |
a59e5536 | 130 | struct block_rec *block_rec; |
d16005f8 PC |
131 | struct block *block; |
132 | int ret; | |
133 | ||
a5a6f7b8 | 134 | while (!!(block_rec = lightrec_get_best_elm(&rec->slist))) { |
98fa08a5 | 135 | block_rec->compiling = true; |
a59e5536 | 136 | block = block_rec->block; |
d16005f8 PC |
137 | |
138 | pthread_mutex_unlock(&rec->mutex); | |
139 | ||
ba3814c1 | 140 | if (likely(!block_has_flag(block, BLOCK_IS_DEAD))) { |
98fa08a5 | 141 | ret = lightrec_compile_block(thd->cstate, block); |
d8b04acd PC |
142 | if (ret == -ENOMEM) { |
143 | /* Code buffer is full. Request the reaper to | |
144 | * flush it. */ | |
145 | ||
146 | pthread_mutex_lock(&rec->mutex); | |
ba3814c1 PC |
147 | block_rec->compiling = false; |
148 | pthread_cond_broadcast(&rec->cond2); | |
149 | ||
d8b04acd | 150 | if (!rec->must_flush) { |
ba3814c1 PC |
151 | rec->must_flush = true; |
152 | lightrec_cancel_list(rec); | |
153 | ||
d8b04acd PC |
154 | lightrec_reaper_add(rec->state->reaper, |
155 | lightrec_flush_code_buffer, | |
156 | rec); | |
d8b04acd PC |
157 | } |
158 | return; | |
159 | } | |
160 | ||
98fa08a5 | 161 | if (ret) { |
f5ee77ca | 162 | pr_err("Unable to compile block at "PC_FMT": %d\n", |
98fa08a5 PC |
163 | block->pc, ret); |
164 | } | |
d16005f8 PC |
165 | } |
166 | ||
167 | pthread_mutex_lock(&rec->mutex); | |
168 | ||
a5a6f7b8 | 169 | slist_remove(&rec->slist, &block_rec->slist); |
d16005f8 | 170 | lightrec_free(rec->state, MEM_FOR_LIGHTREC, |
a59e5536 | 171 | sizeof(*block_rec), block_rec); |
ba3814c1 | 172 | pthread_cond_broadcast(&rec->cond2); |
d16005f8 | 173 | } |
d16005f8 PC |
174 | } |
175 | ||
176 | static void * lightrec_recompiler_thd(void *d) | |
177 | { | |
98fa08a5 PC |
178 | struct recompiler_thd *thd = d; |
179 | struct recompiler *rec = container_of(thd, struct recompiler, thds[thd->tid]); | |
d16005f8 PC |
180 | |
181 | pthread_mutex_lock(&rec->mutex); | |
182 | ||
ea17432f | 183 | while (!rec->stop) { |
d16005f8 PC |
184 | do { |
185 | pthread_cond_wait(&rec->cond, &rec->mutex); | |
186 | ||
a59e5536 | 187 | if (rec->stop) |
188 | goto out_unlock; | |
d16005f8 | 189 | |
a59e5536 | 190 | } while (slist_empty(&rec->slist)); |
d16005f8 | 191 | |
98fa08a5 | 192 | lightrec_compile_list(rec, thd); |
ea17432f | 193 | } |
a59e5536 | 194 | |
195 | out_unlock: | |
196 | pthread_mutex_unlock(&rec->mutex); | |
197 | return NULL; | |
d16005f8 PC |
198 | } |
199 | ||
200 | struct recompiler *lightrec_recompiler_init(struct lightrec_state *state) | |
201 | { | |
202 | struct recompiler *rec; | |
98fa08a5 | 203 | unsigned int i, nb_recs, nb_cpus; |
d16005f8 PC |
204 | int ret; |
205 | ||
98fa08a5 PC |
206 | nb_cpus = get_processors_count(); |
207 | nb_recs = nb_cpus < 2 ? 1 : nb_cpus - 1; | |
208 | ||
209 | rec = lightrec_malloc(state, MEM_FOR_LIGHTREC, sizeof(*rec) | |
210 | + nb_recs * sizeof(*rec->thds)); | |
d16005f8 PC |
211 | if (!rec) { |
212 | pr_err("Cannot create recompiler: Out of memory\n"); | |
213 | return NULL; | |
214 | } | |
215 | ||
98fa08a5 PC |
216 | for (i = 0; i < nb_recs; i++) { |
217 | rec->thds[i].tid = i; | |
218 | rec->thds[i].cstate = NULL; | |
219 | } | |
220 | ||
221 | for (i = 0; i < nb_recs; i++) { | |
222 | rec->thds[i].cstate = lightrec_create_cstate(state); | |
25851a1e | 223 | if (!rec->thds[i].cstate) { |
98fa08a5 PC |
224 | pr_err("Cannot create recompiler: Out of memory\n"); |
225 | goto err_free_cstates; | |
226 | } | |
227 | } | |
228 | ||
d16005f8 PC |
229 | rec->state = state; |
230 | rec->stop = false; | |
d8b04acd | 231 | rec->must_flush = false; |
98fa08a5 | 232 | rec->nb_recs = nb_recs; |
a59e5536 | 233 | slist_init(&rec->slist); |
d16005f8 PC |
234 | |
235 | ret = pthread_cond_init(&rec->cond, NULL); | |
236 | if (ret) { | |
237 | pr_err("Cannot init cond variable: %d\n", ret); | |
98fa08a5 | 238 | goto err_free_cstates; |
d16005f8 PC |
239 | } |
240 | ||
98fa08a5 | 241 | ret = pthread_cond_init(&rec->cond2, NULL); |
d16005f8 | 242 | if (ret) { |
98fa08a5 | 243 | pr_err("Cannot init cond variable: %d\n", ret); |
d16005f8 PC |
244 | goto err_cnd_destroy; |
245 | } | |
246 | ||
d8b04acd PC |
247 | ret = pthread_mutex_init(&rec->alloc_mutex, NULL); |
248 | if (ret) { | |
249 | pr_err("Cannot init alloc mutex variable: %d\n", ret); | |
250 | goto err_cnd2_destroy; | |
251 | } | |
252 | ||
98fa08a5 | 253 | ret = pthread_mutex_init(&rec->mutex, NULL); |
d16005f8 | 254 | if (ret) { |
98fa08a5 | 255 | pr_err("Cannot init mutex variable: %d\n", ret); |
d8b04acd | 256 | goto err_alloc_mtx_destroy; |
d16005f8 PC |
257 | } |
258 | ||
98fa08a5 PC |
259 | for (i = 0; i < nb_recs; i++) { |
260 | ret = pthread_create(&rec->thds[i].thd, NULL, | |
261 | lightrec_recompiler_thd, &rec->thds[i]); | |
262 | if (ret) { | |
263 | pr_err("Cannot create recompiler thread: %d\n", ret); | |
264 | /* TODO: Handle cleanup properly */ | |
265 | goto err_mtx_destroy; | |
266 | } | |
267 | } | |
268 | ||
269 | pr_info("Threaded recompiler started with %u workers.\n", nb_recs); | |
270 | ||
d16005f8 PC |
271 | return rec; |
272 | ||
273 | err_mtx_destroy: | |
274 | pthread_mutex_destroy(&rec->mutex); | |
d8b04acd PC |
275 | err_alloc_mtx_destroy: |
276 | pthread_mutex_destroy(&rec->alloc_mutex); | |
98fa08a5 PC |
277 | err_cnd2_destroy: |
278 | pthread_cond_destroy(&rec->cond2); | |
d16005f8 PC |
279 | err_cnd_destroy: |
280 | pthread_cond_destroy(&rec->cond); | |
98fa08a5 PC |
281 | err_free_cstates: |
282 | for (i = 0; i < nb_recs; i++) { | |
283 | if (rec->thds[i].cstate) | |
284 | lightrec_free_cstate(rec->thds[i].cstate); | |
285 | } | |
d16005f8 PC |
286 | lightrec_free(state, MEM_FOR_LIGHTREC, sizeof(*rec), rec); |
287 | return NULL; | |
288 | } | |
289 | ||
290 | void lightrec_free_recompiler(struct recompiler *rec) | |
291 | { | |
98fa08a5 PC |
292 | unsigned int i; |
293 | ||
d16005f8 PC |
294 | rec->stop = true; |
295 | ||
296 | /* Stop the thread */ | |
297 | pthread_mutex_lock(&rec->mutex); | |
98fa08a5 | 298 | pthread_cond_broadcast(&rec->cond); |
d8b04acd | 299 | lightrec_cancel_list(rec); |
d16005f8 | 300 | pthread_mutex_unlock(&rec->mutex); |
98fa08a5 PC |
301 | |
302 | for (i = 0; i < rec->nb_recs; i++) | |
303 | pthread_join(rec->thds[i].thd, NULL); | |
304 | ||
305 | for (i = 0; i < rec->nb_recs; i++) | |
306 | lightrec_free_cstate(rec->thds[i].cstate); | |
d16005f8 PC |
307 | |
308 | pthread_mutex_destroy(&rec->mutex); | |
d8b04acd | 309 | pthread_mutex_destroy(&rec->alloc_mutex); |
d16005f8 | 310 | pthread_cond_destroy(&rec->cond); |
98fa08a5 | 311 | pthread_cond_destroy(&rec->cond2); |
d16005f8 PC |
312 | lightrec_free(rec->state, MEM_FOR_LIGHTREC, sizeof(*rec), rec); |
313 | } | |
314 | ||
315 | int lightrec_recompiler_add(struct recompiler *rec, struct block *block) | |
316 | { | |
a5a6f7b8 | 317 | struct slist_elm *elm; |
a59e5536 | 318 | struct block_rec *block_rec; |
a5a6f7b8 | 319 | u32 pc1, pc2; |
a59e5536 | 320 | int ret = 0; |
d16005f8 PC |
321 | |
322 | pthread_mutex_lock(&rec->mutex); | |
323 | ||
d8b04acd PC |
324 | /* If the recompiler must flush the code cache, we can't add the new |
325 | * job. It will be re-added next time the block's address is jumped to | |
326 | * again. */ | |
327 | if (rec->must_flush) | |
328 | goto out_unlock; | |
329 | ||
a59e5536 | 330 | /* If the block is marked as dead, don't compile it, it will be removed |
331 | * as soon as it's safe. */ | |
ba3814c1 | 332 | if (block_has_flag(block, BLOCK_IS_DEAD)) |
a59e5536 | 333 | goto out_unlock; |
334 | ||
a5a6f7b8 | 335 | for (elm = slist_first(&rec->slist); elm; elm = elm->next) { |
a59e5536 | 336 | block_rec = container_of(elm, struct block_rec, slist); |
337 | ||
d16005f8 | 338 | if (block_rec->block == block) { |
a5a6f7b8 PC |
339 | /* The block to compile is already in the queue - |
340 | * increment its counter to increase its priority */ | |
341 | block_rec->requests++; | |
342 | goto out_unlock; | |
343 | } | |
d16005f8 | 344 | |
a5a6f7b8 PC |
345 | pc1 = kunseg(block_rec->block->pc); |
346 | pc2 = kunseg(block->pc); | |
347 | if (pc2 >= pc1 && pc2 < pc1 + block_rec->block->nb_ops * 4) { | |
348 | /* The block we want to compile is already covered by | |
349 | * another one in the queue - increment its counter to | |
350 | * increase its priority */ | |
351 | block_rec->requests++; | |
a59e5536 | 352 | goto out_unlock; |
d16005f8 PC |
353 | } |
354 | } | |
355 | ||
356 | /* By the time this function was called, the block has been recompiled | |
357 | * and ins't in the wait list anymore. Just return here. */ | |
ba3814c1 | 358 | if (block->function && !block_has_flag(block, BLOCK_SHOULD_RECOMPILE)) |
a59e5536 | 359 | goto out_unlock; |
d16005f8 PC |
360 | |
361 | block_rec = lightrec_malloc(rec->state, MEM_FOR_LIGHTREC, | |
362 | sizeof(*block_rec)); | |
363 | if (!block_rec) { | |
a59e5536 | 364 | ret = -ENOMEM; |
365 | goto out_unlock; | |
d16005f8 PC |
366 | } |
367 | ||
f5ee77ca | 368 | pr_debug("Adding block "PC_FMT" to recompiler\n", block->pc); |
d16005f8 PC |
369 | |
370 | block_rec->block = block; | |
98fa08a5 | 371 | block_rec->compiling = false; |
a5a6f7b8 | 372 | block_rec->requests = 1; |
a59e5536 | 373 | |
374 | elm = &rec->slist; | |
375 | ||
a5a6f7b8 | 376 | /* Push the new entry to the front of the queue */ |
a59e5536 | 377 | slist_append(elm, &block_rec->slist); |
d16005f8 PC |
378 | |
379 | /* Signal the thread */ | |
380 | pthread_cond_signal(&rec->cond); | |
d16005f8 | 381 | |
a59e5536 | 382 | out_unlock: |
383 | pthread_mutex_unlock(&rec->mutex); | |
98fa08a5 | 384 | |
a59e5536 | 385 | return ret; |
d16005f8 PC |
386 | } |
387 | ||
388 | void lightrec_recompiler_remove(struct recompiler *rec, struct block *block) | |
389 | { | |
390 | struct block_rec *block_rec; | |
a59e5536 | 391 | struct slist_elm *elm; |
d16005f8 PC |
392 | |
393 | pthread_mutex_lock(&rec->mutex); | |
394 | ||
98fa08a5 PC |
395 | while (true) { |
396 | for (elm = slist_first(&rec->slist); elm; elm = elm->next) { | |
397 | block_rec = container_of(elm, struct block_rec, slist); | |
a59e5536 | 398 | |
d8b04acd PC |
399 | if (block_rec->block == block) { |
400 | if (lightrec_cancel_block_rec(rec, block_rec)) | |
401 | goto out_unlock; | |
98fa08a5 | 402 | |
98fa08a5 | 403 | break; |
d16005f8 | 404 | } |
98fa08a5 | 405 | } |
d16005f8 | 406 | |
98fa08a5 | 407 | if (!elm) |
d16005f8 | 408 | break; |
d16005f8 PC |
409 | } |
410 | ||
98fa08a5 | 411 | out_unlock: |
d16005f8 PC |
412 | pthread_mutex_unlock(&rec->mutex); |
413 | } | |
414 | ||
98fa08a5 PC |
415 | void * lightrec_recompiler_run_first_pass(struct lightrec_state *state, |
416 | struct block *block, u32 *pc) | |
d16005f8 | 417 | { |
ba3814c1 | 418 | u8 old_flags; |
d16005f8 | 419 | |
98fa08a5 PC |
420 | /* There's no point in running the first pass if the block will never |
421 | * be compiled. Let the main loop run the interpreter instead. */ | |
ba3814c1 | 422 | if (block_has_flag(block, BLOCK_NEVER_COMPILE)) |
98fa08a5 PC |
423 | return NULL; |
424 | ||
ba3814c1 PC |
425 | /* The block is marked as dead, and will be removed the next time the |
426 | * reaper is run. In the meantime, the old function can still be | |
427 | * executed. */ | |
428 | if (block_has_flag(block, BLOCK_IS_DEAD)) | |
429 | return block->function; | |
430 | ||
98fa08a5 PC |
431 | /* If the block is already fully tagged, there is no point in running |
432 | * the first pass. Request a recompilation of the block, and maybe the | |
433 | * interpreter will run the block in the meantime. */ | |
ba3814c1 | 434 | if (block_has_flag(block, BLOCK_FULLY_TAGGED)) |
98fa08a5 PC |
435 | lightrec_recompiler_add(state->rec, block); |
436 | ||
d16005f8 | 437 | if (likely(block->function)) { |
ba3814c1 PC |
438 | if (block_has_flag(block, BLOCK_FULLY_TAGGED)) { |
439 | old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST); | |
d16005f8 | 440 | |
ba3814c1 | 441 | if (!(old_flags & BLOCK_NO_OPCODE_LIST)) { |
f5ee77ca | 442 | pr_debug("Block "PC_FMT" is fully tagged" |
d16005f8 PC |
443 | " - free opcode list\n", block->pc); |
444 | ||
445 | /* The block was already compiled but the opcode list | |
446 | * didn't get freed yet - do it now */ | |
ba3814c1 | 447 | lightrec_free_opcode_list(state, block->opcode_list); |
d16005f8 PC |
448 | } |
449 | } | |
450 | ||
451 | return block->function; | |
452 | } | |
453 | ||
454 | /* Mark the opcode list as freed, so that the threaded compiler won't | |
455 | * free it while we're using it in the interpreter. */ | |
ba3814c1 | 456 | old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST); |
d16005f8 PC |
457 | |
458 | /* Block wasn't compiled yet - run the interpreter */ | |
98fa08a5 | 459 | *pc = lightrec_emulate_block(state, block, *pc); |
d16005f8 | 460 | |
ba3814c1 PC |
461 | if (!(old_flags & BLOCK_NO_OPCODE_LIST)) |
462 | block_clear_flags(block, BLOCK_NO_OPCODE_LIST); | |
d16005f8 PC |
463 | |
464 | /* The block got compiled while the interpreter was running. | |
465 | * We can free the opcode list now. */ | |
ba3814c1 PC |
466 | if (block->function && block_has_flag(block, BLOCK_FULLY_TAGGED)) { |
467 | old_flags = block_set_flags(block, BLOCK_NO_OPCODE_LIST); | |
d16005f8 | 468 | |
ba3814c1 | 469 | if (!(old_flags & BLOCK_NO_OPCODE_LIST)) { |
f5ee77ca | 470 | pr_debug("Block "PC_FMT" is fully tagged" |
ba3814c1 PC |
471 | " - free opcode list\n", block->pc); |
472 | ||
473 | lightrec_free_opcode_list(state, block->opcode_list); | |
474 | } | |
d16005f8 PC |
475 | } |
476 | ||
477 | return NULL; | |
478 | } | |
d8b04acd PC |
479 | |
480 | void lightrec_code_alloc_lock(struct lightrec_state *state) | |
481 | { | |
482 | pthread_mutex_lock(&state->rec->alloc_mutex); | |
483 | } | |
484 | ||
485 | void lightrec_code_alloc_unlock(struct lightrec_state *state) | |
486 | { | |
487 | pthread_mutex_unlock(&state->rec->alloc_mutex); | |
488 | } |