diff options
Diffstat (limited to '')
| -rw-r--r-- | third_party/dav1d/src/thread_task.c | 936 |
1 files changed, 936 insertions, 0 deletions
diff --git a/third_party/dav1d/src/thread_task.c b/third_party/dav1d/src/thread_task.c new file mode 100644 index 0000000000..bfedf6e5bb --- /dev/null +++ b/third_party/dav1d/src/thread_task.c @@ -0,0 +1,936 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include "common/frame.h" + +#include "src/thread_task.h" +#include "src/fg_apply.h" + +// This function resets the cur pointer to the first frame theoretically +// executable after a task completed (ie. each time we update some progress or +// insert some tasks in the queue). +// When frame_idx is set, it can be either from a completed task, or from tasks +// inserted in the queue, in which case we have to make sure the cur pointer +// isn't past this insert. +// The special case where frame_idx is UINT_MAX is to handle the reset after +// completing a task and locklessly signaling progress. In this case we don't +// enter a critical section, which is needed for this function, so we set an +// atomic for a delayed handling, happening here. Meaning we can call this +// function without any actual update other than what's in the atomic, hence +// this special case. +static inline int reset_task_cur(const Dav1dContext *const c, + struct TaskThreadData *const ttd, + unsigned frame_idx) +{ + const unsigned first = atomic_load(&ttd->first); + unsigned reset_frame_idx = atomic_exchange(&ttd->reset_task_cur, UINT_MAX); + if (reset_frame_idx < first) { + if (frame_idx == UINT_MAX) return 0; + reset_frame_idx = UINT_MAX; + } + if (!ttd->cur && c->fc[first].task_thread.task_cur_prev == NULL) + return 0; + if (reset_frame_idx != UINT_MAX) { + if (frame_idx == UINT_MAX) { + if (reset_frame_idx > first + ttd->cur) + return 0; + ttd->cur = reset_frame_idx - first; + goto cur_found; + } + } else if (frame_idx == UINT_MAX) + return 0; + if (frame_idx < first) frame_idx += c->n_fc; + const unsigned min_frame_idx = umin(reset_frame_idx, frame_idx); + const unsigned cur_frame_idx = first + ttd->cur; + if (ttd->cur < c->n_fc && cur_frame_idx < min_frame_idx) + return 0; + for (ttd->cur = min_frame_idx - first; ttd->cur < c->n_fc; ttd->cur++) + if (c->fc[(first + ttd->cur) % c->n_fc].task_thread.task_head) + break; +cur_found: + for (unsigned i = ttd->cur; i < c->n_fc; i++) + c->fc[(first + i) % c->n_fc].task_thread.task_cur_prev = NULL; + return 1; +} + +static inline void reset_task_cur_async(struct TaskThreadData *const ttd, + unsigned frame_idx, unsigned n_frames) +{ + const unsigned first = atomic_load(&ttd->first); + if (frame_idx < first) frame_idx += n_frames; + unsigned last_idx = frame_idx; + do { + frame_idx = last_idx; + last_idx = atomic_exchange(&ttd->reset_task_cur, frame_idx); + } while (last_idx < frame_idx); + if (frame_idx == first && atomic_load(&ttd->first) != first) { + unsigned expected = frame_idx; + atomic_compare_exchange_strong(&ttd->reset_task_cur, &expected, UINT_MAX); + } +} + +static void insert_tasks_between(Dav1dFrameContext *const f, + Dav1dTask *const first, Dav1dTask *const last, + Dav1dTask *const a, Dav1dTask *const b, + const int cond_signal) +{ + struct TaskThreadData *const ttd = f->task_thread.ttd; + if (atomic_load(f->c->flush)) return; + assert(!a || a->next == b); + if (!a) f->task_thread.task_head = first; + else a->next = first; + if (!b) f->task_thread.task_tail = last; + last->next = b; + reset_task_cur(f->c, ttd, first->frame_idx); + if (cond_signal && !atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); +} + +static void insert_tasks(Dav1dFrameContext *const f, + Dav1dTask *const first, Dav1dTask *const last, + const int cond_signal) +{ + // insert task back into task queue + Dav1dTask *t_ptr, *prev_t = NULL; + for (t_ptr = f->task_thread.task_head; + t_ptr; prev_t = t_ptr, t_ptr = t_ptr->next) + { + // entropy coding precedes other steps + if (t_ptr->type == DAV1D_TASK_TYPE_TILE_ENTROPY) { + if (first->type > DAV1D_TASK_TYPE_TILE_ENTROPY) continue; + // both are entropy + if (first->sby > t_ptr->sby) continue; + if (first->sby < t_ptr->sby) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // same sby + } else { + if (first->type == DAV1D_TASK_TYPE_TILE_ENTROPY) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + if (first->sby > t_ptr->sby) continue; + if (first->sby < t_ptr->sby) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // same sby + if (first->type > t_ptr->type) continue; + if (first->type < t_ptr->type) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // same task type + } + + // sort by tile-id + assert(first->type == DAV1D_TASK_TYPE_TILE_RECONSTRUCTION || + first->type == DAV1D_TASK_TYPE_TILE_ENTROPY); + assert(first->type == t_ptr->type); + assert(t_ptr->sby == first->sby); + const int p = first->type == DAV1D_TASK_TYPE_TILE_ENTROPY; + const int t_tile_idx = (int) (first - f->task_thread.tile_tasks[p]); + const int p_tile_idx = (int) (t_ptr - f->task_thread.tile_tasks[p]); + assert(t_tile_idx != p_tile_idx); + if (t_tile_idx > p_tile_idx) continue; + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // append at the end + insert_tasks_between(f, first, last, prev_t, NULL, cond_signal); +} + +static inline void insert_task(Dav1dFrameContext *const f, + Dav1dTask *const t, const int cond_signal) +{ + insert_tasks(f, t, t, cond_signal); +} + +static inline void add_pending(Dav1dFrameContext *const f, Dav1dTask *const t) { + pthread_mutex_lock(&f->task_thread.pending_tasks.lock); + t->next = NULL; + if (!f->task_thread.pending_tasks.head) + f->task_thread.pending_tasks.head = t; + else + f->task_thread.pending_tasks.tail->next = t; + f->task_thread.pending_tasks.tail = t; + atomic_store(&f->task_thread.pending_tasks.merge, 1); + pthread_mutex_unlock(&f->task_thread.pending_tasks.lock); +} + +static inline int merge_pending_frame(Dav1dFrameContext *const f) { + int const merge = atomic_load(&f->task_thread.pending_tasks.merge); + if (merge) { + pthread_mutex_lock(&f->task_thread.pending_tasks.lock); + Dav1dTask *t = f->task_thread.pending_tasks.head; + f->task_thread.pending_tasks.head = NULL; + f->task_thread.pending_tasks.tail = NULL; + atomic_store(&f->task_thread.pending_tasks.merge, 0); + pthread_mutex_unlock(&f->task_thread.pending_tasks.lock); + while (t) { + Dav1dTask *const tmp = t->next; + insert_task(f, t, 0); + t = tmp; + } + } + return merge; +} + +static inline int merge_pending(const Dav1dContext *const c) { + int res = 0; + for (unsigned i = 0; i < c->n_fc; i++) + res |= merge_pending_frame(&c->fc[i]); + return res; +} + +static int create_filter_sbrow(Dav1dFrameContext *const f, + const int pass, Dav1dTask **res_t) +{ + const int has_deblock = f->frame_hdr->loopfilter.level_y[0] || + f->frame_hdr->loopfilter.level_y[1]; + const int has_cdef = f->seq_hdr->cdef; + const int has_resize = f->frame_hdr->width[0] != f->frame_hdr->width[1]; + const int has_lr = f->lf.restore_planes; + + Dav1dTask *tasks = f->task_thread.tasks; + const int uses_2pass = f->c->n_fc > 1; + int num_tasks = f->sbh * (1 + uses_2pass); + if (num_tasks > f->task_thread.num_tasks) { + const size_t size = sizeof(Dav1dTask) * num_tasks; + tasks = realloc(f->task_thread.tasks, size); + if (!tasks) return -1; + memset(tasks, 0, size); + f->task_thread.tasks = tasks; + f->task_thread.num_tasks = num_tasks; + } + tasks += f->sbh * (pass & 1); + + if (pass & 1) { + f->frame_thread.entropy_progress = 0; + } else { + const int prog_sz = ((f->sbh + 31) & ~31) >> 5; + if (prog_sz > f->frame_thread.prog_sz) { + atomic_uint *const prog = realloc(f->frame_thread.frame_progress, + 2 * prog_sz * sizeof(*prog)); + if (!prog) return -1; + f->frame_thread.frame_progress = prog; + f->frame_thread.copy_lpf_progress = prog + prog_sz; + } + f->frame_thread.prog_sz = prog_sz; + memset(f->frame_thread.frame_progress, 0, prog_sz * sizeof(atomic_uint)); + memset(f->frame_thread.copy_lpf_progress, 0, prog_sz * sizeof(atomic_uint)); + atomic_store(&f->frame_thread.deblock_progress, 0); + } + f->frame_thread.next_tile_row[pass & 1] = 0; + + Dav1dTask *t = &tasks[0]; + t->sby = 0; + t->recon_progress = 1; + t->deblock_progress = 0; + t->type = pass == 1 ? DAV1D_TASK_TYPE_ENTROPY_PROGRESS : + has_deblock ? DAV1D_TASK_TYPE_DEBLOCK_COLS : + has_cdef || has_lr /* i.e. LR backup */ ? DAV1D_TASK_TYPE_DEBLOCK_ROWS : + has_resize ? DAV1D_TASK_TYPE_SUPER_RESOLUTION : + DAV1D_TASK_TYPE_RECONSTRUCTION_PROGRESS; + t->frame_idx = (int)(f - f->c->fc); + + *res_t = t; + return 0; +} + +int dav1d_task_create_tile_sbrow(Dav1dFrameContext *const f, const int pass, + const int cond_signal) +{ + Dav1dTask *tasks = f->task_thread.tile_tasks[0]; + const int uses_2pass = f->c->n_fc > 1; + const int num_tasks = f->frame_hdr->tiling.cols * f->frame_hdr->tiling.rows; + if (pass < 2) { + int alloc_num_tasks = num_tasks * (1 + uses_2pass); + if (alloc_num_tasks > f->task_thread.num_tile_tasks) { + const size_t size = sizeof(Dav1dTask) * alloc_num_tasks; + tasks = realloc(f->task_thread.tile_tasks[0], size); + if (!tasks) return -1; + memset(tasks, 0, size); + f->task_thread.tile_tasks[0] = tasks; + f->task_thread.num_tile_tasks = alloc_num_tasks; + } + f->task_thread.tile_tasks[1] = tasks + num_tasks; + } + tasks += num_tasks * (pass & 1); + + Dav1dTask *pf_t; + if (create_filter_sbrow(f, pass, &pf_t)) + return -1; + + Dav1dTask *prev_t = NULL; + for (int tile_idx = 0; tile_idx < num_tasks; tile_idx++) { + Dav1dTileState *const ts = &f->ts[tile_idx]; + Dav1dTask *t = &tasks[tile_idx]; + t->sby = ts->tiling.row_start >> f->sb_shift; + if (pf_t && t->sby) { + prev_t->next = pf_t; + prev_t = pf_t; + pf_t = NULL; + } + t->recon_progress = 0; + t->deblock_progress = 0; + t->deps_skip = 0; + t->type = pass != 1 ? DAV1D_TASK_TYPE_TILE_RECONSTRUCTION : + DAV1D_TASK_TYPE_TILE_ENTROPY; + t->frame_idx = (int)(f - f->c->fc); + if (prev_t) prev_t->next = t; + prev_t = t; + } + if (pf_t) { + prev_t->next = pf_t; + prev_t = pf_t; + } + prev_t->next = NULL; + + atomic_store(&f->task_thread.done[pass & 1], 0); + + // XXX in theory this could be done locklessly, at this point they are no + // tasks in the frameQ, so no other runner should be using this lock, but + // we must add both passes at once + pthread_mutex_lock(&f->task_thread.pending_tasks.lock); + assert(f->task_thread.pending_tasks.head == NULL || pass == 2); + if (!f->task_thread.pending_tasks.head) + f->task_thread.pending_tasks.head = &tasks[0]; + else + f->task_thread.pending_tasks.tail->next = &tasks[0]; + f->task_thread.pending_tasks.tail = prev_t; + atomic_store(&f->task_thread.pending_tasks.merge, 1); + pthread_mutex_unlock(&f->task_thread.pending_tasks.lock); + + return 0; +} + +void dav1d_task_frame_init(Dav1dFrameContext *const f) { + const Dav1dContext *const c = f->c; + + atomic_store(&f->task_thread.init_done, 0); + // schedule init task, which will schedule the remaining tasks + Dav1dTask *const t = &f->task_thread.init_task; + t->type = DAV1D_TASK_TYPE_INIT; + t->frame_idx = (int)(f - c->fc); + t->sby = 0; + t->recon_progress = t->deblock_progress = 0; + insert_task(f, t, 1); +} + +void dav1d_task_delayed_fg(Dav1dContext *const c, Dav1dPicture *const out, + const Dav1dPicture *const in) +{ + struct TaskThreadData *const ttd = &c->task_thread; + ttd->delayed_fg.in = in; + ttd->delayed_fg.out = out; + ttd->delayed_fg.type = DAV1D_TASK_TYPE_FG_PREP; + atomic_init(&ttd->delayed_fg.progress[0], 0); + atomic_init(&ttd->delayed_fg.progress[1], 0); + pthread_mutex_lock(&ttd->lock); + ttd->delayed_fg.exec = 1; + pthread_cond_signal(&ttd->cond); + pthread_cond_wait(&ttd->delayed_fg.cond, &ttd->lock); + pthread_mutex_unlock(&ttd->lock); +} + +static inline int ensure_progress(struct TaskThreadData *const ttd, + Dav1dFrameContext *const f, + Dav1dTask *const t, const enum TaskType type, + atomic_int *const state, int *const target) +{ + // deblock_rows (non-LR portion) depends on deblock of previous sbrow, + // so ensure that completed. if not, re-add to task-queue; else, fall-through + int p1 = atomic_load(state); + if (p1 < t->sby) { + t->type = type; + t->recon_progress = t->deblock_progress = 0; + *target = t->sby; + add_pending(f, t); + pthread_mutex_lock(&ttd->lock); + return 1; + } + return 0; +} + +static inline int check_tile(Dav1dTask *const t, Dav1dFrameContext *const f, + const int frame_mt) +{ + const int tp = t->type == DAV1D_TASK_TYPE_TILE_ENTROPY; + const int tile_idx = (int)(t - f->task_thread.tile_tasks[tp]); + Dav1dTileState *const ts = &f->ts[tile_idx]; + const int p1 = atomic_load(&ts->progress[tp]); + if (p1 < t->sby) return 1; + int error = p1 == TILE_ERROR; + error |= atomic_fetch_or(&f->task_thread.error, error); + if (!error && frame_mt && !tp) { + const int p2 = atomic_load(&ts->progress[1]); + if (p2 <= t->sby) return 1; + error = p2 == TILE_ERROR; + error |= atomic_fetch_or(&f->task_thread.error, error); + } + if (!error && frame_mt && !IS_KEY_OR_INTRA(f->frame_hdr)) { + // check reference state + const Dav1dThreadPicture *p = &f->sr_cur; + const int ss_ver = p->p.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const unsigned p_b = (t->sby + 1) << (f->sb_shift + 2); + const int tile_sby = t->sby - (ts->tiling.row_start >> f->sb_shift); + const int (*const lowest_px)[2] = ts->lowest_pixel[tile_sby]; + for (int n = t->deps_skip; n < 7; n++, t->deps_skip++) { + unsigned lowest; + if (tp) { + // if temporal mv refs are disabled, we only need this + // for the primary ref; if segmentation is disabled, we + // don't even need that + lowest = p_b; + } else { + // +8 is postfilter-induced delay + const int y = lowest_px[n][0] == INT_MIN ? INT_MIN : + lowest_px[n][0] + 8; + const int uv = lowest_px[n][1] == INT_MIN ? INT_MIN : + lowest_px[n][1] * (1 << ss_ver) + 8; + const int max = imax(y, uv); + if (max == INT_MIN) continue; + lowest = iclip(max, 1, f->refp[n].p.p.h); + } + const unsigned p3 = atomic_load(&f->refp[n].progress[!tp]); + if (p3 < lowest) return 1; + atomic_fetch_or(&f->task_thread.error, p3 == FRAME_ERROR); + } + } + return 0; +} + +static inline int get_frame_progress(const Dav1dContext *const c, + const Dav1dFrameContext *const f) +{ + unsigned frame_prog = c->n_fc > 1 ? atomic_load(&f->sr_cur.progress[1]) : 0; + if (frame_prog >= FRAME_ERROR) + return f->sbh - 1; + int idx = frame_prog >> (f->sb_shift + 7); + int prog; + do { + atomic_uint *state = &f->frame_thread.frame_progress[idx]; + const unsigned val = ~atomic_load(state); + prog = val ? ctz(val) : 32; + if (prog != 32) break; + prog = 0; + } while (++idx < f->frame_thread.prog_sz); + return ((idx << 5) | prog) - 1; +} + +static inline void abort_frame(Dav1dFrameContext *const f, const int error) { + atomic_store(&f->task_thread.error, error == DAV1D_ERR(EINVAL) ? 1 : -1); + atomic_store(&f->task_thread.task_counter, 0); + atomic_store(&f->task_thread.done[0], 1); + atomic_store(&f->task_thread.done[1], 1); + atomic_store(&f->sr_cur.progress[0], FRAME_ERROR); + atomic_store(&f->sr_cur.progress[1], FRAME_ERROR); + dav1d_decode_frame_exit(f, error); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); +} + +static inline void delayed_fg_task(const Dav1dContext *const c, + struct TaskThreadData *const ttd) +{ + const Dav1dPicture *const in = ttd->delayed_fg.in; + Dav1dPicture *const out = ttd->delayed_fg.out; +#if CONFIG_16BPC + int off; + if (out->p.bpc != 8) + off = (out->p.bpc >> 1) - 4; +#endif + switch (ttd->delayed_fg.type) { + case DAV1D_TASK_TYPE_FG_PREP: + ttd->delayed_fg.exec = 0; + if (atomic_load(&ttd->cond_signaled)) + pthread_cond_signal(&ttd->cond); + pthread_mutex_unlock(&ttd->lock); + switch (out->p.bpc) { +#if CONFIG_8BPC + case 8: + dav1d_prep_grain_8bpc(&c->dsp[0].fg, out, in, + ttd->delayed_fg.scaling_8bpc, + ttd->delayed_fg.grain_lut_8bpc); + break; +#endif +#if CONFIG_16BPC + case 10: + case 12: + dav1d_prep_grain_16bpc(&c->dsp[off].fg, out, in, + ttd->delayed_fg.scaling_16bpc, + ttd->delayed_fg.grain_lut_16bpc); + break; +#endif + default: abort(); + } + ttd->delayed_fg.type = DAV1D_TASK_TYPE_FG_APPLY; + pthread_mutex_lock(&ttd->lock); + ttd->delayed_fg.exec = 1; + // fall-through + case DAV1D_TASK_TYPE_FG_APPLY:; + int row = atomic_fetch_add(&ttd->delayed_fg.progress[0], 1); + pthread_mutex_unlock(&ttd->lock); + int progmax = (out->p.h + 31) >> 5; + fg_apply_loop: + if (row + 1 < progmax) + pthread_cond_signal(&ttd->cond); + else if (row + 1 >= progmax) { + pthread_mutex_lock(&ttd->lock); + ttd->delayed_fg.exec = 0; + if (row >= progmax) goto end_add; + pthread_mutex_unlock(&ttd->lock); + } + switch (out->p.bpc) { +#if CONFIG_8BPC + case 8: + dav1d_apply_grain_row_8bpc(&c->dsp[0].fg, out, in, + ttd->delayed_fg.scaling_8bpc, + ttd->delayed_fg.grain_lut_8bpc, row); + break; +#endif +#if CONFIG_16BPC + case 10: + case 12: + dav1d_apply_grain_row_16bpc(&c->dsp[off].fg, out, in, + ttd->delayed_fg.scaling_16bpc, + ttd->delayed_fg.grain_lut_16bpc, row); + break; +#endif + default: abort(); + } + row = atomic_fetch_add(&ttd->delayed_fg.progress[0], 1); + int done = atomic_fetch_add(&ttd->delayed_fg.progress[1], 1) + 1; + if (row < progmax) goto fg_apply_loop; + pthread_mutex_lock(&ttd->lock); + ttd->delayed_fg.exec = 0; + end_add: + done = atomic_fetch_add(&ttd->delayed_fg.progress[1], 1) + 1; + progmax = atomic_load(&ttd->delayed_fg.progress[0]); + // signal for completion only once the last runner reaches this + if (done < progmax) + break; + pthread_cond_signal(&ttd->delayed_fg.cond); + break; + default: abort(); + } +} + +void *dav1d_worker_task(void *data) { + Dav1dTaskContext *const tc = data; + const Dav1dContext *const c = tc->c; + struct TaskThreadData *const ttd = tc->task_thread.ttd; + + dav1d_set_thread_name("dav1d-worker"); + + pthread_mutex_lock(&ttd->lock); + for (;;) { + if (tc->task_thread.die) break; + if (atomic_load(c->flush)) goto park; + + merge_pending(c); + if (ttd->delayed_fg.exec) { // run delayed film grain first + delayed_fg_task(c, ttd); + continue; + } + Dav1dFrameContext *f; + Dav1dTask *t, *prev_t = NULL; + if (c->n_fc > 1) { // run init tasks second + for (unsigned i = 0; i < c->n_fc; i++) { + const unsigned first = atomic_load(&ttd->first); + f = &c->fc[(first + i) % c->n_fc]; + if (atomic_load(&f->task_thread.init_done)) continue; + t = f->task_thread.task_head; + if (!t) continue; + if (t->type == DAV1D_TASK_TYPE_INIT) goto found; + if (t->type == DAV1D_TASK_TYPE_INIT_CDF) { + // XXX This can be a simple else, if adding tasks of both + // passes at once (in dav1d_task_create_tile_sbrow). + // Adding the tasks to the pending Q can result in a + // thread merging them before setting init_done. + // We will need to set init_done before adding to the + // pending Q, so maybe return the tasks, set init_done, + // and add to pending Q only then. + const int p1 = f->in_cdf.progress ? + atomic_load(f->in_cdf.progress) : 1; + if (p1) { + atomic_fetch_or(&f->task_thread.error, p1 == TILE_ERROR); + goto found; + } + } + } + } + while (ttd->cur < c->n_fc) { // run decoding tasks last + const unsigned first = atomic_load(&ttd->first); + f = &c->fc[(first + ttd->cur) % c->n_fc]; + merge_pending_frame(f); + prev_t = f->task_thread.task_cur_prev; + t = prev_t ? prev_t->next : f->task_thread.task_head; + while (t) { + if (t->type == DAV1D_TASK_TYPE_INIT_CDF) goto next; + else if (t->type == DAV1D_TASK_TYPE_TILE_ENTROPY || + t->type == DAV1D_TASK_TYPE_TILE_RECONSTRUCTION) + { + // if not bottom sbrow of tile, this task will be re-added + // after it's finished + if (!check_tile(t, f, c->n_fc > 1)) + goto found; + } else if (t->recon_progress) { + const int p = t->type == DAV1D_TASK_TYPE_ENTROPY_PROGRESS; + int error = atomic_load(&f->task_thread.error); + assert(!atomic_load(&f->task_thread.done[p]) || error); + const int tile_row_base = f->frame_hdr->tiling.cols * + f->frame_thread.next_tile_row[p]; + if (p) { + atomic_int *const prog = &f->frame_thread.entropy_progress; + const int p1 = atomic_load(prog); + if (p1 < t->sby) goto next; + atomic_fetch_or(&f->task_thread.error, p1 == TILE_ERROR); + } + for (int tc = 0; tc < f->frame_hdr->tiling.cols; tc++) { + Dav1dTileState *const ts = &f->ts[tile_row_base + tc]; + const int p2 = atomic_load(&ts->progress[p]); + if (p2 < t->recon_progress) goto next; + atomic_fetch_or(&f->task_thread.error, p2 == TILE_ERROR); + } + if (t->sby + 1 < f->sbh) { + // add sby+1 to list to replace this one + Dav1dTask *next_t = &t[1]; + *next_t = *t; + next_t->sby++; + const int ntr = f->frame_thread.next_tile_row[p] + 1; + const int start = f->frame_hdr->tiling.row_start_sb[ntr]; + if (next_t->sby == start) + f->frame_thread.next_tile_row[p] = ntr; + next_t->recon_progress = next_t->sby + 1; + insert_task(f, next_t, 0); + } + goto found; + } else if (t->type == DAV1D_TASK_TYPE_CDEF) { + atomic_uint *prog = f->frame_thread.copy_lpf_progress; + const int p1 = atomic_load(&prog[(t->sby - 1) >> 5]); + if (p1 & (1U << ((t->sby - 1) & 31))) + goto found; + } else { + assert(t->deblock_progress); + const int p1 = atomic_load(&f->frame_thread.deblock_progress); + if (p1 >= t->deblock_progress) { + atomic_fetch_or(&f->task_thread.error, p1 == TILE_ERROR); + goto found; + } + } + next: + prev_t = t; + t = t->next; + f->task_thread.task_cur_prev = prev_t; + } + ttd->cur++; + } + if (reset_task_cur(c, ttd, UINT_MAX)) continue; + if (merge_pending(c)) continue; + park: + tc->task_thread.flushed = 1; + pthread_cond_signal(&tc->task_thread.td.cond); + // we want to be woken up next time progress is signaled + atomic_store(&ttd->cond_signaled, 0); + pthread_cond_wait(&ttd->cond, &ttd->lock); + tc->task_thread.flushed = 0; + reset_task_cur(c, ttd, UINT_MAX); + continue; + + found: + // remove t from list + if (prev_t) prev_t->next = t->next; + else f->task_thread.task_head = t->next; + if (!t->next) f->task_thread.task_tail = prev_t; + if (t->type > DAV1D_TASK_TYPE_INIT_CDF && !f->task_thread.task_head) + ttd->cur++; + t->next = NULL; + // we don't need to check cond_signaled here, since we found a task + // after the last signal so we want to re-signal the next waiting thread + // and again won't need to signal after that + atomic_store(&ttd->cond_signaled, 1); + pthread_cond_signal(&ttd->cond); + pthread_mutex_unlock(&ttd->lock); + found_unlocked:; + const int flush = atomic_load(c->flush); + int error = atomic_fetch_or(&f->task_thread.error, flush) | flush; + + // run it + tc->f = f; + int sby = t->sby; + switch (t->type) { + case DAV1D_TASK_TYPE_INIT: { + assert(c->n_fc > 1); + int res = dav1d_decode_frame_init(f); + int p1 = f->in_cdf.progress ? atomic_load(f->in_cdf.progress) : 1; + if (res || p1 == TILE_ERROR) { + pthread_mutex_lock(&ttd->lock); + abort_frame(f, res ? res : DAV1D_ERR(EINVAL)); + reset_task_cur(c, ttd, t->frame_idx); + } else { + t->type = DAV1D_TASK_TYPE_INIT_CDF; + if (p1) goto found_unlocked; + add_pending(f, t); + pthread_mutex_lock(&ttd->lock); + } + continue; + } + case DAV1D_TASK_TYPE_INIT_CDF: { + assert(c->n_fc > 1); + int res = DAV1D_ERR(EINVAL); + if (!atomic_load(&f->task_thread.error)) + res = dav1d_decode_frame_init_cdf(f); + if (f->frame_hdr->refresh_context && !f->task_thread.update_set) { + atomic_store(f->out_cdf.progress, res < 0 ? TILE_ERROR : 1); + } + if (!res) { + assert(c->n_fc > 1); + for (int p = 1; p <= 2; p++) { + const int res = dav1d_task_create_tile_sbrow(f, p, 0); + if (res) { + pthread_mutex_lock(&ttd->lock); + // memory allocation failed + atomic_store(&f->task_thread.done[2 - p], 1); + atomic_store(&f->task_thread.error, -1); + atomic_fetch_sub(&f->task_thread.task_counter, + f->frame_hdr->tiling.cols * + f->frame_hdr->tiling.rows + f->sbh); + atomic_store(&f->sr_cur.progress[p - 1], FRAME_ERROR); + if (p == 2 && atomic_load(&f->task_thread.done[1])) { + assert(!atomic_load(&f->task_thread.task_counter)); + dav1d_decode_frame_exit(f, DAV1D_ERR(ENOMEM)); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + atomic_store(&f->task_thread.init_done, 1); + continue; + } else { + pthread_mutex_unlock(&ttd->lock); + } + } + } + atomic_store(&f->task_thread.init_done, 1); + pthread_mutex_lock(&ttd->lock); + } else { + pthread_mutex_lock(&ttd->lock); + abort_frame(f, res); + reset_task_cur(c, ttd, t->frame_idx); + atomic_store(&f->task_thread.init_done, 1); + } + continue; + } + case DAV1D_TASK_TYPE_TILE_ENTROPY: + case DAV1D_TASK_TYPE_TILE_RECONSTRUCTION: { + const int p = t->type == DAV1D_TASK_TYPE_TILE_ENTROPY; + const int tile_idx = (int)(t - f->task_thread.tile_tasks[p]); + Dav1dTileState *const ts = &f->ts[tile_idx]; + + tc->ts = ts; + tc->by = sby << f->sb_shift; + const int uses_2pass = c->n_fc > 1; + tc->frame_thread.pass = !uses_2pass ? 0 : + 1 + (t->type == DAV1D_TASK_TYPE_TILE_RECONSTRUCTION); + if (!error) error = dav1d_decode_tile_sbrow(tc); + const int progress = error ? TILE_ERROR : 1 + sby; + + // signal progress + atomic_fetch_or(&f->task_thread.error, error); + if (((sby + 1) << f->sb_shift) < ts->tiling.row_end) { + t->sby++; + t->deps_skip = 0; + if (!check_tile(t, f, uses_2pass)) { + atomic_store(&ts->progress[p], progress); + reset_task_cur_async(ttd, t->frame_idx, c->n_fc); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + goto found_unlocked; + } + atomic_store(&ts->progress[p], progress); + add_pending(f, t); + pthread_mutex_lock(&ttd->lock); + } else { + pthread_mutex_lock(&ttd->lock); + atomic_store(&ts->progress[p], progress); + reset_task_cur(c, ttd, t->frame_idx); + error = atomic_load(&f->task_thread.error); + if (f->frame_hdr->refresh_context && + tc->frame_thread.pass <= 1 && f->task_thread.update_set && + f->frame_hdr->tiling.update == tile_idx) + { + if (!error) + dav1d_cdf_thread_update(f->frame_hdr, f->out_cdf.data.cdf, + &f->ts[f->frame_hdr->tiling.update].cdf); + if (c->n_fc > 1) + atomic_store(f->out_cdf.progress, error ? TILE_ERROR : 1); + } + if (atomic_fetch_sub(&f->task_thread.task_counter, 1) - 1 == 0 && + atomic_load(&f->task_thread.done[0]) && + (!uses_2pass || atomic_load(&f->task_thread.done[1]))) + { + error = atomic_load(&f->task_thread.error); + dav1d_decode_frame_exit(f, error == 1 ? DAV1D_ERR(EINVAL) : + error ? DAV1D_ERR(ENOMEM) : 0); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + } + assert(atomic_load(&f->task_thread.task_counter) >= 0); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + } + continue; + } + case DAV1D_TASK_TYPE_DEBLOCK_COLS: + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_deblock_cols(f, sby); + if (ensure_progress(ttd, f, t, DAV1D_TASK_TYPE_DEBLOCK_ROWS, + &f->frame_thread.deblock_progress, + &t->deblock_progress)) continue; + // fall-through + case DAV1D_TASK_TYPE_DEBLOCK_ROWS: + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_deblock_rows(f, sby); + // signal deblock progress + if (f->frame_hdr->loopfilter.level_y[0] || + f->frame_hdr->loopfilter.level_y[1]) + { + error = atomic_load(&f->task_thread.error); + atomic_store(&f->frame_thread.deblock_progress, + error ? TILE_ERROR : sby + 1); + reset_task_cur_async(ttd, t->frame_idx, c->n_fc); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + } else if (f->seq_hdr->cdef || f->lf.restore_planes) { + atomic_fetch_or(&f->frame_thread.copy_lpf_progress[sby >> 5], + 1U << (sby & 31)); + // CDEF needs the top buffer to be saved by lr_copy_lpf of the + // previous sbrow + if (sby) { + int prog = atomic_load(&f->frame_thread.copy_lpf_progress[(sby - 1) >> 5]); + if (~prog & (1U << ((sby - 1) & 31))) { + t->type = DAV1D_TASK_TYPE_CDEF; + t->recon_progress = t->deblock_progress = 0; + add_pending(f, t); + pthread_mutex_lock(&ttd->lock); + continue; + } + } + } + // fall-through + case DAV1D_TASK_TYPE_CDEF: + if (f->seq_hdr->cdef) { + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_cdef(tc, sby); + reset_task_cur_async(ttd, t->frame_idx, c->n_fc); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + } + // fall-through + case DAV1D_TASK_TYPE_SUPER_RESOLUTION: + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_resize(f, sby); + // fall-through + case DAV1D_TASK_TYPE_LOOP_RESTORATION: + if (!atomic_load(&f->task_thread.error) && f->lf.restore_planes) + f->bd_fn.filter_sbrow_lr(f, sby); + // fall-through + case DAV1D_TASK_TYPE_RECONSTRUCTION_PROGRESS: + // dummy to cover for no post-filters + case DAV1D_TASK_TYPE_ENTROPY_PROGRESS: + // dummy to convert tile progress to frame + break; + default: abort(); + } + // if task completed [typically LR], signal picture progress as per below + const int uses_2pass = c->n_fc > 1; + const int sbh = f->sbh; + const int sbsz = f->sb_step * 4; + if (t->type == DAV1D_TASK_TYPE_ENTROPY_PROGRESS) { + error = atomic_load(&f->task_thread.error); + const unsigned y = sby + 1 == sbh ? UINT_MAX : (unsigned)(sby + 1) * sbsz; + assert(c->n_fc > 1); + if (f->sr_cur.p.data[0] /* upon flush, this can be free'ed already */) + atomic_store(&f->sr_cur.progress[0], error ? FRAME_ERROR : y); + atomic_store(&f->frame_thread.entropy_progress, + error ? TILE_ERROR : sby + 1); + if (sby + 1 == sbh) + atomic_store(&f->task_thread.done[1], 1); + pthread_mutex_lock(&ttd->lock); + const int num_tasks = atomic_fetch_sub(&f->task_thread.task_counter, 1) - 1; + if (sby + 1 < sbh && num_tasks) { + reset_task_cur(c, ttd, t->frame_idx); + continue; + } + if (!num_tasks && atomic_load(&f->task_thread.done[0]) && + atomic_load(&f->task_thread.done[1])) + { + error = atomic_load(&f->task_thread.error); + dav1d_decode_frame_exit(f, error == 1 ? DAV1D_ERR(EINVAL) : + error ? DAV1D_ERR(ENOMEM) : 0); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + } + reset_task_cur(c, ttd, t->frame_idx); + continue; + } + // t->type != DAV1D_TASK_TYPE_ENTROPY_PROGRESS + atomic_fetch_or(&f->frame_thread.frame_progress[sby >> 5], + 1U << (sby & 31)); + pthread_mutex_lock(&f->task_thread.lock); + sby = get_frame_progress(c, f); + error = atomic_load(&f->task_thread.error); + const unsigned y = sby + 1 == sbh ? UINT_MAX : (unsigned)(sby + 1) * sbsz; + if (c->n_fc > 1 && f->sr_cur.p.data[0] /* upon flush, this can be free'ed already */) + atomic_store(&f->sr_cur.progress[1], error ? FRAME_ERROR : y); + pthread_mutex_unlock(&f->task_thread.lock); + if (sby + 1 == sbh) + atomic_store(&f->task_thread.done[0], 1); + pthread_mutex_lock(&ttd->lock); + const int num_tasks = atomic_fetch_sub(&f->task_thread.task_counter, 1) - 1; + if (sby + 1 < sbh && num_tasks) { + reset_task_cur(c, ttd, t->frame_idx); + continue; + } + if (!num_tasks && atomic_load(&f->task_thread.done[0]) && + (!uses_2pass || atomic_load(&f->task_thread.done[1]))) + { + error = atomic_load(&f->task_thread.error); + dav1d_decode_frame_exit(f, error == 1 ? DAV1D_ERR(EINVAL) : + error ? DAV1D_ERR(ENOMEM) : 0); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + } + reset_task_cur(c, ttd, t->frame_idx); + } + pthread_mutex_unlock(&ttd->lock); + + return NULL; +} |