diff options
Diffstat (limited to '')
-rw-r--r-- | third_party/aom/av1/encoder/superres_scale.c | 423 |
1 files changed, 423 insertions, 0 deletions
diff --git a/third_party/aom/av1/encoder/superres_scale.c b/third_party/aom/av1/encoder/superres_scale.c new file mode 100644 index 0000000000..3b47909b15 --- /dev/null +++ b/third_party/aom/av1/encoder/superres_scale.c @@ -0,0 +1,423 @@ +/* + * Copyright (c) 2020, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include "av1/encoder/encoder_alloc.h" +#include "av1/encoder/superres_scale.h" +#include "av1/encoder/random.h" + +// Compute the horizontal frequency components' energy in a frame +// by calculuating the 16x4 Horizontal DCT. This is to be used to +// decide the superresolution parameters. +static void analyze_hor_freq(const AV1_COMP *cpi, double *energy) { + uint64_t freq_energy[16] = { 0 }; + const YV12_BUFFER_CONFIG *buf = cpi->source; + const int bd = cpi->td.mb.e_mbd.bd; + const int width = buf->y_crop_width; + const int height = buf->y_crop_height; + DECLARE_ALIGNED(16, int32_t, coeff[16 * 4]); + int n = 0; + memset(freq_energy, 0, sizeof(freq_energy)); + if (buf->flags & YV12_FLAG_HIGHBITDEPTH) { + const int16_t *src16 = (const int16_t *)CONVERT_TO_SHORTPTR(buf->y_buffer); + for (int i = 0; i < height - 4; i += 4) { + for (int j = 0; j < width - 16; j += 16) { + av1_fwd_txfm2d_16x4(src16 + i * buf->y_stride + j, coeff, buf->y_stride, + H_DCT, bd); + for (int k = 1; k < 16; ++k) { + const uint64_t this_energy = + ((int64_t)coeff[k] * coeff[k]) + + ((int64_t)coeff[k + 16] * coeff[k + 16]) + + ((int64_t)coeff[k + 32] * coeff[k + 32]) + + ((int64_t)coeff[k + 48] * coeff[k + 48]); + freq_energy[k] += ROUND_POWER_OF_TWO(this_energy, 2 + 2 * (bd - 8)); + } + n++; + } + } + } else { + assert(bd == 8); + DECLARE_ALIGNED(16, int16_t, src16[16 * 4]); + for (int i = 0; i < height - 4; i += 4) { + for (int j = 0; j < width - 16; j += 16) { + for (int ii = 0; ii < 4; ++ii) + for (int jj = 0; jj < 16; ++jj) + src16[ii * 16 + jj] = + buf->y_buffer[(i + ii) * buf->y_stride + (j + jj)]; + av1_fwd_txfm2d_16x4(src16, coeff, 16, H_DCT, bd); + for (int k = 1; k < 16; ++k) { + const uint64_t this_energy = + ((int64_t)coeff[k] * coeff[k]) + + ((int64_t)coeff[k + 16] * coeff[k + 16]) + + ((int64_t)coeff[k + 32] * coeff[k + 32]) + + ((int64_t)coeff[k + 48] * coeff[k + 48]); + freq_energy[k] += ROUND_POWER_OF_TWO(this_energy, 2); + } + n++; + } + } + } + if (n) { + for (int k = 1; k < 16; ++k) energy[k] = (double)freq_energy[k] / n; + // Convert to cumulative energy + for (int k = 14; k > 0; --k) energy[k] += energy[k + 1]; + } else { + for (int k = 1; k < 16; ++k) energy[k] = 1e+20; + } +} + +static uint8_t calculate_next_resize_scale(const AV1_COMP *cpi) { + // Choose an arbitrary random number + static unsigned int seed = 56789; + const ResizeCfg *resize_cfg = &cpi->oxcf.resize_cfg; + if (is_stat_generation_stage(cpi)) return SCALE_NUMERATOR; + uint8_t new_denom = SCALE_NUMERATOR; + + if (cpi->common.seq_params->reduced_still_picture_hdr) return SCALE_NUMERATOR; + switch (resize_cfg->resize_mode) { + case RESIZE_NONE: new_denom = SCALE_NUMERATOR; break; + case RESIZE_FIXED: + if (cpi->common.current_frame.frame_type == KEY_FRAME) + new_denom = resize_cfg->resize_kf_scale_denominator; + else + new_denom = resize_cfg->resize_scale_denominator; + break; + case RESIZE_RANDOM: new_denom = lcg_rand16(&seed) % 9 + 8; break; + default: assert(0); + } + return new_denom; +} + +int av1_superres_in_recode_allowed(const AV1_COMP *const cpi) { + const AV1EncoderConfig *const oxcf = &cpi->oxcf; + // Empirically found to not be beneficial for image coding. + return oxcf->superres_cfg.superres_mode == AOM_SUPERRES_AUTO && + cpi->sf.hl_sf.superres_auto_search_type != SUPERRES_AUTO_SOLO && + cpi->rc.frames_to_key > 1; +} + +#define SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME_SOLO 0.012 +#define SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME 0.008 +#define SUPERRES_ENERGY_BY_Q2_THRESH_ARFFRAME 0.008 +#define SUPERRES_ENERGY_BY_AC_THRESH 0.2 + +static double get_energy_by_q2_thresh(const GF_GROUP *gf_group, + const RATE_CONTROL *rc, + int gf_frame_index) { + // TODO(now): Return keyframe thresh * factor based on frame type / pyramid + // level. + if (gf_group->update_type[gf_frame_index] == ARF_UPDATE) { + return SUPERRES_ENERGY_BY_Q2_THRESH_ARFFRAME; + } else if (gf_group->update_type[gf_frame_index] == KF_UPDATE) { + if (rc->frames_to_key <= 1) + return SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME_SOLO; + else + return SUPERRES_ENERGY_BY_Q2_THRESH_KEYFRAME; + } else { + assert(0); + } + return 0; +} + +static uint8_t get_superres_denom_from_qindex_energy(int qindex, double *energy, + double threshq, + double threshp) { + const double q = av1_convert_qindex_to_q(qindex, AOM_BITS_8); + const double tq = threshq * q * q; + const double tp = threshp * energy[1]; + const double thresh = AOMMIN(tq, tp); + int k; + for (k = SCALE_NUMERATOR * 2; k > SCALE_NUMERATOR; --k) { + if (energy[k - 1] > thresh) break; + } + return 3 * SCALE_NUMERATOR - k; +} + +static uint8_t get_superres_denom_for_qindex(const AV1_COMP *cpi, int qindex, + int sr_kf, int sr_arf) { + // Use superres for Key-frames and Alt-ref frames only. + const GF_GROUP *gf_group = &cpi->ppi->gf_group; + if (gf_group->update_type[cpi->gf_frame_index] != KF_UPDATE && + gf_group->update_type[cpi->gf_frame_index] != ARF_UPDATE) { + return SCALE_NUMERATOR; + } + if (gf_group->update_type[cpi->gf_frame_index] == KF_UPDATE && !sr_kf) { + return SCALE_NUMERATOR; + } + if (gf_group->update_type[cpi->gf_frame_index] == ARF_UPDATE && !sr_arf) { + return SCALE_NUMERATOR; + } + + double energy[16]; + analyze_hor_freq(cpi, energy); + + const double energy_by_q2_thresh = + get_energy_by_q2_thresh(gf_group, &cpi->rc, cpi->gf_frame_index); + int denom = get_superres_denom_from_qindex_energy( + qindex, energy, energy_by_q2_thresh, SUPERRES_ENERGY_BY_AC_THRESH); + /* + printf("\nenergy = ["); + for (int k = 1; k < 16; ++k) printf("%f, ", energy[k]); + printf("]\n"); + printf("boost = %d\n", + (gf_group->update_type[cpi->gf_frame_index] == KF_UPDATE) + ? cpi->ppi->p_rc.kf_boost + : cpi->rc.gfu_boost); + printf("denom = %d\n", denom); + */ + if (av1_superres_in_recode_allowed(cpi)) { + assert(cpi->superres_mode != AOM_SUPERRES_NONE); + // Force superres to be tried in the recode loop, as full-res is also going + // to be tried anyway. + denom = AOMMAX(denom, SCALE_NUMERATOR + 1); + } + return denom; +} + +static uint8_t calculate_next_superres_scale(AV1_COMP *cpi) { + // Choose an arbitrary random number + static unsigned int seed = 34567; + const AV1EncoderConfig *oxcf = &cpi->oxcf; + const SuperResCfg *const superres_cfg = &oxcf->superres_cfg; + const FrameDimensionCfg *const frm_dim_cfg = &oxcf->frm_dim_cfg; + const RateControlCfg *const rc_cfg = &oxcf->rc_cfg; + + if (is_stat_generation_stage(cpi)) return SCALE_NUMERATOR; + uint8_t new_denom = SCALE_NUMERATOR; + + // Make sure that superres mode of the frame is consistent with the + // sequence-level flag. + assert(IMPLIES(superres_cfg->superres_mode != AOM_SUPERRES_NONE, + cpi->common.seq_params->enable_superres)); + assert(IMPLIES(!cpi->common.seq_params->enable_superres, + superres_cfg->superres_mode == AOM_SUPERRES_NONE)); + // Make sure that superres mode for current encoding is consistent with user + // provided superres mode. + assert(IMPLIES(superres_cfg->superres_mode != AOM_SUPERRES_AUTO, + cpi->superres_mode == superres_cfg->superres_mode)); + + // Note: we must look at the current superres_mode to be tried in 'cpi' here, + // not the user given mode in 'oxcf'. + switch (cpi->superres_mode) { + case AOM_SUPERRES_NONE: new_denom = SCALE_NUMERATOR; break; + case AOM_SUPERRES_FIXED: + if (cpi->common.current_frame.frame_type == KEY_FRAME) + new_denom = superres_cfg->superres_kf_scale_denominator; + else + new_denom = superres_cfg->superres_scale_denominator; + break; + case AOM_SUPERRES_RANDOM: new_denom = lcg_rand16(&seed) % 9 + 8; break; + case AOM_SUPERRES_QTHRESH: { + // Do not use superres when screen content tools are used. + if (cpi->common.features.allow_screen_content_tools) break; + if (rc_cfg->mode == AOM_VBR || rc_cfg->mode == AOM_CQ) + av1_set_target_rate(cpi, frm_dim_cfg->width, frm_dim_cfg->height); + + // Now decide the use of superres based on 'q'. + int bottom_index, top_index; + const int q = av1_rc_pick_q_and_bounds( + cpi, frm_dim_cfg->width, frm_dim_cfg->height, cpi->gf_frame_index, + &bottom_index, &top_index); + + const int qthresh = (frame_is_intra_only(&cpi->common)) + ? superres_cfg->superres_kf_qthresh + : superres_cfg->superres_qthresh; + if (q <= qthresh) { + new_denom = SCALE_NUMERATOR; + } else { + new_denom = get_superres_denom_for_qindex(cpi, q, 1, 1); + } + break; + } + case AOM_SUPERRES_AUTO: { + if (cpi->common.features.allow_screen_content_tools) break; + if (rc_cfg->mode == AOM_VBR || rc_cfg->mode == AOM_CQ) + av1_set_target_rate(cpi, frm_dim_cfg->width, frm_dim_cfg->height); + + // Now decide the use of superres based on 'q'. + int bottom_index, top_index; + const int q = av1_rc_pick_q_and_bounds( + cpi, frm_dim_cfg->width, frm_dim_cfg->height, cpi->gf_frame_index, + &bottom_index, &top_index); + + const SUPERRES_AUTO_SEARCH_TYPE sr_search_type = + cpi->sf.hl_sf.superres_auto_search_type; + const int qthresh = (sr_search_type == SUPERRES_AUTO_SOLO) ? 128 : 0; + if (q <= qthresh) { + new_denom = SCALE_NUMERATOR; // Don't use superres. + } else { + if (sr_search_type == SUPERRES_AUTO_ALL) { + if (cpi->common.current_frame.frame_type == KEY_FRAME) + new_denom = superres_cfg->superres_kf_scale_denominator; + else + new_denom = superres_cfg->superres_scale_denominator; + } else { + new_denom = get_superres_denom_for_qindex(cpi, q, 1, 1); + } + } + break; + } + default: assert(0); + } + return new_denom; +} + +static int dimension_is_ok(int orig_dim, int resized_dim, int denom) { + return (resized_dim * SCALE_NUMERATOR >= orig_dim * denom / 2); +} + +static int dimensions_are_ok(int owidth, int oheight, size_params_type *rsz) { + // Only need to check the width, as scaling is horizontal only. + (void)oheight; + return dimension_is_ok(owidth, rsz->resize_width, rsz->superres_denom); +} + +static int validate_size_scales(RESIZE_MODE resize_mode, + aom_superres_mode superres_mode, int owidth, + int oheight, size_params_type *rsz) { + if (dimensions_are_ok(owidth, oheight, rsz)) { // Nothing to do. + return 1; + } + + // Calculate current resize scale. + int resize_denom = + AOMMAX(DIVIDE_AND_ROUND(owidth * SCALE_NUMERATOR, rsz->resize_width), + DIVIDE_AND_ROUND(oheight * SCALE_NUMERATOR, rsz->resize_height)); + + if (resize_mode != RESIZE_RANDOM && superres_mode == AOM_SUPERRES_RANDOM) { + // Alter superres scale as needed to enforce conformity. + rsz->superres_denom = + (2 * SCALE_NUMERATOR * SCALE_NUMERATOR) / resize_denom; + if (!dimensions_are_ok(owidth, oheight, rsz)) { + if (rsz->superres_denom > SCALE_NUMERATOR) --rsz->superres_denom; + } + } else if (resize_mode == RESIZE_RANDOM && + superres_mode != AOM_SUPERRES_RANDOM) { + // Alter resize scale as needed to enforce conformity. + resize_denom = + (2 * SCALE_NUMERATOR * SCALE_NUMERATOR) / rsz->superres_denom; + rsz->resize_width = owidth; + rsz->resize_height = oheight; + av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height, + resize_denom); + if (!dimensions_are_ok(owidth, oheight, rsz)) { + if (resize_denom > SCALE_NUMERATOR) { + --resize_denom; + rsz->resize_width = owidth; + rsz->resize_height = oheight; + av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height, + resize_denom); + } + } + } else if (resize_mode == RESIZE_RANDOM && + superres_mode == AOM_SUPERRES_RANDOM) { + // Alter both resize and superres scales as needed to enforce conformity. + do { + if (resize_denom > rsz->superres_denom) + --resize_denom; + else + --rsz->superres_denom; + rsz->resize_width = owidth; + rsz->resize_height = oheight; + av1_calculate_scaled_size(&rsz->resize_width, &rsz->resize_height, + resize_denom); + } while (!dimensions_are_ok(owidth, oheight, rsz) && + (resize_denom > SCALE_NUMERATOR || + rsz->superres_denom > SCALE_NUMERATOR)); + } else { // We are allowed to alter neither resize scale nor superres + // scale. + return 0; + } + return dimensions_are_ok(owidth, oheight, rsz); +} + +// Calculates resize and superres params for next frame +static size_params_type calculate_next_size_params(AV1_COMP *cpi) { + const AV1EncoderConfig *oxcf = &cpi->oxcf; + ResizePendingParams *resize_pending_params = &cpi->resize_pending_params; + const FrameDimensionCfg *const frm_dim_cfg = &oxcf->frm_dim_cfg; + size_params_type rsz = { frm_dim_cfg->width, frm_dim_cfg->height, + SCALE_NUMERATOR }; + int resize_denom = SCALE_NUMERATOR; + if (has_no_stats_stage(cpi) && cpi->ppi->use_svc && + (cpi->common.width != cpi->oxcf.frm_dim_cfg.width || + cpi->common.height != cpi->oxcf.frm_dim_cfg.height)) { + rsz.resize_width = cpi->common.width; + rsz.resize_height = cpi->common.height; + return rsz; + } + if (is_stat_generation_stage(cpi)) return rsz; + if (resize_pending_params->width && resize_pending_params->height) { + rsz.resize_width = resize_pending_params->width; + rsz.resize_height = resize_pending_params->height; + resize_pending_params->width = resize_pending_params->height = 0; + if (oxcf->superres_cfg.superres_mode == AOM_SUPERRES_NONE) return rsz; + } else { + resize_denom = calculate_next_resize_scale(cpi); + rsz.resize_width = frm_dim_cfg->width; + rsz.resize_height = frm_dim_cfg->height; + av1_calculate_scaled_size(&rsz.resize_width, &rsz.resize_height, + resize_denom); + } + rsz.superres_denom = calculate_next_superres_scale(cpi); + if (!validate_size_scales(oxcf->resize_cfg.resize_mode, cpi->superres_mode, + frm_dim_cfg->width, frm_dim_cfg->height, &rsz)) + assert(0 && "Invalid scale parameters"); + return rsz; +} + +static void setup_frame_size_from_params(AV1_COMP *cpi, + const size_params_type *rsz) { + int encode_width = rsz->resize_width; + int encode_height = rsz->resize_height; + + AV1_COMMON *cm = &cpi->common; + cm->superres_upscaled_width = encode_width; + cm->superres_upscaled_height = encode_height; + cm->superres_scale_denominator = rsz->superres_denom; + av1_calculate_scaled_superres_size(&encode_width, &encode_height, + rsz->superres_denom); + av1_set_frame_size(cpi, encode_width, encode_height); +} + +void av1_setup_frame_size(AV1_COMP *cpi) { + AV1_COMMON *cm = &cpi->common; + // Reset superres params from previous frame. + cm->superres_scale_denominator = SCALE_NUMERATOR; + const size_params_type rsz = calculate_next_size_params(cpi); + setup_frame_size_from_params(cpi, &rsz); + + assert(av1_is_min_tile_width_satisfied(cm)); +} + +void av1_superres_post_encode(AV1_COMP *cpi) { + AV1_COMMON *cm = &cpi->common; + + assert(cpi->oxcf.superres_cfg.enable_superres); + assert(!is_lossless_requested(&cpi->oxcf.rc_cfg)); + assert(!cm->features.all_lossless); + + av1_superres_upscale(cm, NULL, cpi->image_pyramid_levels); + + // If regular resizing is occurring the source will need to be downscaled to + // match the upscaled superres resolution. Otherwise the original source is + // used. + if (!av1_resize_scaled(cm)) { + cpi->source = cpi->unscaled_source; + if (cpi->last_source != NULL) cpi->last_source = cpi->unscaled_last_source; + } else { + assert(cpi->unscaled_source->y_crop_width != cm->superres_upscaled_width); + assert(cpi->unscaled_source->y_crop_height != cm->superres_upscaled_height); + // Do downscale. cm->(width|height) has been updated by + // av1_superres_upscale + cpi->source = realloc_and_scale_source(cpi, cm->superres_upscaled_width, + cm->superres_upscaled_height); + } +} |