mirror of
https://git.ffmpeg.org/ffmpeg.git
synced 2026-08-12 13:23:48 +00:00
Shreesh Adiga
915bac7bdc
Adding crc32 specialization for aarch64 which uses both PMULL and crc32 instructions to perform 192 bytes fold in one iteration, performing 9x PMULL and 6 crc32 in one loop iteration, obtaining higher performance for large inputs >8kB. This approach is based on zlib-ng implementation which is also described at https://github.com/corsix/fast-crc32. For smaller buffer size, it was observed to be slightly slower, thus only for input size >8192 this logic is used, for smaller sizes otherwise the 4x PMULL folding method is used along with scalar crc32 instructions for processing the remainder input size. On a MediaTek Dimensity 9400 Android device in termux environment, with normal checkasm seed 0 which picks random buffer size and max buffer size of 16kB, the data observed on Cortex X925, A720 and X4: X925 Before: crc_32_IEEE_LE_c: 12762.0 crc_32_IEEE_LE_crc: 667.5 (19.11x) crc_32_IEEE_LE_pmull_eor3: 346.9 (26.30x) X925 After: crc_32_IEEE_LE_c: 12707.6 crc_32_IEEE_LE_crc: 665.2 (19.10x) crc_32_IEEE_LE_pmull_eor3: 292.8 (41.90x) A720 Before: crc_32_IEEE_LE_c: 23059.1 crc_32_IEEE_LE_crc: 1220.7 (18.89x) crc_32_IEEE_LE_pmull_eor3: 1198.9 (19.23x) A720 After: crc_32_IEEE_LE_c: 23293.3 crc_32_IEEE_LE_crc: 1209.1 (19.26x) crc_32_IEEE_LE_pmull_eor3: 1150.4 (20.24x) X4 Before: crc_32_IEEE_LE_c: 12405.5 crc_32_IEEE_LE_crc: 664.5 (18.67x) crc_32_IEEE_LE_pmull_eor3: 498.1 (24.90x) X4 After: crc_32_IEEE_LE_c: 12457.2 crc_32_IEEE_LE_crc: 665.5 (18.72x) crc_32_IEEE_LE_pmull_eor3: 468.8 (26.57x) So it seems to work well on high performance core like X925, and results in about 20% better performance, while having tiny gains on other cores. Testing for input size of 160 kB after modifying the checkasm crc test to have buffer size increased to 160kB and always using full capacity instead of a random size results in below observations: X925 Before: crc_32_IEEE_LE_c: 210177.1 crc_32_IEEE_LE_crc: 10313.7 (20.35x) crc_32_IEEE_LE_pmull_eor3: 6580.9 (31.83x) X925 After: crc_32_IEEE_LE_c: 210869.3 crc_32_IEEE_LE_crc: 10304.8 (20.36x) crc_32_IEEE_LE_pmull_eor3: 3098.5 (68.05x) A720 Before: crc_32_IEEE_LE_c: 387502.5 crc_32_IEEE_LE_crc: 19196.7 (19.54x) crc_32_IEEE_LE_pmull_eor3: 18717.1 (20.63x) A720 After: crc_32_IEEE_LE_c: 392090.8 crc_32_IEEE_LE_crc: 19795.1 (18.68x) crc_32_IEEE_LE_pmull_eor3: 14971.4 (24.97x) X4 Before: crc_32_IEEE_LE_c: 196232.0 crc_32_IEEE_LE_crc: 10378.7 (18.68x) crc_32_IEEE_LE_pmull_eor3: 7742.0 (25.29x) X4 After: crc_32_IEEE_LE_c: 199632.9 crc_32_IEEE_LE_crc: 10495.8 (18.32x) crc_32_IEEE_LE_pmull_eor3: 5448.9 (24.69x) Seems to result in about 2x gains on X925, 25% on A70 and 40% on X4. In general the performance gains depends on the CPU Core and input size, and this optimization benefits large input size especially on high performance cores like X925 and Apple M series.
234 lines
7.1 KiB
C
234 lines
7.1 KiB
C
/*
|
|
* This file is part of FFmpeg.
|
|
*
|
|
* FFmpeg is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
*
|
|
* FFmpeg is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
* License along with FFmpeg; if not, write to the Free Software
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
*/
|
|
|
|
#ifndef AVUTIL_AARCH64_CRC_H
|
|
#define AVUTIL_AARCH64_CRC_H
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#include "config.h"
|
|
|
|
#include "cpu.h"
|
|
#include "libavutil/attributes_internal.h"
|
|
#include "libavutil/avassert.h"
|
|
#include "libavutil/cpu.h"
|
|
#include "libavutil/crc.h"
|
|
#include "libavutil/intreadwrite.h"
|
|
|
|
#if HAVE_ARM_CRC
|
|
FF_VISIBILITY_PUSH_HIDDEN
|
|
uint32_t ff_crc32_aarch64(const AVCRC *ctx, uint32_t crc, const uint8_t *buffer,
|
|
size_t length);
|
|
FF_VISIBILITY_POP_HIDDEN
|
|
#endif
|
|
|
|
#if HAVE_PMULL && HAVE_EOR3
|
|
#include "libavutil/crc_internal.h"
|
|
|
|
FF_VISIBILITY_PUSH_HIDDEN
|
|
uint32_t ff_crc_neon_pmull(const AVCRC *ctx, uint32_t crc, const uint8_t *buffer,
|
|
size_t length);
|
|
uint32_t ff_crc_le_neon_pmull(const AVCRC *ctx, uint32_t crc, const uint8_t *buffer,
|
|
size_t length);
|
|
FF_VISIBILITY_POP_HIDDEN
|
|
|
|
enum {
|
|
CRC_C = 0,
|
|
PMULL_BE,
|
|
PMULL_LE,
|
|
CRC32_PMULL_LE,
|
|
};
|
|
|
|
static const AVCRC crc_table_pmull[AV_CRC_MAX][17] = {
|
|
[AV_CRC_8_ATM] = {
|
|
PMULL_BE,
|
|
0xbc000000, 0x0, 0x32000000, 0x0,
|
|
0x94000000, 0x0, 0xc4000000, 0x0,
|
|
0x62000000, 0x0, 0x79000000, 0x0,
|
|
0x07156a16, 0x1, 0x07000000, 0x1,
|
|
},
|
|
[AV_CRC_8_EBU] = {
|
|
PMULL_BE,
|
|
0xf3000000, 0x0, 0xb5000000, 0x0,
|
|
0x0d000000, 0x0, 0xfc000000, 0x0,
|
|
0x6a000000, 0x0, 0x65000000, 0x0,
|
|
0x1c4b8192, 0x1, 0x1d000000, 0x1,
|
|
},
|
|
[AV_CRC_16_ANSI] = {
|
|
PMULL_BE,
|
|
0x807d0000, 0x0, 0xf9e30000, 0x0,
|
|
0xff830000, 0x0, 0xf9130000, 0x0,
|
|
0x807b0000, 0x0, 0x86630000, 0x0,
|
|
0xfffbffe7, 0x1, 0x80050000, 0x1,
|
|
},
|
|
[AV_CRC_16_CCITT] = {
|
|
PMULL_BE,
|
|
0x59b00000, 0x0, 0x60190000, 0x0,
|
|
0x45630000, 0x0, 0xd5f60000, 0x0,
|
|
0xaa510000, 0x0, 0xeb230000, 0x0,
|
|
0x11303471, 0x1, 0x10210000, 0x1,
|
|
},
|
|
[AV_CRC_24_IEEE] = {
|
|
PMULL_BE,
|
|
0x467d2400, 0x0, 0x1f428700, 0x0,
|
|
0x64e4d700, 0x0, 0x2c8c9d00, 0x0,
|
|
0xd9fe8c00, 0x0, 0xfd7e0c00, 0x0,
|
|
0xf845fe24, 0x1, 0x864cfb00, 0x1,
|
|
},
|
|
[AV_CRC_32_IEEE] = {
|
|
PMULL_BE,
|
|
0xe6228b11, 0x0, 0x8833794c, 0x0,
|
|
0xe8a45605, 0x0, 0xc5b9cd4c, 0x0,
|
|
0x490d678d, 0x0, 0xf200aa66, 0x0,
|
|
0x04d101df, 0x1, 0x04c11db7, 0x1,
|
|
},
|
|
[AV_CRC_32_IEEE_LE] = {
|
|
PMULL_LE,
|
|
0x54442bd4, 0x1, 0xc6e41596, 0x1,
|
|
0x751997d0, 0x1, 0xccaa009e, 0x0,
|
|
0xccaa009e, 0x0, 0x63cd6124, 0x1,
|
|
0xf7011640, 0x1, 0xdb710641, 0x1,
|
|
},
|
|
[AV_CRC_16_ANSI_LE] = {
|
|
PMULL_LE,
|
|
0x1b0c2, 0x0, 0x0000bffa, 0x0,
|
|
0x1d0c2, 0x0, 0x00018cc2, 0x0,
|
|
0x00018cc2, 0x0, 0x1bc02, 0x0,
|
|
0xcfffbffe, 0x1, 0x14003, 0x0,
|
|
},
|
|
};
|
|
|
|
|
|
static inline void crc_init_aarch64(AVCRC *ctx, int le, int bits, uint32_t poly, int ctx_size)
|
|
{
|
|
uint64_t poly_;
|
|
if (le) {
|
|
// convert the reversed representation to regular form
|
|
poly = reverse(poly, bits) >> 1;
|
|
}
|
|
// convert to 32 degree polynomial
|
|
poly_ = ((uint64_t)poly) << (32 - bits);
|
|
|
|
uint64_t div;
|
|
uint8_t *dst = (uint8_t*)(ctx + 1);
|
|
if (le) {
|
|
ctx[0] = PMULL_LE;
|
|
AV_WN64(dst + 0, xnmodp(4 * 128 + 32, poly_, 32, &div, le));
|
|
AV_WN64(dst + 8, xnmodp(4 * 128 - 32, poly_, 32, &div, le));
|
|
uint64_t tmp = xnmodp(128 - 32, poly_, 32, &div, le);
|
|
AV_WN64(dst + 16, xnmodp(128 + 32, poly_, 32, &div, le));
|
|
AV_WN64(dst + 24, tmp);
|
|
AV_WN64(dst + 32, tmp);
|
|
AV_WN64(dst + 40, xnmodp(64, poly_, 32, &div, le));
|
|
AV_WN64(dst + 48, div);
|
|
AV_WN64(dst + 56, reverse(poly_ | (1ULL << 32), 32));
|
|
} else {
|
|
ctx[0] = PMULL_BE;
|
|
AV_WN64(dst + 0, xnmodp(4 * 128, poly_, 32, &div, le));
|
|
AV_WN64(dst + 8, xnmodp(4 * 128 + 64, poly_, 32, &div, le));
|
|
AV_WN64(dst + 16, xnmodp(128, poly_, 32, &div, le));
|
|
AV_WN64(dst + 24, xnmodp(128 + 64, poly_, 32, &div, le));
|
|
AV_WN64(dst + 32, xnmodp(64, poly_, 32, &div, le));
|
|
AV_WN64(dst + 48, div);
|
|
AV_WN64(dst + 40, xnmodp(96, poly_, 32, &div, le));
|
|
AV_WN64(dst + 56, poly_ | (1ULL << 32));
|
|
}
|
|
}
|
|
|
|
#if HAVE_ARM_CRC
|
|
FF_VISIBILITY_PUSH_HIDDEN
|
|
uint32_t ff_crc32_pmull_eor3_aarch64(const AVCRC *ctx, uint32_t crc, const uint8_t *buffer,
|
|
size_t length);
|
|
FF_VISIBILITY_POP_HIDDEN
|
|
static const AVCRC crc_table_crc32_pmull[] = {
|
|
CRC32_PMULL_LE,
|
|
0x26b70c3d, 0x0, 0x3f41287a, 0x0,
|
|
0xae689191, 0x0, 0xccaa009e, 0x0,
|
|
0xf1da05aa, 0x0, 0x81256527, 0x0,
|
|
0x8f352d95, 0x0, 0x1d9513d7, 0x0,
|
|
0x54442bd4, 0x1, 0xc6e41596, 0x1,
|
|
0x751997d0, 0x1, 0xccaa009e, 0x0,
|
|
0xccaa009e, 0x0, 0x63cd6124, 0x1,
|
|
0xf7011640, 0x1, 0xdb710641, 0x1,
|
|
};
|
|
#endif
|
|
#endif
|
|
|
|
static inline av_cold int ff_crc_init_aarch64(AVCRC *ctx, int le, int bits, uint32_t poly, int ctx_size)
|
|
{
|
|
#if HAVE_PMULL && HAVE_EOR3
|
|
int cpu_flags = av_get_cpu_flags();
|
|
|
|
if (have_pmull(cpu_flags) && have_eor3(cpu_flags)) {
|
|
crc_init_aarch64(ctx, le, bits, poly, ctx_size);
|
|
return 1;
|
|
}
|
|
#endif
|
|
return 0;
|
|
}
|
|
|
|
static inline uint32_t ff_crc_aarch64(const AVCRC *ctx, uint32_t crc,
|
|
const uint8_t *buffer, size_t length)
|
|
{
|
|
switch (ctx[0]) {
|
|
#if HAVE_PMULL && HAVE_EOR3
|
|
#if HAVE_ARM_CRC
|
|
case CRC32_PMULL_LE: return ff_crc32_pmull_eor3_aarch64(ctx, crc, buffer, length);
|
|
#endif
|
|
case PMULL_BE: return ff_crc_neon_pmull(ctx, crc, buffer, length);
|
|
case PMULL_LE: return ff_crc_le_neon_pmull(ctx, crc, buffer, length);
|
|
#endif
|
|
#if HAVE_ARM_CRC
|
|
case (AV_CRC_32_IEEE_LE + 1): return ff_crc32_aarch64(ctx, crc, buffer, length);
|
|
#endif
|
|
default: av_unreachable("AARCH64 has PMULL_LE, PMULL_BE, CRC32_PMULL_LE, and AV_CRC_32_IEEE_LE arch-specific CRC code");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static inline const AVCRC *ff_crc_get_table_aarch64(AVCRCId crc_id)
|
|
{
|
|
int cpu_flags = av_get_cpu_flags();
|
|
#if HAVE_PMULL && HAVE_EOR3
|
|
if (have_pmull(cpu_flags) && have_eor3(cpu_flags)) {
|
|
#if HAVE_ARM_CRC
|
|
if (crc_id == AV_CRC_32_IEEE_LE && have_arm_crc(cpu_flags)) {
|
|
return crc_table_crc32_pmull;
|
|
}
|
|
#endif
|
|
return crc_table_pmull[crc_id];
|
|
}
|
|
#endif
|
|
#if HAVE_ARM_CRC
|
|
static const AVCRC crc32_ieee_le_ctx[] = {
|
|
AV_CRC_32_IEEE_LE + 1
|
|
};
|
|
|
|
if (crc_id != AV_CRC_32_IEEE_LE)
|
|
return NULL;
|
|
|
|
if (have_arm_crc(cpu_flags)) {
|
|
return crc32_ieee_le_ctx;
|
|
}
|
|
#endif
|
|
return NULL;
|
|
}
|
|
|
|
#endif /* AVUTIL_AARCH64_CRC_H */
|