lavc/hashtable: create generic robin hood hash table
Adds a generic hash table with the DXV encoder as an initial use case. Signed-off-by: Emma Worley <emma@emma.gg>
This commit is contained in:
Emma Worley
parent
854b8690a6
commit
6fdb54ddee
@ -42,6 +42,7 @@ OBJS = ac3_parser.o \
|
||||
dv_profile.o \
|
||||
encode.o \
|
||||
get_buffer.o \
|
||||
hashtable.o \
|
||||
imgconvert.o \
|
||||
jni.o \
|
||||
lcevcdec.o \
|
||||
@ -1326,6 +1327,7 @@ TESTPROGS = avcodec \
|
||||
bitstream_le \
|
||||
celp_math \
|
||||
codec_desc \
|
||||
hashtable \
|
||||
htmlsubtitles \
|
||||
jpeg2000dwt \
|
||||
mathops \
|
||||
|
||||
214
libavcodec/hashtable.c
Normal file
214
libavcodec/hashtable.c
Normal file
@ -0,0 +1,214 @@
|
||||
/*
|
||||
* Generic hashtable
|
||||
* Copyright (C) 2025 Emma Worley <emma@emma.gg>
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "libavutil/crc.h"
|
||||
#include "libavutil/error.h"
|
||||
#include "libavutil/mem.h"
|
||||
#include "hashtable.h"
|
||||
|
||||
#define ALIGN _Alignof(size_t)
|
||||
|
||||
struct FFHashtableContext {
|
||||
size_t key_size;
|
||||
size_t key_size_aligned;
|
||||
size_t val_size;
|
||||
size_t val_size_aligned;
|
||||
size_t entry_size;
|
||||
size_t max_entries;
|
||||
size_t nb_entries;
|
||||
const AVCRC *crc;
|
||||
uint8_t *table;
|
||||
uint8_t *swapbuf;
|
||||
};
|
||||
|
||||
/*
|
||||
* Hash table entries are comprised of a probe sequence length (PSL), key, and
|
||||
* value. When the PSL of an entry is zero, it means it is not occupied by a
|
||||
* key/value pair. When the PSL is non-zero, it represents the "distance" of
|
||||
* the entry from its "home" location plus one, where the "home" location is
|
||||
* hash(key) % max_entries.
|
||||
*/
|
||||
|
||||
#define ENTRY_PSL_VAL(entry) (*(size_t*)(entry))
|
||||
#define ENTRY_KEY_PTR(entry) ((entry) + FFALIGN(sizeof(size_t), ALIGN))
|
||||
#define ENTRY_VAL_PTR(entry) (ENTRY_KEY_PTR(entry) + ctx->key_size_aligned)
|
||||
|
||||
#define KEYS_EQUAL(k1, k2) (!memcmp((k1), (k2), ctx->key_size))
|
||||
|
||||
int ff_hashtable_alloc(struct FFHashtableContext **ctx, size_t key_size, size_t val_size, size_t max_entries)
|
||||
{
|
||||
struct FFHashtableContext *res = av_malloc(sizeof(struct FFHashtableContext));
|
||||
if (!res)
|
||||
return AVERROR(ENOMEM);
|
||||
res->key_size = key_size;
|
||||
res->key_size_aligned = FFALIGN(key_size, ALIGN);
|
||||
res->val_size = val_size;
|
||||
res->val_size_aligned = FFALIGN(val_size, ALIGN);
|
||||
res->entry_size = FFALIGN(sizeof(size_t), ALIGN)
|
||||
+ res->key_size_aligned
|
||||
+ res->val_size_aligned;
|
||||
res->max_entries = max_entries;
|
||||
res->nb_entries = 0;
|
||||
res->crc = av_crc_get_table(AV_CRC_32_IEEE);
|
||||
if (!res->crc) {
|
||||
ff_hashtable_freep(&res);
|
||||
return AVERROR_BUG;
|
||||
}
|
||||
res->table = av_calloc(res->max_entries, res->entry_size);
|
||||
if (!res->table) {
|
||||
ff_hashtable_freep(&res);
|
||||
return AVERROR(ENOMEM);
|
||||
}
|
||||
|
||||
res->swapbuf = av_calloc(2, res->key_size_aligned + res->val_size_aligned);
|
||||
if (!res->swapbuf) {
|
||||
ff_hashtable_freep(&res);
|
||||
return AVERROR(ENOMEM);
|
||||
}
|
||||
*ctx = res;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t hash_key(const struct FFHashtableContext *ctx, const void *key)
|
||||
{
|
||||
return av_crc(ctx->crc, 0, key, ctx->key_size) % ctx->max_entries;
|
||||
}
|
||||
|
||||
int ff_hashtable_get(const struct FFHashtableContext *ctx, const void *key, void *val)
|
||||
{
|
||||
if (!ctx->nb_entries)
|
||||
return 0;
|
||||
|
||||
size_t hash = hash_key(ctx, key);
|
||||
|
||||
for (size_t psl = 1; psl <= ctx->max_entries; psl++) {
|
||||
size_t wrapped_index = (hash + psl) % ctx->max_entries;
|
||||
uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
|
||||
if (ENTRY_PSL_VAL(entry) < psl)
|
||||
// When PSL stops increasing it means there are no further entries
|
||||
// with the same key hash.
|
||||
return 0;
|
||||
if (KEYS_EQUAL(ENTRY_KEY_PTR(entry), key)) {
|
||||
memcpy(val, ENTRY_VAL_PTR(entry), ctx->val_size);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ff_hashtable_set(struct FFHashtableContext *ctx, const void *key, const void *val)
|
||||
{
|
||||
int swapping = 0;
|
||||
size_t psl = 1;
|
||||
size_t hash = hash_key(ctx, key);
|
||||
size_t wrapped_index = hash % ctx->max_entries;
|
||||
uint8_t *set = ctx->swapbuf;
|
||||
uint8_t *tmp = ctx->swapbuf + ctx->key_size_aligned + ctx->val_size_aligned;
|
||||
|
||||
memcpy(set, key, ctx->key_size);
|
||||
memcpy(set + ctx->key_size_aligned, val, ctx->val_size);
|
||||
|
||||
for (size_t i = 0; i < ctx->max_entries; i++) {
|
||||
if (++wrapped_index == ctx->max_entries)
|
||||
wrapped_index = 0;
|
||||
uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
|
||||
if (!ENTRY_PSL_VAL(entry) || (!swapping && KEYS_EQUAL(ENTRY_KEY_PTR(entry), set))) {
|
||||
if (!ENTRY_PSL_VAL(entry))
|
||||
ctx->nb_entries++;
|
||||
ENTRY_PSL_VAL(entry) = psl;
|
||||
memcpy(ENTRY_KEY_PTR(entry), set, ctx->key_size_aligned + ctx->val_size);
|
||||
return 1;
|
||||
}
|
||||
if (ENTRY_PSL_VAL(entry) < psl) {
|
||||
// When PSL stops increasing it means there are no further entries
|
||||
// with the same key hash. We can only hope to find an unoccupied
|
||||
// entry.
|
||||
if (ctx->nb_entries == ctx->max_entries)
|
||||
// The table is full so inserts are impossible.
|
||||
return 0;
|
||||
// Robin Hood hash tables "steal from the rich" by minimizing the
|
||||
// PSL of the inserted entry.
|
||||
swapping = 1;
|
||||
// set needs to swap with entry
|
||||
memcpy(tmp, ENTRY_KEY_PTR(entry), ctx->key_size_aligned + ctx->val_size_aligned);
|
||||
memcpy(ENTRY_KEY_PTR(entry), set, ctx->key_size_aligned + ctx->val_size_aligned);
|
||||
FFSWAP(uint8_t*, set, tmp);
|
||||
FFSWAP(size_t, psl, ENTRY_PSL_VAL(entry));
|
||||
}
|
||||
psl++;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ff_hashtable_delete(struct FFHashtableContext *ctx, const void *key)
|
||||
{
|
||||
if (!ctx->nb_entries)
|
||||
return 0;
|
||||
|
||||
uint8_t *next_entry;
|
||||
size_t hash = hash_key(ctx, key);
|
||||
size_t wrapped_index = hash % ctx->max_entries;
|
||||
|
||||
for (size_t psl = 1; psl <= ctx->max_entries; psl++) {
|
||||
if (++wrapped_index == ctx->max_entries)
|
||||
wrapped_index = 0;
|
||||
uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
|
||||
if (ENTRY_PSL_VAL(entry) < psl)
|
||||
// When PSL stops increasing it means there are no further entries
|
||||
// with the same key hash.
|
||||
return 0;
|
||||
if (KEYS_EQUAL(ENTRY_KEY_PTR(entry), key)) {
|
||||
ENTRY_PSL_VAL(entry) = 0;
|
||||
// Shift each following entry that will benefit from a reduced PSL.
|
||||
for (psl++; psl <= ctx->max_entries; psl++) {
|
||||
if (++wrapped_index == ctx->max_entries)
|
||||
wrapped_index = 0;
|
||||
next_entry = ctx->table + wrapped_index * ctx->entry_size;
|
||||
if (ENTRY_PSL_VAL(next_entry) <= 1) {
|
||||
ctx->nb_entries--;
|
||||
return 1;
|
||||
}
|
||||
memcpy(entry, next_entry, ctx->entry_size);
|
||||
ENTRY_PSL_VAL(entry)--;
|
||||
ENTRY_PSL_VAL(next_entry) = 0;
|
||||
entry = next_entry;
|
||||
}
|
||||
}
|
||||
};
|
||||
return 0;
|
||||
}
|
||||
|
||||
void ff_hashtable_clear(struct FFHashtableContext *ctx)
|
||||
{
|
||||
memset(ctx->table, 0, ctx->entry_size * ctx->max_entries);
|
||||
}
|
||||
|
||||
void ff_hashtable_freep(struct FFHashtableContext **ctx)
|
||||
{
|
||||
if (*ctx) {
|
||||
av_freep(&(*ctx)->table);
|
||||
av_freep(&(*ctx)->swapbuf);
|
||||
}
|
||||
av_freep(ctx);
|
||||
}
|
||||
94
libavcodec/hashtable.h
Normal file
94
libavcodec/hashtable.h
Normal file
@ -0,0 +1,94 @@
|
||||
/*
|
||||
* Generic hashtable
|
||||
* Copyright (C) 2024 Emma Worley <emma@emma.gg>
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#ifndef AVCODEC_HASHTABLE_H
|
||||
#define AVCODEC_HASHTABLE_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/* Implements a hash table using Robin Hood open addressing.
|
||||
* See: https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf
|
||||
*
|
||||
* Keys are placed in the table based on their CRC value and are considered
|
||||
* equal when they are bytewise-identical.
|
||||
*/
|
||||
|
||||
typedef struct FFHashtableContext FFHashtableContext;
|
||||
|
||||
/**
|
||||
* Create a fixed-sized Robin Hood hash table.
|
||||
*
|
||||
* @param ctx context to allocate and initialize
|
||||
* @param key_size size of key type in bytes
|
||||
* @param val_size size of value type in bytes
|
||||
* @param max_entries maximum number of key-value pairs to store
|
||||
*
|
||||
* @return zero on success, nonzero on error
|
||||
*/
|
||||
int ff_hashtable_alloc(struct FFHashtableContext **ctx, size_t key_size, size_t val_size, size_t max_entries);
|
||||
|
||||
/**
|
||||
* Look up a value from a hash table given a key.
|
||||
*
|
||||
* @param ctx hash table context
|
||||
* @param key pointer to key data
|
||||
* @param val destination pointer for value data
|
||||
*
|
||||
* @return 1 if the key is found, zero if the key is not found
|
||||
*/
|
||||
int ff_hashtable_get(const struct FFHashtableContext *ctx, const void *key, void *val);
|
||||
|
||||
/**
|
||||
* Store a value in a hash table given a key.
|
||||
*
|
||||
* @param ctx hash table context
|
||||
* @param key pointer to key data
|
||||
* @param val pointer for value data
|
||||
*
|
||||
* @return 1 if the key is written, zero if the key is not written due to the hash table reaching max capacity
|
||||
*/
|
||||
int ff_hashtable_set(struct FFHashtableContext *ctx, const void *key, const void *val);
|
||||
|
||||
/**
|
||||
* Delete a value from a hash table given a key.
|
||||
*
|
||||
* @param ctx hash table context
|
||||
* @param key pointer to key data
|
||||
*
|
||||
* @return 1 if the key is deleted, zero if the key is not deleted due to not being found
|
||||
*/
|
||||
int ff_hashtable_delete(struct FFHashtableContext *ctx, const void *key);
|
||||
|
||||
/**
|
||||
* Delete all values from a hash table.
|
||||
*
|
||||
* @param ctx hash table context
|
||||
*/
|
||||
void ff_hashtable_clear(struct FFHashtableContext *ctx);
|
||||
|
||||
/**
|
||||
* Free a hash table.
|
||||
*
|
||||
* @param ctx hash table context
|
||||
*/
|
||||
void ff_hashtable_freep(struct FFHashtableContext **ctx);
|
||||
|
||||
#endif
|
||||
110
libavcodec/tests/hashtable.c
Normal file
110
libavcodec/tests/hashtable.c
Normal file
@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Generic hashtable tests
|
||||
* Copyright (C) 2024 Emma Worley <emma@emma.gg>
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "libavutil/avassert.h"
|
||||
#include "libavcodec/hashtable.h"
|
||||
|
||||
int main(void)
|
||||
{
|
||||
struct FFHashtableContext *ctx;
|
||||
uint8_t k;
|
||||
uint64_t v;
|
||||
|
||||
// impossibly large allocation should fail gracefully
|
||||
av_assert0(ff_hashtable_alloc(&ctx, -1, -1, -1) < 0);
|
||||
|
||||
// hashtable can store up to 3 uint8_t->uint64_t entries
|
||||
av_assert0(!ff_hashtable_alloc(&ctx, sizeof(k), sizeof(v), 3));
|
||||
|
||||
// unsuccessful deletes return 0
|
||||
k = 1;
|
||||
av_assert0(!ff_hashtable_delete(ctx, &k));
|
||||
|
||||
// unsuccessful gets return 0
|
||||
k = 1;
|
||||
av_assert0(!ff_hashtable_get(ctx, &k, &v));
|
||||
|
||||
// successful sets returns 1
|
||||
k = 1;
|
||||
v = 1;
|
||||
av_assert0(ff_hashtable_set(ctx, &k, &v));
|
||||
|
||||
// get should now contain 1
|
||||
k = 1;
|
||||
v = 0;
|
||||
av_assert0(ff_hashtable_get(ctx, &k, &v));
|
||||
av_assert0(v == 1);
|
||||
|
||||
// updating sets should return 1
|
||||
k = 1;
|
||||
v = 2;
|
||||
av_assert0(ff_hashtable_set(ctx, &k, &v));
|
||||
|
||||
// get should now contain 2
|
||||
k = 1;
|
||||
v = 0;
|
||||
av_assert0(ff_hashtable_get(ctx, &k, &v));
|
||||
av_assert0(v == 2);
|
||||
|
||||
// fill the table
|
||||
k = 2;
|
||||
v = 2;
|
||||
av_assert0(ff_hashtable_set(ctx, &k, &v));
|
||||
k = 3;
|
||||
v = 3;
|
||||
av_assert0(ff_hashtable_set(ctx, &k, &v));
|
||||
|
||||
// inserting sets on a full table should return 0
|
||||
k = 4;
|
||||
v = 4;
|
||||
av_assert0(!ff_hashtable_set(ctx, &k, &v));
|
||||
|
||||
// updating sets on a full table should return 1
|
||||
k = 1;
|
||||
v = 4;
|
||||
av_assert0(ff_hashtable_set(ctx, &k, &v));
|
||||
v = 0;
|
||||
av_assert0(ff_hashtable_get(ctx, &k, &v));
|
||||
av_assert0(v == 4);
|
||||
|
||||
// successful deletes should return 1
|
||||
k = 1;
|
||||
av_assert0(ff_hashtable_delete(ctx, &k));
|
||||
|
||||
// get should now return 0
|
||||
av_assert0(!ff_hashtable_get(ctx, &k, &v));
|
||||
|
||||
// sanity check remaining keys
|
||||
k = 2;
|
||||
v = 0;
|
||||
av_assert0(ff_hashtable_get(ctx, &k, &v));
|
||||
av_assert0(v == 2);
|
||||
k = 3;
|
||||
v = 0;
|
||||
av_assert0(ff_hashtable_get(ctx, &k, &v));
|
||||
av_assert0(v == 3);
|
||||
|
||||
ff_hashtable_freep(&ctx);
|
||||
|
||||
return 0;
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user