erasurecode.c File Reference

#include <assert.h>
#include <zlib.h>
#include "list.h"
#include "erasurecode.h"
#include "erasurecode_backend.h"
#include "erasurecode_helpers.h"
#include "erasurecode_helpers_ext.h"
#include "erasurecode_preprocessing.h"
#include "erasurecode_postprocessing.h"
#include "erasurecode_stdinc.h"
#include "alg_sig.h"
#include "erasurecode_log.h"

Go to the source code of this file.

Functions
static	SLIST_HEAD (backend_list, ec_backend)
	Look up a backend instance by descriptor.
static int	liberasurecode_backend_alloc_desc (void)
	Allocated backend instance descriptor.
static void	print_dlerror (const char *caller)
static void *	liberasurecode_backend_open (ec_backend_t instance)
static int	liberasurecode_backend_close (ec_backend_t instance)
void	__attribute__ ((constructor))
void	__attribute__ ((destructor))
int	liberasurecode_backend_available (const ec_backend_id_t backend_id)
	Checks if a given backend is available.
int	liberasurecode_instance_create (const ec_backend_id_t id, struct ec_args *args)
	Create a liberasurecode instance and return a descriptor for use with EC operations (encode, decode, reconstruct)
int	liberasurecode_instance_destroy (int desc)
	Close a liberasurecode instance.
int	liberasurecode_encode_cleanup (int desc, char **encoded_data, char **encoded_parity)
	Cleanup structures allocated by librasurecode_encode.
int	liberasurecode_encode (int desc, const char *orig_data, uint64_t orig_data_size, char ***encoded_data, char ***encoded_parity, uint64_t *fragment_len)
	Erasure encode a data buffer.
int	liberasurecode_decode_cleanup (int desc, char *data)
	Cleanup structures allocated by librasurecode_decode.
int	liberasurecode_decode (int desc, char **available_fragments, int num_fragments, uint64_t fragment_len, int force_metadata_checks, char **out_data, uint64_t *out_data_len)
	Reconstruct original data from a set of k encoded fragments.
int	liberasurecode_reconstruct_fragment (int desc, char **available_fragments, int num_fragments, uint64_t fragment_len, int destination_idx, char *out_fragment)
	Reconstruct a missing fragment from a subset of available fragments.
int	liberasurecode_fragments_needed (int desc, int *fragments_to_reconstruct, int *fragments_to_exclude, int *fragments_needed)
	Return a list of lists with valid rebuild indexes given a list of missing indexes.
int	liberasurecode_get_fragment_metadata (char *fragment, fragment_metadata_t *fragment_metadata)
	Get opaque metadata for a fragment.
int	is_invalid_fragment_header (fragment_header_t *header)
int	liberasurecode_verify_fragment_metadata (ec_backend_t be, fragment_metadata_t *md)
static int	is_invalid_fragment_metadata (int desc, fragment_metadata_t *fragment_metadata)
int	is_invalid_fragment (int desc, char *fragment)
int	liberasurecode_verify_stripe_metadata (int desc, char **fragments, int num_fragments)
int	liberasurecode_get_aligned_data_size (int desc, uint64_t data_len)
	This computes the aligned size of a buffer passed into the encode function.
int	liberasurecode_get_minimum_encode_size (int desc)
	This will return the minumum encode size, which is the minimum buffer size that can be encoded.
int	liberasurecode_get_fragment_size (int desc, int data_len)
uint32_t	liberasurecode_get_version (void)
	This will return the liberasurecode version for the descriptor.

Variables
struct ec_backend_common	backend_null
struct ec_backend_common	backend_flat_xor_hd
struct ec_backend_common	backend_jerasure_rs_vand
struct ec_backend_common	backend_jerasure_rs_cauchy
struct ec_backend_common	backend_isa_l_rs_vand
struct ec_backend_common	backend_shss
struct ec_backend_common	backend_liberasurecode_rs_vand
struct ec_backend_common	backend_isa_l_rs_cauchy
struct ec_backend_common	backend_libphazr
struct ec_backend_common	backend_isa_l_rs_vand_inv
struct ec_backend_common	backend_isa_l_rs_lrc
static ec_backend_t	ec_backends_supported []

Function Documentation

__attribute__() [1/2]

void __attribute__

(

(constructor)

)

Definition at line 157 of file erasurecode.c.

__attribute__() [2/2]

void __attribute__

(

(destructor)

)

Definition at line 163 of file erasurecode.c.

is_invalid_fragment()

int is_invalid_fragment	(	int	desc,
		char *	fragment
	)

Definition at line 1222 of file erasurecode.c.

References get_libec_version(), is_invalid_fragment_metadata(), and liberasurecode_get_fragment_metadata().

Referenced by liberasurecode_decode().

is_invalid_fragment_header()

int is_invalid_fragment_header

(

fragment_header_t *

header

)

Definition at line 1135 of file erasurecode.c.

References liberasurecode_crc32_alt().

Referenced by liberasurecode_decode(), liberasurecode_get_fragment_metadata(), and liberasurecode_reconstruct_fragment().

is_invalid_fragment_metadata()

static int is_invalid_fragment_metadata ( int  desc, fragment_metadata_t *  fragment_metadata  )

static

Definition at line 1189 of file erasurecode.c.

References liberasurecode_verify_fragment_metadata().

Referenced by is_invalid_fragment(), and liberasurecode_verify_stripe_metadata().

liberasurecode_backend_alloc_desc()

static int liberasurecode_backend_alloc_desc ( void  )

static

Allocated backend instance descriptor.

Returns a unique descriptor for a new backend. The caller must hold active_instances_rwlock

Definition at line 113 of file erasurecode.c.

Referenced by liberasurecode_instance_create().

liberasurecode_backend_available()

int liberasurecode_backend_available

(

const ec_backend_id_t

backend_id

)

Checks if a given backend is available.

Parameters

backend_id

- one of the supported backends.

Returns

1 if a backend is available; 0 otherwise

Definition at line 177 of file erasurecode.c.

References ec_backends_supported, liberasurecode_backend_close(), and liberasurecode_backend_open().

liberasurecode_backend_close()

static int liberasurecode_backend_close ( ec_backend_t  instance )

static

Definition at line 143 of file erasurecode.c.

Referenced by liberasurecode_backend_available(), liberasurecode_instance_create(), and liberasurecode_instance_destroy().

liberasurecode_backend_open()

static void * liberasurecode_backend_open ( ec_backend_t  instance )

static

Definition at line 135 of file erasurecode.c.

Referenced by liberasurecode_backend_available(), and liberasurecode_instance_create().

liberasurecode_decode()

int liberasurecode_decode	(	int	desc,
		char **	available_fragments,
		int	num_fragments,
		uint64_t	fragment_len,
		int	force_metadata_checks,
		char **	out_data,
		uint64_t *	out_data_len
	)

Reconstruct original data from a set of k encoded fragments.

Parameters

desc	- liberasurecode descriptor/handle from liberasurecode_instance_create()
available_fragments	- erasure encoded fragments (> = k)
num_fragments	- number of fragments being passed in
fragment_len	- length of each fragment (assume they are the same)
force_metadata_checks	- force fragment metadata checks (default: 0)
out_data	- output pointer to decoded data
out_data_len	- output length of decoded output

Returns

0 on success, -error code otherwise

Definition at line 530 of file erasurecode.c.

References alloc_and_set_buffer(), get_fragment_partition(), is_invalid_fragment(), and is_invalid_fragment_header().

liberasurecode_decode_cleanup()

int liberasurecode_decode_cleanup	(	int	desc,
		char *	data
	)

Cleanup structures allocated by librasurecode_decode.

The caller has no context, so cannot safely free memory allocated by liberasurecode, so it must pass the deallocation responsibility back to liberasurecode.

Parameters

desc	- liberasurecode descriptor/handle from liberasurecode_instance_create()
data	- (char *) buffer of data decoded by librasurecode_decode

Returns

0 in success; -error otherwise

Definition at line 499 of file erasurecode.c.

liberasurecode_encode()

int liberasurecode_encode	(	int	desc,
		const char *	orig_data,
		uint64_t	orig_data_size,
		char ***	encoded_data,
		char ***	encoded_parity,
		uint64_t *	fragment_len
	)

Erasure encode a data buffer.

Parameters

desc	- liberasurecode descriptor/handle from liberasurecode_instance_create()
orig_data	- data to encode
orig_data_size	- length of data to encode
encoded_data	- pointer to output array (char **) of k data fragments (char *), allocated by the callee
encoded_parity	- pointer to output array (char **) of m parity fragments (char *), allocated by the callee
fragment_len	- pointer to output length of each fragment, assuming all fragments are the same length

Returns

0 on success, -error code otherwise

Definition at line 389 of file erasurecode.c.

References liberasurecode_encode_cleanup().

liberasurecode_encode_cleanup()

int liberasurecode_encode_cleanup	(	int	desc,
		char **	encoded_data,
		char **	encoded_parity
	)

Cleanup structures allocated by librasurecode_encode.

The caller has no context, so cannot safely free memory allocated by liberasurecode, so it must pass the deallocation responsibility back to liberasurecode.

Parameters

desc	- liberasurecode descriptor/handle from liberasurecode_instance_create()
encoded_data	- (char **) array of k data fragments (char *), allocated by liberasurecode_encode
encoded_parity	- (char **) array of m parity fragments (char *), allocated by liberasurecode_encode

Returns

0 in success; -error otherwise

Definition at line 334 of file erasurecode.c.

Referenced by liberasurecode_encode().

liberasurecode_fragments_needed()

int liberasurecode_fragments_needed	(	int	desc,
		int *	fragments_to_reconstruct,
		int *	fragments_to_exclude,
		int *	fragments_needed
	)

Return a list of lists with valid rebuild indexes given a list of missing indexes.

@desc: liberasurecode instance descriptor (obtained with liberasurecode_instance_create) @fragments_to_reconstruct list of indexes to reconstruct @fragments_to_exclude list of indexes to exclude from reconstruction equation @fragments_needed list of fragments needed to reconstruct fragments in fragments_to_reconstruct; output param

Returns

0 on success; -error code otherwise

Definition at line 991 of file erasurecode.c.

liberasurecode_get_aligned_data_size()

int liberasurecode_get_aligned_data_size	(	int	desc,
		uint64_t	data_len
	)

This computes the aligned size of a buffer passed into the encode function.

The encode function must pad fragments to be aligned with the word size (w) and the last fragment also needs to be aligned. This computes the sum of the aligned fragment sizes for a given buffer to encode.

Definition at line 1303 of file erasurecode.c.

Referenced by liberasurecode_get_minimum_encode_size().

liberasurecode_get_fragment_metadata()

int liberasurecode_get_fragment_metadata	(	char *	fragment,
		fragment_metadata_t *	fragment_metadata
	)

Get opaque metadata for a fragment.

The metadata is opaque to the client, but meaningful to the underlying library. It is used to verify stripes in verify_stripe_metadata().

Parameters

fragment	- fragment pointer
fragment_metadata	- pointer to output fragment metadata struct (reference passed by the user)

Definition at line 1050 of file erasurecode.c.

References get_data_ptr_from_fragment(), is_invalid_fragment_header(), and liberasurecode_crc32_alt().

Referenced by is_invalid_fragment().

liberasurecode_get_fragment_size()

int liberasurecode_get_fragment_size	(	int	desc,
		int	data_len
	)

Definition at line 1345 of file erasurecode.c.

liberasurecode_get_minimum_encode_size()

int liberasurecode_get_minimum_encode_size

(

int

desc

)

This will return the minumum encode size, which is the minimum buffer size that can be encoded.

Definition at line 1340 of file erasurecode.c.

References liberasurecode_get_aligned_data_size().

liberasurecode_get_version()

uint32_t liberasurecode_get_version

(

void

)

This will return the liberasurecode version for the descriptor.

Definition at line 1374 of file erasurecode.c.

liberasurecode_instance_create()

int liberasurecode_instance_create	(	const ec_backend_id_t	id,
		struct ec_args *	args
	)

Create a liberasurecode instance and return a descriptor for use with EC operations (encode, decode, reconstruct)

Parameters

id	- one of the supported backends as defined by ec_backend_id_t
args	- arguments to the EC backend arguments common to all backends k - number of data fragments m - number of parity fragments w - word size, in bits hd - hamming distance (=m for Reed-Solomon) ct - fragment checksum type (stored with the fragment metadata) backend-specific arguments null_args - arguments for the null backend flat_xor_hd, jerasure do not require any special args

Returns

liberasurecode instance descriptor (int > 0)

Definition at line 211 of file erasurecode.c.

References ec_backends_supported, liberasurecode_backend_alloc_desc(), liberasurecode_backend_close(), liberasurecode_backend_open(), and print_dlerror().

liberasurecode_instance_destroy()

int liberasurecode_instance_destroy

(

int

desc

)

Close a liberasurecode instance.

Parameters

desc	- liberasurecode descriptor to close

Definition at line 292 of file erasurecode.c.

References liberasurecode_backend_close().

liberasurecode_reconstruct_fragment()

int liberasurecode_reconstruct_fragment	(	int	desc,
		char **	available_fragments,
		int	num_fragments,
		uint64_t	fragment_len,
		int	destination_idx,
		char *	out_fragment
	)

Reconstruct a missing fragment from a subset of available fragments.

Parameters

desc	- liberasurecode descriptor/handle from liberasurecode_instance_create()
fragment_len	- size in bytes of the fragments
available_fragments	- erasure encoded fragments
num_fragments	- number of fragments being passed in
destination_idx	- missing idx to reconstruct
out_fragment	- output of reconstruct

Returns

0 on success, -error code otherwise

Definition at line 764 of file erasurecode.c.

References alloc_and_set_buffer(), get_fragment_partition(), and is_invalid_fragment_header().

liberasurecode_verify_fragment_metadata()

int liberasurecode_verify_fragment_metadata	(	ec_backend_t	be,
		fragment_metadata_t *	md
	)

Definition at line 1172 of file erasurecode.c.

Referenced by is_invalid_fragment_metadata().

liberasurecode_verify_stripe_metadata()

int liberasurecode_verify_stripe_metadata	(	int	desc,
		char **	fragments,
		int	num_fragments
	)

Definition at line 1262 of file erasurecode.c.

References is_invalid_fragment_metadata().

print_dlerror()

static void print_dlerror ( const char *  caller )

static

Definition at line 125 of file erasurecode.c.

Referenced by liberasurecode_instance_create().

SLIST_HEAD()

static SLIST_HEAD ( backend_list  , ec_backend    )

static

Look up a backend instance by descriptor.

Returns

pointer to a registered liberasurecode instance The caller must hold active_instances_rwlock

Definition at line 76 of file erasurecode.c.

Variable Documentation

backend_flat_xor_hd

struct ec_backend_common backend_flat_xor_hd

extern

Definition at line 50 of file flat_xor_hd.c.

Referenced by flat_xor_is_compatible_with().

backend_isa_l_rs_cauchy

struct ec_backend_common backend_isa_l_rs_cauchy

extern

Definition at line 47 of file isa_l_rs_cauchy.c.

Referenced by isa_l_rs_cauchy_is_compatible_with().

backend_isa_l_rs_lrc

struct ec_backend_common backend_isa_l_rs_lrc

extern

Definition at line 46 of file isa_l_rs_lrc.c.

Referenced by isa_l_rs_lrc_is_compatible_with().

backend_isa_l_rs_vand

struct ec_backend_common backend_isa_l_rs_vand

extern

Definition at line 46 of file isa_l_rs_vand.c.

Referenced by isa_l_rs_vand_is_compatible_with().

backend_isa_l_rs_vand_inv

struct ec_backend_common backend_isa_l_rs_vand_inv

extern

Definition at line 47 of file isa_l_rs_vand_inv.c.

Referenced by isa_l_rs_vand_inv_is_compatible_with().

backend_jerasure_rs_cauchy

struct ec_backend_common backend_jerasure_rs_cauchy

extern

Definition at line 49 of file jerasure_rs_cauchy.c.

Referenced by jerasure_rs_cauchy_is_compatible_with().

backend_jerasure_rs_vand

struct ec_backend_common backend_jerasure_rs_vand

extern

Definition at line 49 of file jerasure_rs_vand.c.

Referenced by jerasure_rs_vand_is_compatible_with().

backend_liberasurecode_rs_vand

struct ec_backend_common backend_liberasurecode_rs_vand

extern

Definition at line 47 of file liberasurecode_rs_vand.c.

Referenced by liberasurecode_rs_vand_is_compatible_with().

backend_libphazr

struct ec_backend_common backend_libphazr

extern

Definition at line 48 of file libphazr.c.

Referenced by pio_is_compatible_with().

backend_null

struct ec_backend_common backend_null

extern

backend_shss

struct ec_backend_common backend_shss

extern

Definition at line 41 of file shss.c.

Referenced by shss_is_compatible_with().

ec_backends_supported

ec_backend_t ec_backends_supported[]

static

Initial value:

= {
    (ec_backend_t) &backend_null,
    (ec_backend_t) &backend_jerasure_rs_vand,
    (ec_backend_t) &backend_jerasure_rs_cauchy,
    (ec_backend_t) &backend_flat_xor_hd,
    (ec_backend_t) &backend_isa_l_rs_vand,
    (ec_backend_t) &backend_shss,
    (ec_backend_t) &backend_liberasurecode_rs_vand,
    (ec_backend_t) &backend_isa_l_rs_cauchy,
    (ec_backend_t) &backend_libphazr,
    (ec_backend_t) &backend_isa_l_rs_vand_inv,
    (ec_backend_t) &backend_isa_l_rs_lrc,
    NULL,
}

Definition at line 58 of file erasurecode.c.

Referenced by liberasurecode_backend_available(), and liberasurecode_instance_create().