MAOS
Multithreaded Adaptive Optics Simulator
Loading...
Searching...
No Matches
bin.h File Reference

Data Structures

struct  header_t

Macros

#define swap2bytes(data)
#define swap4bytes(data)
#define swap8bytes(data)
#define M_INT   M_INT32
#define M_STR   M_INT8
#define M_SPINT   M_INT
#define M_DSP   M_DSP32
#define M_CSP   M_CSP32
#define M_SSP   M_SSP32
#define M_ZSP   M_ZSP32
#define IS_SHM(name)

Typedefs

typedef uint32_t M_ID
typedef int spint

Enumerations

enum  {
  M_0 =0x0 , M_DBL =0x6402 , M_INT64 =0x6403 , M_CMP =0x6404 ,
  M_INT32 =0x6405 , M_FLT =0x6408 , M_ZMP =0x6409 , M_INT8 =0x640A ,
  M_INT16 =0x640B , MCC_ANY =0x6421 , M_CSP64 =0x6400 , M_DSP64 =0x6401 ,
  M_CSP32 =0x6406 , M_DSP32 =0x6407 , M_ZSP64 =0x6432 , M_SSP64 =0x6430 ,
  M_ZSP32 =0x6433 , M_SSP32 =0x6431 , M_COMMENT =0x6500 , M_SKIP =0x6600 ,
  M_MAP32 =0x016408 , M_RMAP32 =0x026408 , M_LOC32 =0x036408 , M_MAP64 =0x016402 ,
  M_RMAP64 =0x026402 , M_LOC64 =0x036402 , M_EOD =0x64FF
}

Functions

int zfexist (const char *format,...)
void zftouch (const char *format,...)
int zfeof (file_t *fp)
long zfpos (file_t *fp)
long zfseek (file_t *fp, long offset, int whence)
void zfrewind (file_t *fp)
long zflen (file_t *fp)
file_t * zfopen (const char *fn, const char *mod)
file_t * zfdopen (int fd)
file_t * zfdopen_as (int fd, const char *mod, const char *suffix)
const char * zfname (file_t *fp)
int zfisfits (file_t *fp)
void zfclose (file_t *fp)
int zfwrite (const void *ptr, const size_t size, const size_t nmemb, file_t *fp)
int zfread (void *ptr, const size_t size, const size_t nmemb, file_t *fp)
int zferrno (file_t *fp)
uint64_t bytes_header (const char *keywords)
void write_timestamp (file_t *fp)
void write_header (const header_t *header, file_t *fp)
int read_header (header_t *header, file_t *fp)
long writearr (const void *fpn, const int isfile, const size_t size, const uint32_t magic, const char *keywords, const void *p, const uint64_t nx, const uint64_t ny)
struct mem_t * mem_new (void *p) __attribute__((warn_unused_result))
unsigned int mem_unref (mem_t **in)
mem_t * mem_ref (mem_t *in) __attribute__((warn_unused_result))
void mem_replace (mem_t *in, void *p)
int mem_nref (const mem_t *in)
void * mem_p (const mem_t *in)
mem_t * mmap_open (const char *fn, size_t msize, int rw)
void mmap_write_header (char **p0, uint32_t magic, long nx, long ny, const char *keywords)
void mmap_read_header (char **p0, uint32_t *magic, long *nx, long *ny, const char **pkeywords)
async_t * async_init (file_t *fp, const size_t size, const uint32_t magic, const char *str, const void *p, const uint64_t nx, const uint64_t ny)
int async_write (async_t *async, long offset, int wait)
void async_free (async_t *async)

Detailed Description

Defines our custom file format .bin that may be gzipped and the basic IO functions. All file read/write operators are through functions in this file. The routines can also operate on .fits files.


Data Structure Documentation

◆ header_t

struct header_t
Collaboration diagram for header_t:
Data Fields
uint32_t id
uint64_t nx
uint64_t ny
char * str

Macro Definition Documentation

◆ swap2bytes

#define swap2bytes ( data)
Value:
( (((data) >> 8) & 0x00FF) | (((data) << 8) & 0xFF00) )

◆ swap4bytes

#define swap4bytes ( data)
Value:
( (((data) >> 24) & 0x000000FF) | (((data) >> 8) & 0x0000FF00) | \
(((data) << 8) & 0x00FF0000) | (((data) << 24) & 0xFF000000) )

◆ swap8bytes

#define swap8bytes ( data)
Value:
( (((data) >> 56) & 0x00000000000000FF) | (((data) >> 40) & 0x000000000000FF00) | \
(((data) >> 24) & 0x0000000000FF0000) | (((data) >> 8) & 0x00000000FF000000) | \
(((data) << 8) & 0x000000FF00000000) | (((data) << 24) & 0x0000FF0000000000) | \
(((data) << 40) & 0x00FF000000000000) | (((data) << 56) & 0xFF00000000000000) )

◆ IS_SHM

#define IS_SHM ( name)
Value:
(name && ((name[0]=='/' && !strchr(name+1, '/')) || !mystrcmp(name, "/shm")))

Function Documentation

◆ zfexist()

int zfexist ( const char * format,
... )

Test whether a bin file exist.

Parameters
[in]formatprintf style format of the file name, followed by its arguments.
Returns
1 if the file exists, 0 otherwise.

◆ zftouch()

void zftouch ( const char * format,
... )

Update the modification time of a bin file.

Parameters
[in]formatprintf style format of the file name, followed by its arguments.

◆ zfeof()

int zfeof ( file_t * fp)

Return 1 if end of file is reached.

Parameters
[in]fpFile handle.
Returns
1 if an EOF error was recorded or the position is at the end of the file, 0 otherwise or if the position cannot be determined.

◆ zfpos()

long zfpos ( file_t * fp)

Tell position pointer in file.

Parameters
[in]fpFile handle.
Returns
Current position in bytes, or -1 on error.

◆ zfseek()

long zfseek ( file_t * fp,
long offset,
int whence )

Move the current position pointer, like fseek

Parameters
[in,out]fpFile handle.
[in]offsetOffset in bytes.
[in]whenceSEEK_SET, SEEK_CUR, or SEEK_END.
Returns
The resulting offset from the beginning of the file, or -1 on error. Seeking is not supported for zstd compressed files (returns -1) because the underlying fd offset is not the uncompressed position.

◆ zfrewind()

void zfrewind ( file_t * fp)

Move the file position pointer to the beginning and clear the error on success.

Parameters
[in,out]fpFile handle.

◆ zflen()

long zflen ( file_t * fp)

Return file length. The current file position is preserved.

Parameters
[in]fpFile handle.
Returns
File length in bytes, or -1 on error.

◆ zfopen()

file_t * zfopen ( const char * fni,
const char * mod )

Open a bin file for read/write access.

Whether the file is gzipped or not is automatically determined when open for reading. When open for writing, the file is compressed with gzip or zstd if the name ends with .gz or .zst. If the file has no recognized suffix, .bin is appended to the file name. Files opened for writing are locked.

Parameters
[in]fniFile name.
[in]modAccess mode: r for read, w for write (truncate), a for append.
Returns
Newly allocated file_t; close it with zfclose(). NULL on failure.

◆ zfdopen()

file_t * zfdopen ( int fd)

convert a socket or fd into file_t.

Parameters
[in]fdFile or socket descriptor. Closed by zfclose().
Returns
Newly allocated file_t; free it with zfclose().

◆ zfdopen_as()

file_t * zfdopen_as ( int fd,
const char * mod,
const char * suffix )

convert a file descriptor, socket, or pipe into file_t with the format implied by suffix.

Unlike zfopen(), the format is not detected from the content, so the descriptor does not need to be seekable.

Parameters
[in]fdFile, socket, or pipe descriptor. Closed by zfclose().
[in]modAccess mode: r for read, w for write.
[in]suffixFile name suffix that sets the format: .bin, .fits, with optional .gz or .zst appended. NULL is .bin.
Returns
Newly allocated file_t; free it with zfclose(). NULL on failure.

◆ zfname()

const char * zfname ( file_t * fp)

Open a bin file or socket for read/write access.

Whether the file is gzipped or not is automatically determined when open for reading. When open for writing, if the file name ends with .bin or .fits, will not be gzipped. If the file has no suffix, the file will be gzipped and .bin is appened to file name. Return the underlining filename

Parameters
[in]fpFile handle.
Returns
Reference to the file name; do not free. NULL if fp is NULL.

◆ zfisfits()

int zfisfits ( file_t * fp)

Return 1 when it is a fits file.

Parameters
[in]fpFile handle.
Returns
1 if fp is a fits file, 0 otherwise.

◆ zfclose()

void zfclose ( file_t * fp)

Close the file.

Parameters
[in]fpFile handle; freed. May be NULL.

◆ zfwrite()

int zfwrite ( const void * ptr,
const size_t size,
const size_t nmemb,
file_t * fp )

Handles byteswapping in fits file format then call zfwrite_do to do the actual writing. For fits files, each call writes one complete data unit padded to a full FITS block. Complex numbers must be written with size of each real component and twice the number of elements so that the real and imaginary parts are byte swapped separately (see writearr()).

Parameters
[in]ptrData to write.
[in]sizeSize of each element in bytes.
[in]nmembNumber of elements.
[in,out]fpFile handle; its error field is set on failure.
Returns
0 if success, non-zero otherwise.

◆ zfread()

int zfread ( void * ptr,
const size_t size,
const size_t nmemb,
file_t * fp )

Handles byteswapping in fits file format then call zfread_do to do the actual reading. For fits files, each call reads one complete data unit: the padding to a full FITS block (2880 bytes) after the data is skipped for all element sizes. Complex numbers must be read with size of each real component and twice the number of elements (see readvec()).

Parameters
[out]ptrBuffer to read into.
[in]sizeSize of each element in bytes.
[in]nmembNumber of elements.
[in,out]fpFile handle; its error field is set on failure.
Returns
0 when succeed, 1 at EOF, 2 on read error, and -1 if fp already has an error.

◆ zferrno()

int zferrno ( file_t * fp)

Return the error number

Parameters
[in]fpFile handle.
Returns
The error field (0: none, 1: EOF, 2: other error), or -1 if fp is NULL.

◆ bytes_header()

uint64_t bytes_header ( const char * keywords)

Get the length of mem to storge the header and its dimension, rounded to multiple of 8.

Parameters
[in]keywordsHeader string. May be NULL.
Returns
Number of bytes of the keyword block (string rounded to multiple of 8 plus 24), or 0 if keywords is NULL.

◆ write_header()

void write_header ( const header_t * header,
file_t * fp )

A unified header writing routine for .bin and .fits files. It write the array information and string header if any.

Parameters
[in]headerHeader with magic number, dimension, and optional string. For fits files, magic MCC_ANY saves the string as the global keyword.
[in,out]fpFile handle to write to.

◆ read_header()

int read_header ( header_t * header,
file_t * fp )

A unified header reading routine for .bin and .fits files. It read the array information and string header if any.

Parameters
[out]headerHeader to fill. header->str is newly allocated; the caller frees it.
[in,out]fpFile handle, read from its current position.
Returns
0 on success, non zero value if reading failed (1 at the end of data marker for bin files).

◆ writearr()

long writearr ( const void * fpn,
const int isfn,
const size_t size,
const uint32_t magic,
const char * str,
const void * p,
const uint64_t nx,
const uint64_t ny )

Write the time stamp as header into current location in the file. Write an 1-d or 2-d array into the file. First write a magic number that represents the data type. Then write two numbers representing the dimension. Finally write the data itself.

Parameters
[in]fpnThe file name (if isfn) or the opened file_t pointer.
[in]isfnIs this a filename or already opened file.
[in]sizeSize of each element.
[in]magicThe magic number. see bin.h.
[in]strThe header as string.
[in]pThe data of the array.
[in]nxNumber of rows. this index changes fastest.
[in]nyNumber of columns. 1 for vector.
Returns
0 on success, -1 if the file cannot be opened, or the zfwrite error code.

◆ mem_new()

struct mem_t * mem_new ( void * p)

Create a mem_t object.

Notice that nref is set to 0 by design: each user (e.g., X(new_do)) takes its own reference with mem_ref() and releases it with mem_unref(), which frees the object when the last reference is released. Default is set to reference heap memory.

Parameters
[in]pMemory to manage; owned by the returned mem_t.
Returns
Newly allocated mem_t with nref=0.

◆ mem_unref()

unsigned int mem_unref ( mem_t ** pin)

Unreference the mmaped memory. When the reference drops to zero free or unmap it.

Parameters
[in,out]pinPointer to the mem_t; set to NULL.
Returns
The remaining reference count. 0 if freed or pin is empty.

◆ mem_ref()

mem_t * mem_ref ( mem_t * in)

Add a reference to a mem_t.

Parameters
[in,out]inThe mem_t. May be NULL.
Returns
in. Release it with mem_unref().

◆ mem_replace()

void mem_replace ( mem_t * in,
void * p )

Replace internal vector. Use with care. Errors if the memory is mmaped or referenced more than once.

Parameters
[in,out]inThe mem_t.
[in]pNew memory pointer; the old one is not freed.

◆ mem_nref()

int mem_nref ( const mem_t * in)

Check whether it is a referenced data.

Parameters
[in]inThe mem_t. May be NULL.
Returns
kind+nref (non-zero if mmaped or referenced), or 0 if in is NULL.

◆ mem_p()

void * mem_p ( const mem_t * in)

Return internal pointer

Parameters
[in]inThe mem_t. May be NULL.
Returns
Reference to the internal memory; do not free. NULL if in is NULL.

◆ mmap_open()

mem_t * mmap_open ( const char * fn,
size_t msize,
int rw )

Open a file for write with mmmap. We don't provide an access control here for generic usage of the function. Lock on a special dummy file for access control.

Parameters
[in]fnFile name (.bin only), or a shared memory name. May be NULL for writing.
[in]msizeSize in bytes to map. For reading, 0 means the file length.
[in]rw1 to open for read/write (the file is created and resized), 0 for read only.
Returns
Newly allocated mem_t of the mapping with nref=0 (see mem_new()); each user takes a reference with mem_ref() and releases it with mem_unref(). NULL if the file cannot be found or opened for reading.

◆ mmap_write_header()

void mmap_write_header ( char ** p0,
uint32_t magic,
long nx,
long ny,
const char * keywords )

Initialize the header in the mmaped file. Upon exit, p0 points to the location of data p.

Parameters
[in,out]p0Pointer to the header location in the mmaped memory; advanced past the header.
[in]magicMagic number of the data type.
[in]nxNumber of rows.
[in]nyNumber of columns.
[in]keywordsHeader string. May be NULL.

◆ mmap_read_header()

void mmap_read_header ( char ** p0,
uint32_t * magic,
long * nx,
long * ny,
const char ** pkeywords )

Initialize the dimension from the header in the mmaped file.

Parameters
[in,out]p0Pointer to the header location in the mmaped memory; advanced to the data.
[out]magicMagic number of the data type.
[out]nxNumber of rows.
[out]nyNumber of columns.
[out]pkeywordsReference to the header string in the mapped memory; do not free. NULL if there is none. May be NULL.

◆ async_init()

async_t * async_init ( file_t * fp,
const size_t size,
const uint32_t magic,
const char * str,
const void * p,
const uint64_t nx,
const uint64_t ny )

Allocates space in fp for our block of data by writing header first and skip to the end of data offset. It has the same interface as writearr except only fp is allowed Use aio_* api if USE_ASYNC is set, otherwise use pwrite() aio_* api launches threads for async writing. pwrite is much simpler for our usecase. There is no impact on simulation step timing. we don't use mmap to write to file as it causes excessive traffic to NFS file system.

Parameters
[in,out]fpFile handle opened for writing.
[in]sizeSize of each element.
[in]magicThe magic number. see bin.h.
[in]strThe header as string.
[in]pThe data of the array. Must stay valid until async_free().
[in]nxNumber of rows.
[in]nyNumber of columns.
Returns
Newly allocated async_t; free it with async_free(). NULL on failure or for fits files, which are not supported.

◆ async_write()

int async_write ( async_t * async,
long offset,
int wait )

Write to file asynchronously.

Parameters
[in,out]asyncThe async writing state. May be NULL.
[in]offsetThe end (in bytes from the start of the data) of the block that is ready to write.
[in]waitWait for the previous write to finish (only with USE_ASYNC).
Returns
0 on success or if async is NULL, -1 on error.

◆ async_free()

void async_free ( async_t * async)

Wait for pending asynchronous writes to finish and free async.

Parameters
[in]asyncThe async writing state; freed. May be NULL.