libunicode.c File Reference

#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <assert.h>
#include "cutils.h"
#include "libunicode.h"
#include "libunicode-table.h"

Include dependency graph for libunicode.c:

Macros
#define	UNICODE_INDEX_BLOCK_LEN   32
#define	CASE_U   (1 << 0)
#define	CASE_L   (1 << 1)
#define	CASE_F   (1 << 2)
#define	MR(x)   (1 << RUN_TYPE_ ## x)
#define	UNICODE_DECOMP_LEN_MAX   18
#define	M(id)   (1U << UNICODE_GC_ ## id)
#define	POP_STACK_LEN_MAX   4
#define	S   UNICODE_C_SPACE
#define	D   UNICODE_C_DIGIT
#define	X   UNICODE_C_XDIGIT
#define	U   UNICODE_C_UPPER
#define	L   UNICODE_C_LOWER
#define	_   UNICODE_C_UNDER
#define	d   UNICODE_C_DOLLAR
#define	SEQ_MAX_LEN   16

Enumerations
enum	{   RUN_TYPE_U , RUN_TYPE_L , RUN_TYPE_UF , RUN_TYPE_LF ,   RUN_TYPE_UL , RUN_TYPE_LSU , RUN_TYPE_U2L_399_EXT2 , RUN_TYPE_UF_D20 ,   RUN_TYPE_UF_D1_EXT , RUN_TYPE_U_EXT , RUN_TYPE_LF_EXT , RUN_TYPE_UF_EXT2 ,   RUN_TYPE_LF_EXT2 , RUN_TYPE_UF_EXT3 }
enum	DecompTypeEnum {   DECOMP_TYPE_C1 , DECOMP_TYPE_L1 , DECOMP_TYPE_L2 , DECOMP_TYPE_L3 ,   DECOMP_TYPE_L4 , DECOMP_TYPE_L5 , DECOMP_TYPE_L6 , DECOMP_TYPE_L7 ,   DECOMP_TYPE_LL1 , DECOMP_TYPE_LL2 , DECOMP_TYPE_S1 , DECOMP_TYPE_S2 ,   DECOMP_TYPE_S3 , DECOMP_TYPE_S4 , DECOMP_TYPE_S5 , DECOMP_TYPE_I1 ,   DECOMP_TYPE_I2_0 , DECOMP_TYPE_I2_1 , DECOMP_TYPE_I3_1 , DECOMP_TYPE_I3_2 ,   DECOMP_TYPE_I4_1 , DECOMP_TYPE_I4_2 , DECOMP_TYPE_B1 , DECOMP_TYPE_B2 ,   DECOMP_TYPE_B3 , DECOMP_TYPE_B4 , DECOMP_TYPE_B5 , DECOMP_TYPE_B6 ,   DECOMP_TYPE_B7 , DECOMP_TYPE_B8 , DECOMP_TYPE_B18 , DECOMP_TYPE_LS2 ,   DECOMP_TYPE_PAT3 , DECOMP_TYPE_S2_UL , DECOMP_TYPE_LS2_UL }
enum	PropOPEnum {   POP_GC , POP_PROP , POP_CASE , POP_UNION ,   POP_INTER , POP_XOR , POP_INVERT , POP_END }

Functions
static int	lre_case_conv1 (uint32_t c, int conv_type)
static int	lre_case_conv_entry (uint32_t *res, uint32_t c, int conv_type, uint32_t idx, uint32_t v)
int	lre_case_conv (uint32_t *res, uint32_t c, int conv_type)
static int	lre_case_folding_entry (uint32_t c, uint32_t idx, uint32_t v, BOOL is_unicode)
int	lre_canonicalize (uint32_t c, BOOL is_unicode)
static uint32_t	get_le24 (const uint8_t *ptr)
static int	get_index_pos (uint32_t *pcode, uint32_t c, const uint8_t *index_table, int index_table_len)
static BOOL	lre_is_in_table (uint32_t c, const uint8_t *table, const uint8_t *index_table, int index_table_len)
BOOL	lre_is_cased (uint32_t c)
BOOL	lre_is_case_ignorable (uint32_t c)
static __maybe_unused void	cr_dump (CharRange *cr)
static void *	cr_default_realloc (void *opaque, void *ptr, size_t size)
void	cr_init (CharRange *cr, void *mem_opaque, DynBufReallocFunc *realloc_func)
void	cr_free (CharRange *cr)
int	cr_realloc (CharRange *cr, int size)
int	cr_copy (CharRange *cr, const CharRange *cr1)
static void	cr_compress (CharRange *cr)
int	cr_op (CharRange *cr, const uint32_t *a_pt, int a_len, const uint32_t *b_pt, int b_len, int op)
int	cr_op1 (CharRange *cr, const uint32_t *b_pt, int b_len, int op)
int	cr_invert (CharRange *cr)
static int	unicode_case1 (CharRange *cr, int case_mask)
static int	point_cmp (const void *p1, const void *p2, void *arg)
static void	cr_sort_and_remove_overlap (CharRange *cr)
int	cr_regexp_canonicalize (CharRange *cr, BOOL is_unicode)
BOOL	lre_is_id_start (uint32_t c)
BOOL	lre_is_id_continue (uint32_t c)
static uint32_t	unicode_get_short_code (uint32_t c)
static uint32_t	unicode_get_lower_simple (uint32_t c)
static uint16_t	unicode_get16 (const uint8_t *p)
static int	unicode_decomp_entry (uint32_t *res, uint32_t c, int idx, uint32_t code, uint32_t len, uint32_t type)
static int	unicode_decomp_char (uint32_t *res, uint32_t c, BOOL is_compat1)
static int	unicode_compose_pair (uint32_t c0, uint32_t c1)
static int	unicode_get_cc (uint32_t c)
static void	sort_cc (int *buf, int len)
static void	to_nfd_rec (DynBuf *dbuf, const int *src, int src_len, int is_compat)
static int	compose_pair (uint32_t c0, uint32_t c1)
int	unicode_normalize (uint32_t **pdst, const uint32_t *src, int src_len, UnicodeNormalizationEnum n_type, void *opaque, DynBufReallocFunc *realloc_func)
static int	unicode_find_name (const char *name_table, const char *name)
int	unicode_script (CharRange *cr, const char *script_name, BOOL is_ext)
static int	unicode_general_category1 (CharRange *cr, uint32_t gc_mask)
static int	unicode_prop1 (CharRange *cr, int prop_idx)
static int	unicode_prop_ops (CharRange *cr,...)
int	unicode_general_category (CharRange *cr, const char *gc_name)
int	unicode_prop (CharRange *cr, const char *prop_name)
BOOL	lre_is_space_non_ascii (uint32_t c)
static int	unicode_sequence_prop1 (int seq_prop_idx, UnicodeSequencePropCB *cb, void *opaque, CharRange *cr)
int	unicode_sequence_prop (const char *prop_name, UnicodeSequencePropCB *cb, void *opaque, CharRange *cr)

Variables
static const uint32_t	unicode_gc_mask_table []
uint8_t const	lre_ctype_bits [256]
static const uint16_t	char_range_s []

Macro Definition Documentation

UNICODE_INDEX_BLOCK_LEN

#define UNICODE_INDEX_BLOCK_LEN   32

CASE_U

#define CASE_U   (1 << 0)

CASE_L

#define CASE_L   (1 << 1)

CASE_F

#define CASE_F   (1 << 2)

MR

#define MR

(

x

)

(1 << RUN_TYPE_ ## x)

UNICODE_DECOMP_LEN_MAX

#define UNICODE_DECOMP_LEN_MAX   18

M

#define M

(

id

)

(1U << UNICODE_GC_ ## id)

POP_STACK_LEN_MAX

#define POP_STACK_LEN_MAX   4

S

#define S   UNICODE_C_SPACE

D

#define D   UNICODE_C_DIGIT

X

#define X   UNICODE_C_XDIGIT

U

#define U   UNICODE_C_UPPER

L

#define L   UNICODE_C_LOWER

_

#define _   UNICODE_C_UNDER

d

#define d   UNICODE_C_DOLLAR

SEQ_MAX_LEN

#define SEQ_MAX_LEN   16

Enumeration Type Documentation

anonymous enum

anonymous enum

Enumerator
RUN_TYPE_U
RUN_TYPE_L
RUN_TYPE_UF
RUN_TYPE_LF
RUN_TYPE_UL
RUN_TYPE_LSU
RUN_TYPE_U2L_399_EXT2
RUN_TYPE_UF_D20
RUN_TYPE_UF_D1_EXT
RUN_TYPE_U_EXT
RUN_TYPE_LF_EXT
RUN_TYPE_UF_EXT2
RUN_TYPE_LF_EXT2
RUN_TYPE_UF_EXT3

DecompTypeEnum

enum DecompTypeEnum

Enumerator
DECOMP_TYPE_C1
DECOMP_TYPE_L1
DECOMP_TYPE_L2
DECOMP_TYPE_L3
DECOMP_TYPE_L4
DECOMP_TYPE_L5
DECOMP_TYPE_L6
DECOMP_TYPE_L7
DECOMP_TYPE_LL1
DECOMP_TYPE_LL2
DECOMP_TYPE_S1
DECOMP_TYPE_S2
DECOMP_TYPE_S3
DECOMP_TYPE_S4
DECOMP_TYPE_S5
DECOMP_TYPE_I1
DECOMP_TYPE_I2_0
DECOMP_TYPE_I2_1
DECOMP_TYPE_I3_1
DECOMP_TYPE_I3_2
DECOMP_TYPE_I4_1
DECOMP_TYPE_I4_2
DECOMP_TYPE_B1
DECOMP_TYPE_B2
DECOMP_TYPE_B3
DECOMP_TYPE_B4
DECOMP_TYPE_B5
DECOMP_TYPE_B6
DECOMP_TYPE_B7
DECOMP_TYPE_B8
DECOMP_TYPE_B18
DECOMP_TYPE_LS2
DECOMP_TYPE_PAT3
DECOMP_TYPE_S2_UL
DECOMP_TYPE_LS2_UL

PropOPEnum

enum PropOPEnum

Enumerator
POP_GC
POP_PROP
POP_CASE
POP_UNION
POP_INTER
POP_XOR
POP_INVERT
POP_END

Function Documentation

lre_case_conv1()

static int lre_case_conv1 ( uint32_t  c, int  conv_type  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

lre_case_conv_entry()

static int lre_case_conv_entry ( uint32_t *  res, uint32_t  c, int  conv_type, uint32_t  idx, uint32_t  v  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

lre_case_conv()

int lre_case_conv	(	uint32_t *	res,
		uint32_t	c,
		int	conv_type
	)

Here is the call graph for this function:

Here is the caller graph for this function:

lre_case_folding_entry()

static int lre_case_folding_entry ( uint32_t  c, uint32_t  idx, uint32_t  v, BOOL  is_unicode  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

lre_canonicalize()

int lre_canonicalize	(	uint32_t	c,
		BOOL	is_unicode
	)

Here is the call graph for this function:

Here is the caller graph for this function:

get_le24()

static uint32_t get_le24 ( const uint8_t *  ptr )

static

Here is the caller graph for this function:

get_index_pos()

static int get_index_pos ( uint32_t *  pcode, uint32_t  c, const uint8_t *  index_table, int  index_table_len  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

lre_is_in_table()

static BOOL lre_is_in_table ( uint32_t  c, const uint8_t *  table, const uint8_t *  index_table, int  index_table_len  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

lre_is_cased()

BOOL lre_is_cased

(

uint32_t

c

)

Here is the call graph for this function:

Here is the caller graph for this function:

lre_is_case_ignorable()

BOOL lre_is_case_ignorable

(

uint32_t

c

)

Here is the call graph for this function:

Here is the caller graph for this function:

cr_dump()

static __maybe_unused void cr_dump ( CharRange *  cr )

static

cr_default_realloc()

static void * cr_default_realloc ( void *  opaque, void *  ptr, size_t  size  )

static

Here is the caller graph for this function:

cr_init()

void cr_init	(	CharRange *	cr,
		void *	mem_opaque,
		DynBufReallocFunc *	realloc_func
	)

Here is the call graph for this function:

Here is the caller graph for this function:

cr_free()

void cr_free

(

CharRange *

cr

)

Here is the caller graph for this function:

cr_realloc()

int cr_realloc	(	CharRange *	cr,
		int	size
	)

Here is the call graph for this function:

Here is the caller graph for this function:

cr_copy()

int cr_copy	(	CharRange *	cr,
		const CharRange *	cr1
	)

Here is the call graph for this function:

Here is the caller graph for this function:

cr_compress()

static void cr_compress ( CharRange *  cr )

static

Here is the caller graph for this function:

cr_op()

int cr_op	(	CharRange *	cr,
		const uint32_t *	a_pt,
		int	a_len,
		const uint32_t *	b_pt,
		int	b_len,
		int	op
	)

Here is the call graph for this function:

Here is the caller graph for this function:

cr_op1()

int cr_op1	(	CharRange *	cr,
		const uint32_t *	b_pt,
		int	b_len,
		int	op
	)

Here is the call graph for this function:

Here is the caller graph for this function:

cr_invert()

int cr_invert

(

CharRange *

cr

)

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_case1()

static int unicode_case1 ( CharRange *  cr, int  case_mask  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

point_cmp()

static int point_cmp ( const void *  p1, const void *  p2, void *  arg  )

static

Here is the caller graph for this function:

cr_sort_and_remove_overlap()

static void cr_sort_and_remove_overlap ( CharRange *  cr )

static

Here is the call graph for this function:

Here is the caller graph for this function:

cr_regexp_canonicalize()

int cr_regexp_canonicalize	(	CharRange *	cr,
		BOOL	is_unicode
	)

Here is the call graph for this function:

Here is the caller graph for this function:

lre_is_id_start()

BOOL lre_is_id_start

(

uint32_t

c

)

Here is the call graph for this function:

Here is the caller graph for this function:

lre_is_id_continue()

BOOL lre_is_id_continue

(

uint32_t

c

)

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_get_short_code()

static uint32_t unicode_get_short_code ( uint32_t  c )

static

Here is the caller graph for this function:

unicode_get_lower_simple()

static uint32_t unicode_get_lower_simple ( uint32_t  c )

static

Here is the caller graph for this function:

unicode_get16()

static uint16_t unicode_get16 ( const uint8_t *  p )

static

Here is the caller graph for this function:

unicode_decomp_entry()

static int unicode_decomp_entry ( uint32_t *  res, uint32_t  c, int  idx, uint32_t  code, uint32_t  len, uint32_t  type  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_decomp_char()

static int unicode_decomp_char ( uint32_t *  res, uint32_t  c, BOOL  is_compat1  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_compose_pair()

static int unicode_compose_pair ( uint32_t  c0, uint32_t  c1  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_get_cc()

static int unicode_get_cc ( uint32_t  c )

static

Here is the call graph for this function:

Here is the caller graph for this function:

sort_cc()

static void sort_cc ( int *  buf, int  len  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

to_nfd_rec()

static void to_nfd_rec ( DynBuf *  dbuf, const int *  src, int  src_len, int  is_compat  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

compose_pair()

static int compose_pair ( uint32_t  c0, uint32_t  c1  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_normalize()

int unicode_normalize	(	uint32_t **	pdst,
		const uint32_t *	src,
		int	src_len,
		UnicodeNormalizationEnum	n_type,
		void *	opaque,
		DynBufReallocFunc *	realloc_func
	)

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_find_name()

static int unicode_find_name ( const char *  name_table, const char *  name  )

static

Here is the caller graph for this function:

unicode_script()

int unicode_script	(	CharRange *	cr,
		const char *	script_name,
		BOOL	is_ext
	)

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_general_category1()

static int unicode_general_category1 ( CharRange *  cr, uint32_t  gc_mask  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_prop1()

static int unicode_prop1 ( CharRange *  cr, int  prop_idx  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_prop_ops()

static int unicode_prop_ops ( CharRange *  cr,   ...  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_general_category()

int unicode_general_category	(	CharRange *	cr,
		const char *	gc_name
	)

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_prop()

int unicode_prop	(	CharRange *	cr,
		const char *	prop_name
	)

Here is the call graph for this function:

Here is the caller graph for this function:

lre_is_space_non_ascii()

BOOL lre_is_space_non_ascii

(

uint32_t

c

)

Here is the caller graph for this function:

unicode_sequence_prop1()

static int unicode_sequence_prop1 ( int  seq_prop_idx, UnicodeSequencePropCB *  cb, void *  opaque, CharRange *  cr  )

static

Here is the call graph for this function:

Here is the caller graph for this function:

unicode_sequence_prop()

int unicode_sequence_prop	(	const char *	prop_name,
		UnicodeSequencePropCB *	cb,
		void *	opaque,
		CharRange *	cr
	)

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

unicode_gc_mask_table

const uint32_t unicode_gc_mask_table[]

static

Initial value:

= {
    M(Lu) | M(Ll) | M(Lt), 
    M(Lu) | M(Ll) | M(Lt) | M(Lm) | M(Lo), 
    M(Mn) | M(Mc) | M(Me), 
    M(Nd) | M(Nl) | M(No), 
    M(Sm) | M(Sc) | M(Sk) | M(So), 
    M(Pc) | M(Pd) | M(Ps) | M(Pe) | M(Pi) | M(Pf) | M(Po), 
    M(Zs) | M(Zl) | M(Zp), 
    M(Cc) | M(Cf) | M(Cs) | M(Co) | M(Cn), 
}

lre_ctype_bits

uint8_t const lre_ctype_bits[256]

char_range_s

const uint16_t char_range_s[]

static

Initial value:

= {
    10,
    0x0009, 0x000D + 1,
    0x0020, 0x0020 + 1,
    0x00A0, 0x00A0 + 1,
    0x1680, 0x1680 + 1,
    0x2000, 0x200A + 1,
    
    
    0x2028, 0x2029 + 1,
    0x202F, 0x202F + 1,
    0x205F, 0x205F + 1,
    0x3000, 0x3000 + 1,
    
    0xFEFF, 0xFEFF + 1,
}