mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-01-28 17:54:16 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

WL#916: Unicode collations for some languages

Browse source
This commit is contained in:
unknown 2004-06-08 17:56:15 +05:00
parent 19480ed616
commit 8ab01b335d
17 changed files with 214 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
include/m_ctype.h
View file

@ -200,6 +200,7 @@ typedef struct charset_info_st
const char *csname;
const char *name;
const char *comment;
const char *tailoring;
uchar *ctype;
uchar *to_lower;
uchar *to_upper;

210
mysys/charset.c
View file

@ -624,6 +624,123 @@ err:
#ifdef HAVE_CHARSET_ucs2
typedef struct my_tailoring_st
{
uint number;
const char *name;
const char *tailoring;
} my_tailoring;
static my_tailoring tailoring[]=
{
{
0, "icelandic",
/*
Some sources treat LETTER A WITH DIARESIS (00E4,00C4)
secondary greater than LETTER AE (00E6,00C6).
http://www.evertype.com/alphabets/icelandic.pdf
http://developer.mimer.com/collations/charts/icelandic.htm
Other sources do not provide any special rules
for LETTER A WITH DIARESIS:
http://www.omniglot.com/writing/icelandic.htm
http://en.wikipedia.org/wiki/Icelandic_alphabet
http://oss.software.ibm.com/icu/charts/collation/is.html
Let's go the first way.
*/
"& A < \\u00E1 <<< \\u00C1 "
"& D < \\u00F0 <<< \\u00D0 "
"& E < \\u00E9 <<< \\u00C9 "
"& I < \\u00ED <<< \\u00CD "
"& O < \\u00F3 <<< \\u00D3 "
"& U < \\u00FA <<< \\u00DA "
"& Y < \\u00FD <<< \\u00DD "
"& Z < \\u00FE <<< \\u00DE "
"< \\u00E6 <<< \\u00C6 << \\u00E4 <<< \\u00C4 "
"< \\u00F6 <<< \\u00D6 << \\u00F8 <<< \\u00D8 "
"< \\u00E5 <<< \\u00C5 "
},
{
1, "latvian",
/*
Some sources treat I and Y primary different.
Other sources treat I and Y the same on primary level.
We'll go the first way.
*/
"& C < \\u010D <<< \\u010C "
"& G < \\u0123 <<< \\u0122 "
"& I < \\u0079 <<< \\u0059 "
"& K < \\u0137 <<< \\u0136 "
"& L < \\u013C <<< \\u013B "
"& N < \\u0146 <<< \\u0145 "
"& R < \\u0157 <<< \\u0156 "
"& S < \\u0161 <<< \\u0160 "
"& Z < \\u017E <<< \\u017D "
},
{
2, "romanian",
"& A < \\u0103 <<< \\u0102 < \\u00E2 <<< \\u00C2 "
"& I < \\u00EE <<< \\u00CE "
"& S < \\u0219 <<< \\u0218 << \\u015F <<< \\u015E "
"& T < \\u021B <<< \\u021A << \\u0163 <<< \\u0162 "
},
{
3, "slovenian",
"& C < \\u010D <<< \\u010C "
"& S < \\u0161 <<< \\u0160 "
"& Z < \\u017E <<< \\u017D "
},
{
4, "polish",
"& A < \\u0105 <<< \\u0104 "
"& C < \\u0107 <<< \\u0106 "
"& E < \\u0119 <<< \\u0118 "
"& L < \\u0142 <<< \\u0141 "
"& N < \\u0144 <<< \\u0143 "
"& O < \\u00F3 <<< \\u00D3 "
"& S < \\u015B <<< \\u015A "
"& Z < \\u017A <<< \\u017B "
},
{
5, "estonian",
"& S < \\u0161 <<< \\u0160 "
" < \\u007A <<< \\u005A "
" < \\u017E <<< \\u017D "
"& W < \\u00F5 <<< \\u00D5 "
"< \\u00E4 <<< \\u00C4 "
"< \\u00F6 <<< \\u00D6 "
"< \\u00FC <<< \\u00DC "
},
{
6, "spanish",
"& N < \\u00F1 <<< \\u00D1 "
},
{
7, "swedish",
/*
Some sources treat V and W as similar on primary level.
We'll treat V and W as different on primary level.
*/
"& Y <<\\u00FC <<< \\u00DC "
"& Z < \\u00E5 <<< \\u00C5 "
"< \\u00E4 <<< \\u00C4 << \\u00E6 <<< \\u00C6 "
"< \\u00F6 <<< \\u00D6 << \\u00F8 <<< \\u00D8 "
},
{
8, "turkish",
"& C < \\u00E7 <<< \\u00C7 "
"& G < \\u011F <<< \\u011E "
"& H < \\u0131 <<< \\u0049 "
"& O < \\u00F6 <<< \\u00D6 "
"& S < \\u015F <<< \\u015E "
"& U < \\u00FC <<< \\u00DC "
},
{
0, NULL, NULL
}
};
#define MY_MAX_COLL_RULE 64
/*
@ -643,7 +760,7 @@ err:
default weights.
*/
static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from)
static my_bool create_tailoring(CHARSET_INFO *cs)
{
MY_COLL_RULE rule[MY_MAX_COLL_RULE];
char errstr[128];
@ -652,32 +769,14 @@ static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from)
const uchar *deflengths= my_charset_ucs2_general_uca.sort_order;
uint16 **defweights= my_charset_ucs2_general_uca.sort_order_big;
int rc, i;
to->number= from->number ? from->number : to->number;
if (from->csname)
if (!(to->csname= my_once_strdup(from->csname,MYF(MY_WME))))
goto err;
if (from->name)
if (!(to->name= my_once_strdup(from->name,MYF(MY_WME))))
goto err;
if (from->comment)
if (!(to->comment= my_once_strdup(from->comment,MYF(MY_WME))))
goto err;
to->strxfrm_multiply= my_charset_ucs2_general_uca.strxfrm_multiply;
to->min_sort_char= my_charset_ucs2_general_uca.min_sort_char;
to->max_sort_char= my_charset_ucs2_general_uca.max_sort_char;
to->mbminlen= 2;
to->mbmaxlen= 2;
if (!cs->tailoring)
return 1;
/* Parse ICU Collation Customization expression */
if ((rc= my_coll_rule_parse(rule, MY_MAX_COLL_RULE,
from->sort_order,
from->sort_order + strlen(from->sort_order),
cs->tailoring,
cs->tailoring + strlen(cs->tailoring),
errstr, sizeof(errstr))) <= 0)
{
/*
@ -687,13 +786,12 @@ static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from)
return 1;
}
if (!(newweights= (uint16**) my_once_alloc(256*sizeof(uint16*),MYF(MY_WME))))
goto err;
return 1;
bzero(newweights, 256*sizeof(uint16*));
if (!(newlengths= (uchar*) my_once_memdup(deflengths,256,MYF(MY_WME))))
goto err;
return 1;
/*
Calculate maximum lenghts for the pages
@ -720,7 +818,7 @@ static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from)
uint size= 256*newlengths[pagec]*sizeof(uint16);
if (!(newweights[pagec]= (uint16*) my_once_alloc(size,MYF(MY_WME))))
goto err;
return 1;
bzero((void*) newweights[pagec], size);
for (chc=0 ; chc < 256; chc++)
@ -749,10 +847,41 @@ static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from)
if (!newweights[i])
newweights[i]= defweights[i];
to->sort_order= newlengths;
to->sort_order_big= newweights;
cs->sort_order= newlengths;
cs->sort_order_big= newweights;
return 0;
}
static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from)
{
to->number= from->number ? from->number : to->number;
if (from->csname)
if (!(to->csname= my_once_strdup(from->csname,MYF(MY_WME))))
goto err;
if (from->name)
if (!(to->name= my_once_strdup(from->name,MYF(MY_WME))))
goto err;
if (from->comment)
if (!(to->comment= my_once_strdup(from->comment,MYF(MY_WME))))
goto err;
if (from->tailoring)
if (!(to->tailoring= my_once_strdup(from->tailoring,MYF(MY_WME))))
goto err;
to->strxfrm_multiply= my_charset_ucs2_general_uca.strxfrm_multiply;
to->min_sort_char= my_charset_ucs2_general_uca.min_sort_char;
to->max_sort_char= my_charset_ucs2_general_uca.max_sort_char;
to->mbminlen= 2;
to->mbmaxlen= 2;
return create_tailoring(to);
err:
return 1;
@ -848,6 +977,24 @@ static int add_collation(CHARSET_INFO *cs)
return MY_XML_OK;
}
#ifdef HAVE_CHARSET_ucs2
static my_bool init_uca_charsets()
{
my_tailoring *t;
CHARSET_INFO cs= my_charset_ucs2_general_uca;
cs.state= MY_CS_STRNXFRM|MY_CS_UNICODE|MY_CS_NONTEXT;
char name[64];
for (t= tailoring; t->tailoring; t++)
{
cs.number= 128 + t->number;
cs.tailoring= t->tailoring;
cs.name= name;
sprintf(name, "ucs2_%s_ci", t->name);
add_collation(&cs);
}
return 0;
}
#endif
#define MY_MAX_ALLOWED_BUF 1024*1024
#define MY_CHARSET_INDEX "Index.xml"
@ -947,6 +1094,9 @@ static my_bool init_available_charsets(myf myflags)
bzero(&all_charsets,sizeof(all_charsets));
init_compiled_charsets(myflags);
#ifdef HAVE_CHARSET_ucs2
init_uca_charsets();
#endif
/* Copy compiled charsets */
for (cs=all_charsets;

2
strings/ctype-big5.c
View file

@ -6313,6 +6313,7 @@ CHARSET_INFO my_charset_big5_chinese_ci=
"big5", /* cs name */
"big5_chinese_ci", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_big5,
to_lower_big5,
to_upper_big5,
@ -6339,6 +6340,7 @@ CHARSET_INFO my_charset_big5_bin=
"big5", /* cs name */
"big5_bin", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_big5,
to_lower_big5,
to_upper_big5,

1
strings/ctype-bin.c
View file

@ -378,6 +378,7 @@ CHARSET_INFO my_charset_bin =
"binary", /* cs name */
"binary", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_bin, /* ctype */
bin_char_array, /* to_lower */
bin_char_array, /* to_upper */

1
strings/ctype-czech.c
View file

@ -589,6 +589,7 @@ CHARSET_INFO my_charset_latin2_czech_ci =
"latin2", /* cs name */
"latin2_czech_cs", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_czech,
to_lower_czech,
to_upper_czech,

2
strings/ctype-euc_kr.c
View file

@ -8681,6 +8681,7 @@ CHARSET_INFO my_charset_euckr_korean_ci=
"euckr", /* cs name */
"euckr_korean_ci", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_euc_kr,
to_lower_euc_kr,
to_upper_euc_kr,
@ -8707,6 +8708,7 @@ CHARSET_INFO my_charset_euckr_bin=
"euckr", /* cs name */
"euckr_bin", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_euc_kr,
to_lower_euc_kr,
to_upper_euc_kr,

2
strings/ctype-gb2312.c
View file

@ -5732,6 +5732,7 @@ CHARSET_INFO my_charset_gb2312_chinese_ci=
"gb2312", /* cs name */
"gb2312_chinese_ci",/* name */
"", /* comment */
NULL, /* tailoring */
ctype_gb2312,
to_lower_gb2312,
to_upper_gb2312,
@ -5757,6 +5758,7 @@ CHARSET_INFO my_charset_gb2312_bin=
"gb2312", /* cs name */
"gb2312_bin", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_gb2312,
to_lower_gb2312,
to_upper_gb2312,

2
strings/ctype-gbk.c
View file

@ -9962,6 +9962,7 @@ CHARSET_INFO my_charset_gbk_chinese_ci=
"gbk", /* cs name */
"gbk_chinese_ci", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_gbk,
to_lower_gbk,
to_upper_gbk,
@ -9987,6 +9988,7 @@ CHARSET_INFO my_charset_gbk_bin=
"gbk", /* cs name */
"gbk_bin", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_gbk,
to_lower_gbk,
to_upper_gbk,

3
strings/ctype-latin1.c
View file

@ -412,6 +412,7 @@ CHARSET_INFO my_charset_latin1=
"latin1", /* cs name */
"latin1_swedish_ci", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_latin1,
to_lower_latin1,
to_upper_latin1,
@ -690,6 +691,7 @@ CHARSET_INFO my_charset_latin1_german2_ci=
"latin1", /* cs name */
"latin1_german2_ci", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_latin1,
to_lower_latin1,
to_upper_latin1,
@ -715,6 +717,7 @@ CHARSET_INFO my_charset_latin1_bin=
"latin1", /* cs name */
"latin1_bin", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_latin1,
to_lower_latin1,
to_upper_latin1,

2
strings/ctype-sjis.c
View file

@ -4579,6 +4579,7 @@ CHARSET_INFO my_charset_sjis_japanese_ci=
"sjis", /* cs name */
"sjis_japanese_ci", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_sjis,
to_lower_sjis,
to_upper_sjis,
@ -4604,6 +4605,7 @@ CHARSET_INFO my_charset_sjis_bin=
"sjis", /* cs name */
"sjis_bin", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_sjis,
to_lower_sjis,
to_upper_sjis,

2
strings/ctype-tis620.c
View file

@ -951,6 +951,7 @@ CHARSET_INFO my_charset_tis620_thai_ci=
"tis620", /* cs name */
"tis620_thai_ci", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_tis620,
to_lower_tis620,
to_upper_tis620,
@ -976,6 +977,7 @@ CHARSET_INFO my_charset_tis620_bin=
"tis620", /* cs name */
"tis620_bin", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_tis620,
to_lower_tis620,
to_upper_tis620,

1
strings/ctype-uca.c
View file

@ -7055,6 +7055,7 @@ CHARSET_INFO my_charset_ucs2_general_uca=
"ucs2", /* cs name */
"ucs2_general_uca", /* name */
"", /* comment */
NULL, /* tailoring */
NULL, /* ctype */
NULL, /* to_lower */
NULL, /* to_upper */

2
strings/ctype-ucs2.c
View file

@ -1431,6 +1431,7 @@ CHARSET_INFO my_charset_ucs2_general_ci=
"ucs2", /* cs name */
"ucs2_general_ci", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_ucs2, /* ctype */
to_lower_ucs2, /* to_lower */
to_upper_ucs2, /* to_upper */
@ -1456,6 +1457,7 @@ CHARSET_INFO my_charset_ucs2_bin=
"ucs2", /* cs name */
"ucs2_bin", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_ucs2, /* ctype */
to_lower_ucs2, /* to_lower */
to_upper_ucs2, /* to_upper */

2
strings/ctype-ujis.c
View file

@ -8468,6 +8468,7 @@ CHARSET_INFO my_charset_ujis_japanese_ci=
"ujis", /* cs name */
"ujis_japanese_ci", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_ujis,
to_lower_ujis,
to_upper_ujis,
@ -8494,6 +8495,7 @@ CHARSET_INFO my_charset_ujis_bin=
"ujis", /* cs name */
"ujis_bin", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_ujis,
to_lower_ujis,
to_upper_ujis,

2
strings/ctype-utf8.c
View file

@ -2090,6 +2090,7 @@ CHARSET_INFO my_charset_utf8_general_ci=
"utf8", /* cs name */
"utf8_general_ci", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_utf8, /* ctype */
to_lower_utf8, /* to_lower */
to_upper_utf8, /* to_upper */
@ -2116,6 +2117,7 @@ CHARSET_INFO my_charset_utf8_bin=
"utf8", /* cs name */
"utf8_bin", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_utf8, /* ctype */
to_lower_utf8, /* to_lower */
to_upper_utf8, /* to_upper */

1
strings/ctype-win1250ch.c
View file

@ -623,6 +623,7 @@ CHARSET_INFO my_charset_cp1250_czech_ci =
"cp1250", /* cs name */
"cp1250_czech_cs", /* name */
"", /* comment */
NULL, /* tailoring */
ctype_win1250ch,
to_lower_win1250ch,
to_upper_win1250ch,

14
strings/ctype.c
View file

@ -124,6 +124,7 @@ static struct my_cs_file_section_st * cs_file_sec(const char *attr, uint len)
}
#define MY_CS_CSDESCR_SIZE 64
#define MY_CS_TAILORING_SIZE 128
typedef struct my_cs_file_info
{
@ -135,7 +136,8 @@ typedef struct my_cs_file_info
uchar sort_order[MY_CS_SORT_ORDER_TABLE_SIZE];
uint16 tab_to_uni[MY_CS_TO_UNI_TABLE_SIZE];
char comment[MY_CS_CSDESCR_SIZE];
size_t sort_order_length;
char tailoring[MY_CS_TAILORING_SIZE];
size_t tailoring_length;
CHARSET_INFO cs;
int (*add_collation)(CHARSET_INFO *cs);
} MY_CHARSET_LOADER;
@ -186,7 +188,7 @@ static int cs_enter(MY_XML_PARSER *st,const char *attr, uint len)
bzero(&i->cs,sizeof(i->cs));
if (s && (s->state == _CS_COLLATION))
i->sort_order_length= 0;
i->tailoring_length= 0;
return MY_XML_OK;
}
@ -283,12 +285,12 @@ static int cs_value(MY_XML_PARSER *st,const char *attr, uint len)
*/
char arg[16];
const char *cmd[]= {"&","<","<<","<<<"};
i->cs.sort_order= i->sort_order;
i->cs.tailoring= i->tailoring;
mstr(arg,attr,len,sizeof(arg)-1);
if (i->sort_order_length + 20 < sizeof(i->sort_order))
if (i->tailoring_length + 20 < sizeof(i->tailoring))
{
char *dst= i->sort_order_length + i->sort_order;
i->sort_order_length+= sprintf(dst," %s %s",cmd[state-_CS_RESET],arg);
char *dst= i->tailoring_length + i->tailoring;
i->tailoring_length+= sprintf(dst," %s %s",cmd[state-_CS_RESET],arg);
}
}
}