[CPyUG:82453] 如何判断一个中文字符串的编码 ？

--000e0cd1481a173dbe04663ae589
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: quoted-printable

=B8=F7=CE=BB=D0=D6=B5=DC=A3=BA

=D3=D0=C3=BB=D3=D0=D7=D4=B6=AF=C5=D0=B6=CF=D2=BB=B6=CE=D6=D0=CE=C4=B4=FA=C2=
=EB=B1=E0=C2=EB=B5=C4=CB=E3=B7=A8=A3=AC=D2=AA=B8=DF=BF=C9=BF=BF=D0=D4=B5=C4
=CA=D4=B9=FD=C1=BD=BD=DA=B4=FA=C2=EB=A3=AC=B6=BC=B2=BB=CA=C7=BA=DC=B3=C9=B9=
=A6.


1)=B4=FA=C2=EB=A3=B1=A3=BA=C0=B4=D7=D4modou=D0=D6
def utf8Detect(text):
"""Detect if a string is utf-8 encoding"""
lastch=3D0
begin=3D0
BOM=3DTrue
BOMchs=3D(0xEF, 0xBB, 0xBF)
good=3D0
bad=3D0
for char in text:
ch=3Dord(char)
if begin<3:
BOM=3D(BOMchs[begin]=3D=3Dch) and BOM
begin +=3D 1
continue
if (begin=3D=3D4) and (BOM=3D=3DTrue):
break;
if (ch & 0xC0) =3D=3D 0x80:
if (lastch & 0xC0) =3D=3D 0xC0:
good +=3D 1
elif (lastch &0x80) =3D=3D 0:
bad +=3D 1
elif (lastch & 0xC0) =3D=3D 0xC0:
bad +=3D 1
lastch =3D ch
if (((begin =3D=3D 4) and (BOM =3D=3D True)) or
(good >=3D bad)):
return True
else:
return False

2)=B4=FA=C2=EB=A3=B2
def zh2uni(s):
"""Auto converter encodings to unicode

Chinese (PRC): gb2312 gbk gb18030 big5hkscs hz
Chinese (ROC): big5 cp950
Japanese: cp932 shift-jis shift-jisx0213 shift-jis-2004 euc-jp
euc-jisx0213 euc-jis-2004 iso-2022-jp iso-2022-jp-1
iso-2022-jp-2 iso-2022-jp-3 iso-2022-jp-ext iso-2022-jp-2004
Korean: cp949 euc-kr johab iso-2022-kr
Unicode: utf-7 utf-8 utf-16 utf-16-be utf-16-le
-------------------------------------------------------
It will test utf8,gbk,big5,jp,kr to converter"""
if type(s)=3D=3Dunicode:
return 'utf-8',s
for c in ('gb18030','utf-8','big5','jp','utf-16','hz','euc_kr'):
try:
return c,s.decode(c)
except:
pass
return 'unk',s

--~--~---------~--~----~------------~-------~--~----~
=C0=B4=D7=D4: `python-cn`:CPyUG ~ =BB=AA=F2=FE=D3=C3=BB=A7=D7=E9 | =B7=A2=
=D1=D4:python-cn-/[email protected]
=CD=CB=B6=A9: http://tinyurl.com/45a9tb /=D5=EB=B6=D4163/qq=D3=CA=CF=E4:htt=
p://tinyurl.com/4dg6hc
=CF=EA=C7=E9: https://groups.google.com/group/python-cn
=D1=CF=D5=FD: =C0=ED=BD=E2=C1=D0=B1=ED! =D6=C7=BB=DB=CC=E1=CE=CA! http://wi=
ki.woodpecker.org.cn/moin/AskForHelp
-~----------~----~----~----~------~----~------~--~---


--000e0cd1481a173dbe04663ae589
Content-Type: text/html; charset=GB2312
Content-Transfer-Encoding: quoted-printable

=B8=F7=CE=BB=D0=D6=B5=DC=A3=BA<br><br>=D3=D0=C3=BB=D3=D0=D7=D4=B6=AF=C5=D0=
=B6=CF=D2=BB=B6=CE=D6=D0=CE=C4=B4=FA=C2=EB=B1=E0=C2=EB=B5=C4=CB=E3=B7=A8=A3=
=AC=D2=AA=B8=DF=BF=C9=BF=BF=D0=D4=B5=C4<br>=CA=D4=B9=FD=C1=BD=BD=DA=B4=FA=
=C2=EB=A3=AC=B6=BC=B2=BB=CA=C7=BA=DC=B3=C9=B9=A6.<br><br><br>1)=B4=FA=C2=EB=
=A3=B1=A3=BA=C0=B4=D7=D4modou=D0=D6<br>def utf8Detect(text):<br>    &quot;&=
quot;&quot;Detect if a string is utf-8 encoding&quot;&quot;&quot;<br>    la=
stch=3D0<br>    begin=3D0<br>
    BOM=3DTrue<br>    BOMchs=3D(0xEF, 0xBB, 0xBF)<br>    good=3D0<br>    ba=
d=3D0<br>    for char in text:<br>        ch=3Dord(char)<br>        if begi=
n&lt;3:<br>            BOM=3D(BOMchs[begin]=3D=3Dch) and BOM<br>           =
 begin +=3D 1<br>
            continue<br>        if (begin=3D=3D4) and (BOM=3D=3DTrue):<br> =
           break;<br>        if (ch &amp; 0xC0) =3D=3D 0x80:<br>           =
 if (lastch &amp; 0xC0) =3D=3D 0xC0:<br>                good +=3D 1<br>    =
        elif (lastch &amp;0x80) =3D=3D 0:<br>
                bad +=3D 1<br>        elif (lastch &amp; 0xC0) =3D=3D 0xC0:=
<br>            bad +=3D 1<br>                lastch =3D ch<br>            =
    if (((begin =3D=3D 4) and (BOM =3D=3D True)) or<br>        (good &gt;=
=3D bad)):<br>        return True<br>
    else:<br>        return False<br><br>2)=B4=FA=C2=EB=A3=B2<br>def zh2uni=
(s): <br>    &quot;&quot;&quot;Auto converter encodings to unicode <br><br>=
    Chinese (PRC): gb2312 gbk gb18030 big5hkscs hz<br>    Chinese (ROC): bi=
g5 cp950<br>
    Japanese: cp932 shift-jis shift-jisx0213 shift-jis-2004 euc-jp<br>     =
         euc-jisx0213 euc-jis-2004 iso-2022-jp iso-2022-jp-1 <br>          =
    iso-2022-jp-2 iso-2022-jp-3 iso-2022-jp-ext iso-2022-jp-2004 <br>    Ko=
rean: cp949 euc-kr johab iso-2022-kr<br>
    Unicode: utf-7 utf-8 utf-16 utf-16-be utf-16-le<br>    ----------------=
---------------------------------------<br>    It will test utf8,gbk,big5,j=
p,kr to converter&quot;&quot;&quot; <br>    if type(s)=3D=3Dunicode:<br>   =
     return &#39;utf-8&#39;,s<br>
    for c in (&#39;gb18030&#39;,&#39;utf-8&#39;,&#39;big5&#39;,&#39;jp&#39;=
,&#39;utf-16&#39;,&#39;hz&#39;,&#39;euc_kr&#39;): <br>        try:<br>     =
       return c,s.decode(c)<br>        except:<br>            pass<br>    r=
eturn &#39;unk&#39;,s<br>
<br><br><br><br>
--~--~---------~--~----~------------~-------~--~----~<br>
=C0=B4=D7=D4: `python-cn`:CPyUG ~ =BB=AA=F2=FE=D3=C3=BB=A7=D7=E9 | =B7=A2=
=D1=D4:python-cn-/[email protected]
 <br> =CD=CB=B6=A9: http://tinyurl.com/45a9tb /=D5=EB=B6=D4163/qq=D3=CA=CF=
=E4:http://tinyurl.com/4dg6hc
 <br> =CF=EA=C7=E9: https://groups.google.com/group/python-cn
 <br> =D1=CF=D5=FD: =C0=ED=BD=E2=C1=D0=B1=ED! =D6=C7=BB=DB=CC=E1=CE=CA! htt=
p://wiki.woodpecker.org.cn/moin/AskForHelp<br>
-~----------~----~----~----~------~----~------~--~---<br>
<br>

--000e0cd1481a173dbe04663ae589--