Read a file character by character/UTF8: Difference between revisions

Line 727:

precisely one unicode character from a file. If there is a genuine demand for it, I

could easily add this to that file permanently, and document/autoinclude it properly.

<lang Phix>constant INVALID_UTF8 = #FFFD

function get_one_utf8_char(integer fn)

constant INVALID_UTF8 = #FFFD

-- returns INVALID_UTF8 on error, else a string of 1..4 bytes representing one character

object res

function get_one_utf8_char(integer fn)

integer headb, bytes, c

-- returns INVALID_UTF8 on error, else a string of 1..4 bytes representing one character

object res

integer headb, bytes, c

-- headb = first byte of utf-8 character:

headb = getc(fn)

if headb=-1 then return -1 end if

res = ""&headb

-- calculate length of utf-8 character in bytes (1..4):

if headb<0 then bytes = 0 -- (utf-8 starts at #0)

elsif headb<=0b01111111 then bytes = 1 -- 0b_0xxx_xxxx

elsif headb<=0b10111111 then bytes = 0 -- (it's a tail byte)

elsif headb<=0b11011111 then bytes = 2 -- 0b_110x_xxxx

elsif headb<=0b11101111 then bytes = 3 -- 0b_1110_xxxx

elsif headb<=0b11110100 then bytes = 4 -- 0b_1111_0xzz

else bytes = 0 -- (utf-8 ends at #10FFFF)

end if

-- 2..4 bytes encoding (tail range: 0b_1000_0000..0b_1011_1111);

for j=1 to bytes-1 do -- tail bytes are valid?

c = getc(fn)

if c<#80 or c>#BF then

bytes = 0 -- invalid tail byte or eof

exit

end if

res &= c

end for

-- 1 byte encoding (head range: 0b_0000_0000..0b_0111_1111):

if bytes=1 then

c = headb -- UTF-8 = ASCII

-- 2 bytes encoding (head range: 0b_1100_0000..0b_1101_1111):

elsif bytes=2 then

c = and_bits(headb, #1F)*#40 + -- 0b110[7..11] headb

and_bits(res[2], #3F) -- 0b10[1..6] tail

if c>#7FF then ?9/0 end if -- sanity check

if c<#80 then -- long form?

res = INVALID_UTF8

end if

-- 3 bytes encoding (head range: 0b_1110_0000..0b_1110_1111):

elsif bytes=3 then

c = and_bits(headb, #0F)*#1000 + -- 0b1110[13..16] head

and_bits(res[2], #3F)*#40 + -- 0b10[7..12] tail

and_bits(res[3], #3F) -- 0b10[1..6] tail

if c>#FFFF then ?9/0 end if -- sanity check

if c<#800 -- long form?

or (c>=#D800 and c<=#DFFF) then -- utf-16 incompatible

res = INVALID_UTF8

end if

-- 4 bytes encoding (head range: 0b_1111_0000..0b_1111_0111):

elsif bytes=4 then

c = and_bits(headb, #07)*#040000 + -- 0b11110[19..21] head

and_bits(res[2], #3F)*#1000 + -- 0b10[13..18] tail

and_bits(res[3], #3F)*#0040 + -- 0b10[7..12] tail

and_bits(res[4], #3F) -- 0b10[1..6] tail

if c<#10000 -- long form?

or c>#10FFFF then

res = INVALID_UTF8 -- utf-8 ends at #10FFFF

end if

-- bytes = 0; current byte is not encoded correctly:

else

res = INVALID_UTF8

end if

return res

end function

Test code:

-- headb = first byte of utf-8 character:

headb = getc(fn)

if headb=-1 then return -1 end if

res = ""&headb

-- calculate length of utf-8 character in bytes (1..4):

--string utf8 = "aă€⼥" -- (same results as next)

if headb<0 then bytes = 0 -- (utf-8 starts at #0)

string utf8 = utf32_to_utf8({#0061,#0103,#20ac,#2f25})

elsif headb<=0b01111111 then bytes = 1 -- 0b_0xxx_xxxx

printf(1,"length of utf8 is %d bytes\n",length(utf8))

elsif headb<=0b10111111 then bytes = 0 -- (it's a tail byte)

integer fn = open("test.txt","wb")

elsif headb<=0b11011111 then bytes = 2 -- 0b_110x_xxxx

puts(fn,utf8)

elsif headb<=0b11101111 then bytes = 3 -- 0b_1110_xxxx

close(fn)

elsif headb<=0b11110100 then bytes = 4 -- 0b_1111_0xzz

fn = open("test.txt","r")

else bytes = 0 -- (utf-8 ends at #10FFFF)

for i=1 to 5 do

end if

object res = get_one_utf8_char(fn)

if string(res) then

if platform()=LINUX then

printf(1,"char %d (%s) is %d bytes\n",{i,res,length(res)})

else

-- unicode and consoles tricky on windows, so I'm

-- just avoiding that issue altogther (t)here.

printf(1,"char %d is %d bytes\n",{i,length(res)})

end if

elsif res=-1 then

printf(1,"char %d - EOF\n",i)

exit

else

printf(1,"char %d - INVALID_UTF8\n",i)

exit

end if

end for

close(fn)

-- 2..4 bytes encoding (tail range: 0b_1000_0000..0b_1011_1111);

for j=1 to bytes-1 do -- tail bytes are valid?

c = getc(fn)

if c<#80 or c>#BF then

bytes = 0 -- invalid tail byte or eof

exit

end if

res &= c

end for

-- 1 byte encoding (head range: 0b_0000_0000..0b_0111_1111):

if bytes=1 then

c = headb -- UTF-8 = ASCII

-- 2 bytes encoding (head range: 0b_1100_0000..0b_1101_1111):

elsif bytes=2 then

c = and_bits(headb, #1F)*#40 + -- 0b110[7..11] headb

and_bits(res[2], #3F) -- 0b10[1..6] tail

if c>#7FF then ?9/0 end if -- sanity check

if c<#80 then -- long form?

res = INVALID_UTF8

end if

-- 3 bytes encoding (head range: 0b_1110_0000..0b_1110_1111):

elsif bytes=3 then

c = and_bits(headb, #0F)*#1000 + -- 0b1110[13..16] head

and_bits(res[2], #3F)*#40 + -- 0b10[7..12] tail

and_bits(res[3], #3F) -- 0b10[1..6] tail

if c>#FFFF then ?9/0 end if -- sanity check

if c<#800 -- long form?

or (c>=#D800 and c<=#DFFF) then -- utf-16 incompatible

res = INVALID_UTF8

end if

-- 4 bytes encoding (head range: 0b_1111_0000..0b_1111_0111):

elsif bytes=4 then

c = and_bits(headb, #07)*#040000 + -- 0b11110[19..21] head

and_bits(res[2], #3F)*#1000 + -- 0b10[13..18] tail

and_bits(res[3], #3F)*#0040 + -- 0b10[7..12] tail

and_bits(res[4], #3F) -- 0b10[1..6] tail

if c<#10000 -- long form?

or c>#10FFFF then

res = INVALID_UTF8 -- utf-8 ends at #10FFFF

end if

-- bytes = 0; current byte is not encoded correctly:

else

res = INVALID_UTF8

end if

return res

end function</lang>

Test code:

<lang Phix>--string utf8 = "aă€⼥" -- (same results as next)

string utf8 = utf32_to_utf8({#0061,#0103,#20ac,#2f25})

printf(1,"length of utf8 is %d bytes\n",length(utf8))

integer fn = open("test.txt","wb")

puts(fn,utf8)

close(fn)

fn = open("test.txt","r")

for i=1 to 5 do

object res = get_one_utf8_char(fn)

if string(res) then

if platform()=LINUX then

printf(1,"char %d (%s) is %d bytes\n",{i,res,length(res)})

else

-- unicode and consoles tricky on windows, so I'm

-- just avoiding that issue altogther (t)here.

printf(1,"char %d is %d bytes\n",{i,length(res)})

end if

elsif res=-1 then

printf(1,"char %d - EOF\n",i)

exit

else

printf(1,"char %d - INVALID_UTF8\n",i)

exit

end if

end for

close(fn)</lang>

<pre>