Rosetta Code/Find bare lang tags: Difference between revisions

Line 1,461:

=={{header|Phix}}==

Both extra credits. Would probably benefit from excluding <pre></pre> sections first.

<lang Phix>-- demo\rosetta\Find_bare_lang_tags.exw

--

--

-- demo\rosetta\Find_bare_lang_tags.exw

-- Finds/counts no of "<lang>" as opposed to eg "<lang Phix>" tags.

-- ====================================

-- Since downloading all the pages can be very slow, this uses a cache.

--

-- (Uses '&' instead of/as well as 'a', for everyone's sanity..)

constant include_drafts = true,

-- Finds/counts no of "<l&ng>" as opposed to eg "<l&ng Phix>" tags.

sort_by_task = false,

-- Since downloading all the pages can be very slow, this uses a cache.

sort_by_lang = not sort_by_task -- (one or t'other)

--

without js -- (fairly obviously this will never ever run in a browser!)

integer lp = 0

constant include_drafts = true,

procedure progress(string msg, sequence args = {})

sort_by_task = true,

if length(args) then msg = sprintf(msg,args) end if

sort_by_lang = not sort_by_task -- (one or t'other)

integer lm = length(msg)

if lm<lp then msg[$..$] = repeat(' ',lp-lm)&msg[$] end if

include rosettacode_cache.e -- see [[Rosetta_Code/Count_examples#Phix]]

puts(1,msg)

lp = iff(msg[$]='\r'?lm:0)

constant {utf8,ansi} = columnize({{x"E28093","-"},

end procedure

{x"E28099","'"},

{x"C3A8","e"},

include builtins\timedate.e

{x"C3A9","e"},

integer refresh_cache = timedelta(days:=365) -- 0 for always

{x"D09A","K"},

--integer refresh_cache = timedelta(days:=1) -- 0 for always

{x"D09C","M"}})

include builtins\libcurl.e

function utf8_clean(string s)

atom curl = NULL

return substitute_all(s,utf8,ansi)

atom pErrorBuffer

end function

function write_callback(atom pData, integer size, integer nmemb, integer fn)

function multi_lang(sequence s)

integer bytes_written = size * nmemb

-- Convert eg {"Algol","Algol","C","C","C"} to "Algol[2],C[3]"

puts(fn,peek({pData,bytes_written}))

integer i = 1, j = 2

return bytes_written

while i<length(s) do

end function

if s[i]=s[j] then

constant write_cb = call_back({'+', routine_id("write_callback")})

while j<length(s) and s[i]=s[j+1] do j+=1 end while

s[i..j] = {sprintf("%s[%d]",{s[i],j-i+1})}

function open_download(string filename, url)

end if

bool refetch = true

i += 1

if get_file_type("rc_cache")!=FILETYPE_DIRECTORY then

j = i+1

if not create_directory("rc_cache") then

end while

crash("cannot create rc_cache directory")

return join(s,",")

end if

end function

end if

filename = join_path({"rc_cache",filename})

function multi_task(sequence s, tasks)

if file_exists(filename) then

-- Similar to multi_lang() but with task[indexes]

-- use existing file if <= refresh_cache (365 days) old

integer i = 1, j = 2

sequence last_mod = get_file_date(filename) -- (0.8.1+)

while i<=length(s) do

atom delta = timedate_diff(last_mod,date())

integer si = s[i]

refetch = (delta>refresh_cache) or get_file_size(filename)=0

string tsi = html_clean(tasks[si])

else

if j<=length(s) and si=s[j] then

string directory = get_file_path(filename)

while j<length(s) and si=s[j+1] do j+=1 end while

if get_file_type(directory)!=FILETYPE_DIRECTORY then

s[i..j] = {sprintf("%s[%d]",{tsi,j-i+1})}

if not create_directory(directory,make_parent:=true) then

else

crash("cannot create %s directory",{directory})

s[i] = tsi

end if

end if

end if

i += 1

end if

j = i+1

object text

end while

if not refetch then

if length(s)>8 then s[4..-4] = {"..."} end if

text = trim(get_text(filename))

return join(s,",")

refetch = (not sequence(text)) or (length(text)<10)

end function

end if

if refetch then

bool first = true

progress("Downloading %s...\r",{filename})

if curl=NULL then

function find_bare_lang_tags()

curl_global_init()

if get_file_type("rc_cache")!=FILETYPE_DIRECTORY then

curl = curl_easy_init()

if not create_directory("rc_cache") then

pErrorBuffer = allocate(CURL_ERROR_SIZE)

crash("cannot create rc_cache directory")

curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, pErrorBuffer)

end if

curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb)

end if

end if

-- note this lot use web scraping (as cribbed from a similar task) ...

url = substitute(url,"%3A",":")

sequence tasks = dewiki(open_category("Programming_Tasks"))

url = substitute(url,"%2A","*")

if include_drafts then

curl_easy_setopt(curl, CURLOPT_URL, url)

tasks &= dewiki(open_category("Draft_Programming_Tasks"))

integer fn = open(filename,"wb")

tasks = sort(tasks)

if fn=-1 then ?9/0 end if

end if

curl_easy_setopt(curl, CURLOPT_WRITEDATA, fn)

integer blt = find("Rosetta_Code/Find_bare_lang_tags",tasks) -- not this one!

while true do

tasks[blt..blt] = {}

CURLcode res = curl_easy_perform(curl)

if res=CURLE_OK then exit end if

-- ... whereas the individual tasks use the web api instead (3x smaller/faster)

string error = sprintf("%d",res)

integer total_count = 0,

if res=CURLE_COULDNT_RESOLVE_HOST then

lt = length(tasks),

error &= " [CURLE_COULDNT_RESOLVE_HOST]"

kept = 0

end if

progress("%d tasks found\n",{lt})

progress("Error %s downloading file, retry?(Y/N):",{error})

sequence task_langs = {},

if lower(wait_key())!='y' then abort(0) end if

task_counts = iff(sort_by_task?repeat(0,lt):{}),

printf(1,"Y\n")

task_things = iff(sort_by_task?repeat({},lt):{})

end while

for i=1 to length(tasks) do

close(fn)

string ti = tasks[i],

refresh_cache += timedelta(days:=1) -- did I mention it is slow?

url = sprintf("http://rosettacode.org/mw/index.php?title=%s&action=raw",{ti}),

text = get_text(filename)

contents = open_download(ti&".raw",url),

end if

curr

return text

integer count = 0, start = 1, header

end function

while true do

start = match(`<l`&`ang>`,contents,start)

function open_category(string filename)

if start=0 then exit end if

return open_download(filename&".htm","http://rosettacode.org/wiki/Category:"&filename)

-- look backward for the nearest header

end function

header = rmatch(`{`&`{he`&`ader|`,contents,start)

if header=0 then

function dewiki(string s)

curr = "no language"

-- extract tasks from eg `<li><a href="/wiki/100_doors"`

else

sequence tasks = {}

header += length(`{`&`{he`&`ader|`)

integer start = 1, finish = match(`<div class="printfooter">`,s)

curr = utf8_clean(contents[header..match(`<nowiki>}}</nowiki>`,contents,header)-1])

s = s[1..finish-1]

end if

while true do

if sort_by_lang then

start = match("<li><a href=\"/wiki/",s,start)

integer k = find(curr,task_langs)

if start=0 then exit end if

if k=0 then

start += length("<li><a href=\"/wiki/")

task_langs = append(task_langs,curr)

finish = find('"',s,start)

task_things = append(task_things,{i})

string task = s[start..finish-1]

task_counts = append(task_counts,1)

task = substitute_all(task,{"*",":"},{"%2A","%3A"})

else

if task!="Rosetta_Code/Find_bare_lang_tags" then -- not this one!

task_things[k] = append(task_things[k],i)

tasks = append(tasks,task)

task_counts[k] += 1

end if

end if

-- if length(tasks)>10 then exit end if -- (debug aid)

else

start = finish+1

task_things[i] = append(task_things[i],curr)

end while

end if

return tasks

count += 1

end function

start += length(`<l`&`ang>`)

end while

constant {html,ascii} = columnize({{"%2A","*"},

if count!=0 then

{"%3A",":"},

if sort_by_task then

{"%27","'"},

task_counts[i] = count

{"%2B","+"},

end if

{"%22","\""},

kept += 1

{"%E2%80%93","-"},

end if

{"%E2%80%99","'"},

progress("%d tasks kept, %d to go\r",{kept,lt-i})

{"%C3%A8","e"},

total_count += count

{"%C3%A9","e"}})

if get_key()=#1B then progress("escape keyed\n") exit end if

end for

function html_clean(string s)

curl_cleanup()

return substitute_all(s,html,ascii)

progress("%d tasks with bare lang tags\n",{kept})

end function

sequence tags = custom_sort(task_counts,tagset(length(task_counts)))

for i=length(tags) to 1 by -1 do

constant {utf8,ansi} = columnize({{x"E28093","-"},

integer ti = tags[i],

{x"E28099","'"},

tc = task_counts[ti]

{x"C3A8","e"},

if tc=0 then exit end if

{x"C3A9","e"},

if sort_by_task then

{x"D09A","K"},

progress("%s %d (%s)\n",{html_clean(tasks[ti]),tc,multi_lang(task_things[ti])})

{x"D09C","M"}})

else -- (sort_by_count)

progress("%s %d (%s)\n",{task_langs[ti],tc,multi_task(task_things[ti],tasks)})

function utf8_clean(string s)

end if

return substitute_all(s,utf8,ansi)

end for

end function

return total_count

end function

function multi_lang(sequence s)

-- Convert eg {"Algol","Algol","C","C","C"} to "Algol[2],C[3]"

progress("Total: %d\n",{find_bare_lang_tags()})

integer i = 1, j = 2

while i<length(s) do

?"done"

if s[i]=s[j] then

{} = wait_key()

while j<length(s) and s[i]=s[j+1] do j+=1 end while

s[i..j] = {sprintf("%s[%d]",{s[i],j-i+1})}

end if

i += 1

j = i+1

end while

return join(s,",")

end function

function multi_task(sequence s, tasks)

-- Similar to multi_lang() but with task[indexes]

integer i = 1, j = 2

while i<=length(s) do

integer si = s[i]

string tsi = html_clean(tasks[si])

if j<=length(s) and si=s[j] then

while j<length(s) and si=s[j+1] do j+=1 end while

s[i..j] = {sprintf("%s[%d]",{tsi,j-i+1})}

else

s[i] = tsi

end if

i += 1

j = i+1

end while

if length(s)>8 then s[4..-4] = {"..."} end if

return join(s,",")

end function

function find_bare_lang_tags()

-- note this lot use web scraping (as cribbed from a similar task) ...

sequence tasks = dewiki(open_category("Programming_Tasks"))

if include_drafts then

tasks &= dewiki(open_category("Draft_Programming_Tasks"))

tasks = sort(tasks)

end if

-- ... whereas the individual tasks use the web api instead (3x smaller/faster)

integer total_count = 0,

lt = length(tasks),

kept = 0

progress("%d tasks found\n",{lt})

sequence task_langs = {},

task_counts = iff(sort_by_task?repeat(0,lt):{}),

task_things = iff(sort_by_task?repeat({},lt):{})

for i=1 to length(tasks) do

string ti = tasks[i],

url = sprintf("http://rosettacode.org/mw/index.php?title=%s&action=raw",{ti}),

contents = open_download(ti&".raw",url),

this

integer count = 0, start = 1, header

while true do

start = match(`<lang>`,contents,start)

if start=0 then exit end if

-- look backward for the nearest header

header = rmatch(`{{header|`,contents,start)

if header=0 then

-- this = ""

this = "no language"

else

header += length(`{{header|`)

this = utf8_clean(contents[header..match(`}}`,contents,header)-1])

end if

if sort_by_lang then

integer k = find(this,task_langs)

if k=0 then

task_langs = append(task_langs,this)

task_things = append(task_things,{i})

task_counts = append(task_counts,1)

else

task_things[k] = append(task_things[k],i)

task_counts[k] += 1

end if

else

task_things[i] = append(task_things[i],this)

end if

count += 1

start += length(`<lang>`)

end while

if count!=0 then

if sort_by_task then

task_counts[i] = count

end if

kept += 1

end if

progress("%d tasks kept, %d to go\r",{kept,lt-i})

total_count += count

if get_key()=#1B then progress("escape keyed\n") exit end if

end for

if curl!=NULL then

curl_easy_cleanup(curl)

free(pErrorBuffer)

curl = NULL

pErrorBuffer = NULL

end if

progress("%d tasks with bare lang tags\n",{kept})

sequence tags = custom_sort(task_counts,tagset(length(task_counts)))

for i=length(tags) to 1 by -1 do

integer ti = tags[i],

tc = task_counts[ti]

if tc=0 then exit end if

--if tc>5 then

if sort_by_task then

progress("%s %d (%s)\n",{html_clean(tasks[ti]),tc,multi_lang(task_things[ti])})

else -- (sort_by_count)

progress("%s %d (%s)\n",{task_langs[ti],tc,multi_task(task_things[ti],tasks)})

end if

--end if

end for

return total_count

end function

progress("Total: %d\n",{find_bare_lang_tags()})</lang>

as of 26/7/19, sort_by_task: