Rosetta Code/Tasks without examples: Difference between revisions

Line 559:

Output similar to zkl, I assume the first four constants are self-explanatory.

<lang Phix>-- demo\rosetta\Tasks_without_examples.exw

-- demo\rosetta\Tasks_without_examples.exw

constant output_html = true,

without js -- (libcurl, file i/o, peek, progress..)

include_drafts = true,

constant output_html = false,

summary = false,

include_drafts = true,

notlang = "Phix" -- "" for all

summary = true,

notlang = "Phix" -- or "" (ie a zero length string) for all

include builtins\timedate.e

integer refresh_cache = timedelta(days:=30) -- 0 for always

include rosettacode_cache.e -- see [[Rosetta_Code/Count_examples#Phix]]

include builtins\libcurl.e

function extract_tasks()

atom curl = NULL

if get_file_type("rc_cache")!=FILETYPE_DIRECTORY then

atom pErrorBuffer

if not create_directory("rc_cache") then

crash("cannot create rc_cache directory")

function write_callback(atom pData, integer size, integer nmemb, integer fn)

end if

integer bytes_written = size * nmemb

end if

puts(fn,peek({pData,bytes_written}))

-- extract tasks from eg `<li><a href="/wiki/100_doors"`

return bytes_written

sequence tasks = dewiki(open_category("Programming_Tasks"))

end function

if include_drafts then

constant write_cb = call_back({'+', routine_id("write_callback")})

tasks &= dewiki(open_category("Draft_Programming_Tasks"))

end if

function open_download(string filename, url)

if length(notlang) then

bool refetch = true

-- filter already done in specified language

if get_file_type("rc_cache")!=FILETYPE_DIRECTORY then

string langurl = "http://rosettacode.org/wiki/Category:"&notlang

if not create_directory("rc_cache") then

sequence done = dewiki(open_download(notlang&".htm",langurl))

crash("cannot create rc_cache directory")

integer k = 0

end if

for i=1 to length(tasks) do

end if

if not find(tasks[i],done) then

filename = join_path({"rc_cache",filename})

k += 1

if file_exists(filename) then

tasks[k] = tasks[i]

-- use existing file if <= refresh_cache (30+ days) old

end if

sequence last_mod = get_file_date(filename) -- (0.8.1+)

end for

atom delta = timedate_diff(last_mod,date())

tasks = tasks[1..k]

refetch = (delta>refresh_cache)

done = {}

else

end if

string directory = get_file_path(filename)

if not summary then

if get_file_type(directory)!=FILETYPE_DIRECTORY then

-- replace with contents

if not create_directory(directory,make_parent:=true) then

for i=1 to length(tasks) do

crash("cannot create %s directory",{directory})

string ti = tasks[i],

end if

url = sprintf("http://rosettacode.org/wiki/%s",{ti}),

end if

contents = open_download(ti&".htm",url)

end if

integer start = match(`</div>`,contents,match(`<div class="infobox"`,contents))+length(`</div>`)

if refetch then

integer finish = match(`<div id="toc"`,contents,start)-1

printf(1,"Downloading %s...\n",{filename})

-- ... but draft tasks with too few languages have no toc:

if curl=NULL then

if finish=-1 then finish = match(`<h2>`,contents,start)-1 end if

curl_global_init()

-- ... and if no languages at all, use the footer:

curl = curl_easy_init()

if finish=-1 then finish = match(`</div><div class="printfooter">`,contents,start)-1 end if

pErrorBuffer = allocate(CURL_ERROR_SIZE)

if finish=-1 then ?9/0 end if

curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, pErrorBuffer)

contents = contents[start..finish]

curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb)

ti = substitute(ti,"_"," ")

end if

if not match(""&ti&"",contents) then

url = substitute(url,"%3A",":")

-- (ps: I refuse to panic over the occasional replicated header...)

url = substitute(url,"%2A","*")

contents = sprintf("<h3>%s</h3>%s",{ti,contents})

curl_easy_setopt(curl, CURLOPT_URL, url)

end if

integer fn = open(filename,"wb")

tasks[i] = contents

if fn=-1 then ?9/0 end if

if get_key()=#1B then exit end if

curl_easy_setopt(curl, CURLOPT_WRITEDATA, fn)

end for

CURLcode res = curl_easy_perform(curl)

end if

if res!=CURLE_OK then

curl_cleanup()

string error = sprintf("%d",res)

return tasks

if res=CURLE_COULDNT_RESOLVE_HOST then

end function

error &= " [CURLE_COULDNT_RESOLVE_HOST]"

end if

constant html_header = """

printf(1, "Error %s downloading file\n", error)

<!DOCTYPE html>

{} = wait_key()

abort(0)

<head>

end if

close(fn)

refresh_cache += timedelta(days:=1) -- did I mention it is slow?

<title>Rosettacode Tasks without examples</title>

end if

</head>

return get_text(filename)

<body>

end function

<h2>Rosettacode Tasks without examples</h2>

Generated %s, %d entries

function open_category(string filename)

return open_download(filename&".htm","http://rosettacode.org/wiki/Category:"&filename)

""",

end function

html_footer = """

</body>

function dewiki(string s)

</html>

sequence tasks = {}

"""

integer start = 1, finish = match(`<div class="printfooter">`,s)

s = s[1..finish-1]

sequence results = extract_tasks()

while true do

if output_html then

start = match("<li><a href=\"/wiki/",s,start)

integer fn = open("Tasks_Without_Examples.html","w")

if start=0 then exit end if

printf(fn,html_header,{format_timedate(date()),length(results)})

start += length("<li><a href=\"/wiki/")

for i=1 to length(results) do

finish = find('"',s,start)

printf(fn,"%s \n",html_clean(results[i]))

string task = s[start..finish-1]

end for

task = substitute(task,"*","%2A")

puts(fn,html_footer)

task = substitute(task,":","%3A")

close(fn)

tasks = append(tasks,task)

else

start = finish+1

progress("")

end while

for i=1 to length(results) do

return tasks

printf(1,"%s\n",html_clean(results[i]))

end function

end for

end if

function extract_tasks()

printf(1,"%d tasks\n",length(results))

-- extract tasks from eg `<li><a href="/wiki/100_doors"`

sequence tasks = dewiki(open_category("Programming_Tasks"))

?"done"

if include_drafts then

{} = wait_key()

tasks &= dewiki(open_category("Draft_Programming_Tasks"))

end if

if length(notlang) then

-- filter already done in specified language

string langurl = "http://rosettacode.org/wiki/Category:"&notlang

sequence done = dewiki(open_download(notlang&".htm",langurl))

integer k = 0

for i=1 to length(tasks) do

if not find(tasks[i],done) then

k += 1

tasks[k] = tasks[i]

end if

end for

tasks = tasks[1..k]

done = {}

end if

if not summary then

-- replace with contents

for i=1 to length(tasks) do

string ti = tasks[i],

url = sprintf("http://rosettacode.org/wiki/%s",{ti}),

contents = open_download(ti&".htm",url)

integer start = match(`</div>`,contents,match(`<div class="infobox"`,contents))+length(`</div>`)

integer finish = match(`<div id="toc"`,contents,start)-1

-- ... but draft tasks with too few languages have no toc:

if finish=-1 then finish = match(`<h2>`,contents,start)-1 end if

-- ... and if no languages at all, use the footer:

if finish=-1 then finish = match(`</div><div class="printfooter">`,contents,start)-1 end if

if finish=-1 then ?9/0 end if

contents = contents[start..finish]

ti = substitute(ti,"_"," ")

if not match(""&ti&"",contents) then

-- (ps: I refuse to panic over the occasional replicated header...)

contents = sprintf("<h3>%s</h3>%s",{ti,contents})

end if

tasks[i] = contents

if get_key()=#1B then exit end if

end for

end if

if curl!=NULL then

curl_easy_cleanup(curl)

free(pErrorBuffer)

curl = NULL

pErrorBuffer = NULL

end if

return tasks

end function

function html_clean(string ri)

ri = substitute(ri,"%3A",":")

ri = substitute(ri,"%E2%80%93","-")

ri = substitute(ri,"%E2%80%99","'")

ri = substitute(ri,"%27","'")

ri = substitute(ri,"%2B","+")

ri = substitute(ri,"%C3%A8","e")

ri = substitute(ri,"%C3%A9","e")

ri = substitute(ri,"%22","\"")

ri = substitute(ri,"%2A","*")

return ri

end function

constant html_header = """

<!DOCTYPE html>

<head>

<title>Rosettacode Tasks without examples</title>

</head>

<body>

<h2>Rosettacode Tasks without examples</h2>

Generated %s, %d entries

""",

html_footer = """

</body>

</html>

"""

sequence results = extract_tasks()

if output_html then

integer fn = open("Tasks_Without_Examples.html","w")

printf(fn,html_header,{format_timedate(date()),length(results)})

for i=1 to length(results) do

printf(fn,"%s ",html_clean(results[i]))

end for

puts(fn,html_footer)

close(fn)

else

for i=1 to length(results) do

printf(1,"%s\n",html_clean(results[i]))

end for

end if

?"done"

{} = wait_key()</lang>

=={{header|Raku}}==