Rosetta Code/Count examples: Difference between revisions
→{{header|Python}}: SMW alternative
(→{{header|C sharp|C#}}: Added some error handling and URL Encode the task page name so it can handle e.g. A+B) |
(→{{header|Python}}: SMW alternative) |
||
Line 2,264:
print(f'\nTotal: {sum(tasks)} examples.')
</syntaxhighlight>
===Using Semantic MediaWiki===
{{libheader|Requests}}
Here we use the [https://www.semantic-mediawiki.org/wiki/Help:API:ask ask] API provided by the [https://www.semantic-mediawiki.org/wiki/Semantic_MediaWiki Semantic MediaWiki extension] to query page [https://www.semantic-mediawiki.org/wiki/Help:Properties_and_types properties] rather than parse page content, producing subtly different counts to some other solutions. Maybe too different for this task.
We only count task examples that have a corresponding category page AND that page has the "Is Language" property set to True. In other words, <code>Implemented In</code> + <code>Not Implemented In</code> + <code>Omitted From</code> is always equal to the total number of languages.
The output includes debugging information so we can see what requests are being made.
<syntaxhighlight lang="python">"""Count Rosetta Code tasks implementations using the Semantic MediaWiki API.
Works with Python >= 3.7."""
import json
import logging
from dataclasses import dataclass
from dataclasses import field
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
import requests
from requests.adapters import HTTPAdapter
from requests.adapters import Retry
logging.basicConfig(level=logging.DEBUG)
# See https://www.semantic-mediawiki.org/wiki/Help:API:ask
_SM_ASK: Dict[str, str] = {
"action": "ask",
"format": "json",
"formatversion": "2",
"api_version": "3",
}
_SM_ASK_REQUEST_BLOCK_SIZE = 500
@dataclass(frozen=True)
class Page:
# fulltext is the page's title, not the page content.
fulltext: str
fullurl: str
namespace: int
exists: str
displaytitle: str
@dataclass(frozen=True, eq=False)
class Lang(Page):
def __eq__(self, other: object) -> bool:
if isinstance(other, Lang):
return self.fullurl == other.fullurl
elif isinstance(other, str):
return self.fullurl == other
return False
def __hash__(self) -> int:
return hash(self.fullurl)
def unimplemented(self, tasks: Set["Task"]) -> Set["Task"]:
return {task for task in tasks if self.fullurl not in task.exclude}
def omitted_from(self, tasks: Set["Task"]) -> Set["Task"]:
return {task for task in tasks if self.fullurl in task.omitted_from}
@dataclass(frozen=True)
class Task(Page):
title: str
implemented_in: Set[Lang] = field(repr=False, compare=False)
omitted_from: Set[Lang] = field(repr=False, compare=False)
exclude: Set[Lang] = field(repr=False, compare=False)
def not_implemented_in(self, langs: Set[Lang]) -> Set[Lang]:
return langs.difference(self.exclude)
@dataclass
class TaskResponseBlock:
tasks: List[Task]
continue_offset: Optional[int] = None
@dataclass
class LanguageResponseBlock:
langs: List[Lang]
continue_offset: Optional[int] = None
def sm_ask_category(
session: requests.Session,
url: str,
category: str,
limit: int,
offset: int,
known_langs: Set[Lang],
) -> TaskResponseBlock:
query_params = {
**_SM_ASK,
"query": (
f"[[Category:{category}]]"
"|?Implemented in language"
"|?Omitted from language"
f"|limit={limit}"
f"|offset={offset}"
),
}
response = session.get(url, params=query_params)
response.raise_for_status()
data = response.json()
handle_warnings_and_errors(data)
return _transform_implemented_in_response_data(data, known_langs)
def sm_ask_tasks(
session: requests.Session,
url: str,
limit: int,
offset: int,
known_langs: Set[Lang],
):
return sm_ask_category(
session, url, "Programming Tasks", limit, offset, known_langs
)
def sm_ask_drafts(
session: requests.Session,
url: str,
limit: int,
offset: int,
known_langs: Set[Lang],
):
return sm_ask_category(
session, url, "Draft Programming Tasks", limit, offset, known_langs
)
def sm_ask_languages(
session: requests.Session,
url: str,
limit: int,
offset: int,
) -> LanguageResponseBlock:
query_params = {
**_SM_ASK,
"query": (
"[[Is language::+]]"
"|?Implemented in language"
"|?Omitted from language"
f"|limit={limit}"
f"|offset={offset}"
),
}
response = session.get(url, params=query_params)
response.raise_for_status()
data = response.json()
handle_warnings_and_errors(data)
return _transform_language_response_data(data)
def sm_ask_all_tasks(
session: requests.Session, url: str, known_langs: Set[Lang]
) -> List[Task]:
block = sm_ask_tasks(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=0,
known_langs=known_langs,
)
tasks = block.tasks
while block.continue_offset:
block = sm_ask_tasks(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=block.continue_offset,
known_langs=known_langs,
)
tasks.extend(block.tasks)
return tasks
def sm_ask_all_drafts(
session: requests.Session, url: str, known_langs: Set[Lang]
) -> List[Task]:
block = sm_ask_drafts(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=0,
known_langs=known_langs,
)
tasks = block.tasks
while block.continue_offset:
block = sm_ask_drafts(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=block.continue_offset,
known_langs=known_langs,
)
tasks.extend(block.tasks)
return tasks
def sm_ask_all_languages(session: requests.Session, url: str) -> List[Lang]:
block = sm_ask_languages(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=0,
)
langs = block.langs
while block.continue_offset:
block = sm_ask_languages(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=block.continue_offset,
)
langs.extend(block.langs)
return langs
def _transform_implemented_in_response_data(
data: Any, known_langs: Set[Lang]
) -> TaskResponseBlock:
tasks: List[Task] = []
for result in data["query"]["results"]:
for task_title, task_page in result.items():
# We're excluding implementations that don't have a corresponding
# category page with an "Is Language" property.
implemented_in = {
Lang(**lang)
for lang in task_page["printouts"]["Implemented in language"]
}.intersection(known_langs)
omitted_from = (
{
Lang(**lang)
for lang in task_page["printouts"]["Omitted from language"]
}
.intersection(known_langs)
.difference(implemented_in)
)
tasks.append(
Task(
title=task_title,
implemented_in=implemented_in,
omitted_from=omitted_from,
fulltext=task_page["fulltext"],
fullurl=task_page["fullurl"],
namespace=task_page["namespace"],
exists=task_page["exists"],
displaytitle=task_page["displaytitle"],
exclude=implemented_in.union(omitted_from),
)
)
return TaskResponseBlock(
tasks=tasks, continue_offset=data.get("query-continue-offset", None)
)
def _transform_language_response_data(data: Any) -> LanguageResponseBlock:
langs: List[Lang] = []
for result in data["query"]["results"]:
for _, task_page in result.items():
langs.append(
Lang(
fulltext=task_page["fulltext"],
fullurl=task_page["fullurl"],
namespace=task_page["namespace"],
exists=task_page["exists"],
displaytitle=task_page["displaytitle"],
)
)
return LanguageResponseBlock(
langs=langs, continue_offset=data.get("query-continue-offset", None)
)
def get_session() -> requests.Session:
"""Setup a requests.Session with retries."""
retry_strategy = Retry(
total=5,
status_forcelist=[429, 500, 502, 503, 504],
allowed_methods=["HEAD", "GET", "OPTIONS"],
)
adapter = HTTPAdapter(max_retries=retry_strategy)
session = requests.Session()
session.mount("https://", adapter)
session.mount("http://", adapter)
return session
def handle_warnings_and_errors(data: Any) -> None:
if data.get("errors"):
for error in data["errors"]:
logging.error(json.dumps(error))
# legacy format
if data.get("error"):
logging.error(json.dumps(data["error"]))
if data.get("warnings"):
for warning in data["warnings"]:
logging.warning(json.dumps(warning))
def count_examples(url: str, n: int = 30) -> None:
"""Print a table to stdout containing implementation counts for the first
`n` tasks, sorted by number implementations (most to least)."""
session = get_session()
langs = set(sm_ask_all_languages(session, url))
tasks = sm_ask_all_tasks(session, url, langs)
drafts = sm_ask_all_drafts(session, url, langs)
all_tasks = [*tasks, *drafts]
# Map of task to (implemented in, not implemented in, omitted from)
counts: Dict[Task, Tuple[int, int, int]] = {}
# Running total of examples for all tasks. Where a language has multiple examples
# for a single tasks, we only count one example.
total: int = 0
for task in all_tasks:
total += len(task.implemented_in)
counts[task] = (
len(task.implemented_in),
len(task.not_implemented_in(langs)),
len(task.omitted_from),
)
# Pretty print
top = sorted(counts.items(), key=lambda it: it[1][0], reverse=True)[:n]
pad = max([len(task.fulltext) for task, _ in top])
print("Known languages:", len(langs))
print("Total tasks:", len(all_tasks))
print("Total examples:", total)
print(f"{'Task':>{pad}} | Implemented In | Not Implemented In | Omitted From")
print("-" * (pad + 1), "+", "-" * 16, "+", "-" * 20, "+", "-" * 13, sep="")
for task, _counts in top:
implemented_in, not_implemented_in, omitted_from = _counts
print(
f"{task.fulltext:>{pad}} |"
f"{implemented_in:>15} |"
f"{not_implemented_in:>19} |"
f"{omitted_from:>13}"
)
if __name__ == "__main__":
import argparse
URL = "https://rosettacode.org/w/api.php"
parser = argparse.ArgumentParser(description="Count tasks on Rosetta Code.")
parser.add_argument(
"--rows",
"-n",
type=int,
default=30,
dest="n",
help="number of rows to display in the output table (default: 30)",
)
parser.add_argument(
"--url",
default=URL,
help=f"target MediaWiki URL (default: {URL})",
)
args = parser.parse_args()
count_examples(args.url, args.n)</syntaxhighlight>
{{out}}
<pre>
DEBUG:urllib3.connectionpool:Starting new HTTPS connection (1): rosettacode.org:443
DEBUG:urllib3.connectionpool:https://rosettacode.org:443 "GET /w/api.php?action=ask&format=json&formatversion=2&api_version=3&query=%5B%5BIs+language%3A%3A%2B%5D%5D%7C%3FImplemented+in+language%7C%3FOmitted+from+language%7Climit%3D500%7Coffset%3D0 HTTP/1.1" 200 None
DEBUG:urllib3.connectionpool:https://rosettacode.org:443 "GET /w/api.php?action=ask&format=json&formatversion=2&api_version=3&query=%5B%5BIs+language%3A%3A%2B%5D%5D%7C%3FImplemented+in+language%7C%3FOmitted+from+language%7Climit%3D500%7Coffset%3D500 HTTP/1.1" 200 None
DEBUG:urllib3.connectionpool:https://rosettacode.org:443 "GET /w/api.php?action=ask&format=json&formatversion=2&api_version=3&query=%5B%5BCategory%3AProgramming+Tasks%5D%5D%7C%3FImplemented+in+language%7C%3FOmitted+from+language%7Climit%3D500%7Coffset%3D0 HTTP/1.1" 200 None
DEBUG:urllib3.connectionpool:https://rosettacode.org:443 "GET /w/api.php?action=ask&format=json&formatversion=2&api_version=3&query=%5B%5BCategory%3AProgramming+Tasks%5D%5D%7C%3FImplemented+in+language%7C%3FOmitted+from+language%7Climit%3D500%7Coffset%3D500 HTTP/1.1" 200 None
DEBUG:urllib3.connectionpool:https://rosettacode.org:443 "GET /w/api.php?action=ask&format=json&formatversion=2&api_version=3&query=%5B%5BCategory%3AProgramming+Tasks%5D%5D%7C%3FImplemented+in+language%7C%3FOmitted+from+language%7Climit%3D500%7Coffset%3D1000 HTTP/1.1" 200 None
DEBUG:urllib3.connectionpool:https://rosettacode.org:443 "GET /w/api.php?action=ask&format=json&formatversion=2&api_version=3&query=%5B%5BCategory%3ADraft+Programming+Tasks%5D%5D%7C%3FImplemented+in+language%7C%3FOmitted+from+language%7Climit%3D500%7Coffset%3D0 HTTP/1.1" 200 None
Known languages: 888
Total tasks: 1598
Total examples: 90914
Task | Implemented In | Not Implemented In | Omitted From
--------------------------------+----------------+--------------------+-------------
Hello world/Text | 498 | 390 | 0
99 bottles of beer | 370 | 518 | 0
100 doors | 333 | 554 | 1
FizzBuzz | 325 | 563 | 0
Fibonacci sequence | 307 | 581 | 0
Factorial | 301 | 587 | 0
Comments | 296 | 591 | 1
A+B | 293 | 595 | 0
Empty program | 274 | 614 | 0
Loops/Infinite | 254 | 633 | 1
Function definition | 252 | 634 | 2
Loops/For | 248 | 639 | 1
Loops/While | 243 | 644 | 1
Arrays | 237 | 650 | 1
Ackermann function | 234 | 651 | 3
Reverse a string | 231 | 655 | 2
Conditional structures | 223 | 664 | 1
Arithmetic/Integer | 213 | 675 | 0
Greatest common divisor | 213 | 675 | 0
Array concatenation | 208 | 680 | 0
Greatest element of a list | 208 | 680 | 0
Even or odd | 207 | 681 | 0
Loops/Downward for | 204 | 684 | 0
Increment a numerical string | 202 | 686 | 0
Integer comparison | 202 | 686 | 0
Sieve of Eratosthenes | 202 | 686 | 0
Repeat a string | 199 | 689 | 0
Boolean values | 198 | 689 | 1
Loops/For with a specified step | 198 | 689 | 1
Copy a string | 196 | 691 | 1
</pre>
=={{header|R}}==
| |||