Rosetta Code/Count examples

From Rosetta Code

Find the total number of programming examples for each task and the total for all tasks.

Essentially, count the number of occurrences of =={{header| on each task page.

Output:

100 doors: 20 examples.
99 Bottles of Beer: 29 examples.
Abstract type: 10 examples.
 
Total: X examples.

Contents 1 AutoHotkey 2 C# 3 D 4 Haskell 5 J 6 Java 7 OCaml 8 Oz 9 Perl 10 Python 11 R 12 Ruby 13 Scala 14 Tcl

[edit] AutoHotkey

UrlDownloadToFile
  , http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml
  , tasks.xml
FileRead, tasks, tasks.xml 
pos = 0
quote = "  ; "
regtitle := "<cm.*?title=" . quote . "(.*?)" . quote
While, pos := RegExMatch(tasks, regtitle, title, pos + 1)
{
  UrlDownloadToFile
    , % "http://www.rosettacode.org/w/index.php?title=" . title1 . "&action=raw"
    , task.xml
  FileRead, task, task.xml
  RegExReplace(task, "\{\{header\|", "", count)
  current :=  title1 . ": " . count . " examples.`n"
  output .= current
  TrayTip, current, % current
}
MsgBox % output
Return

[edit] C#

Object-oriented solution.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Net;
 
class Task {
    private string _task;
    private int _examples;
 
    public Task(string task, int examples) {
        _task = task;
        _examples = examples;
    }
 
    public string Name {
        get { return _task; }
    }
 
    public int Examples {
        get { return _examples; }
    }
 
    public override string ToString() {
        return String.Format("{0}: {1} examples.", this._task, this._examples);
    }
}
 
class Program {
    static List<string> GetTitlesFromCategory(string category, WebClient wc) {
        string content = wc.DownloadString(
            String.Format("http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:{0}&cmlimit=500&format=json", category)
        );
 
        return new Regex("\"title\":\"(.+?)\"").Matches(content).Cast<Match>().Select(x => x.Groups[1].Value.Replace("\\/", "/")).ToList();
    }
 
    static string GetSourceCodeFromPage(string page, WebClient wc) {
        return wc.DownloadString(
            String.Format("http://www.rosettacode.org/w/index.php?title={0}&action=raw", page)
        );
    }
 
    static void Main(string[] args) {
        WebClient wc = new WebClient();
        List<Task> tasks = new List<Task>();
        List<string> tasknames = GetTitlesFromCategory("Programming_Tasks", wc);
 
        foreach (string task in tasknames) {
            string content = GetSourceCodeFromPage(task, wc);
            int count = new Regex("=={{header", RegexOptions.IgnoreCase).Matches(content).Count;
            Task t = new Task(task, count);
 
            Console.WriteLine(t);
            tasks.Add(t);
        }
 
        Console.WriteLine("\nTotal: {0} examples.", tasks.Select(x => x.Examples).Sum());
    }
}

[edit] D

Works with: Tango

 
import tango.io.Stdout;
import tango.net.http.HttpClient;
import tango.net.http.HttpHeaders;
import tango.text.xml.Document;
import tango.text.Util;
 
alias HttpHeader.ContentLength CL;
 
auto url = "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml";
void main()
{
    auto client = new HttpClient (HttpClient.Get, url);
    client.open();
    char[] mainData, tmp;
    int total, i;
 
    void cat(void[] content) { tmp ~= cast(char[]) content; }
 
    if (client.isResponseOK) {
        client.read(&cat, client.getResponseHeaders.getInt(CL));
        mainData = tmp;
        tmp = null;
 
        auto doc = new Document!(char);
        doc.parse(mainData);
        foreach (n; doc.query.descendant("cm").attribute("title")) {
            auto subClient = new HttpClient(HttpClient.Get, 
                    "http://www.rosettacode.org/w/index.php?title=" ~
                    replace(n.value.dup, ' ', '_') ~ "&action=raw");
            subClient.open();
            if (! subClient.isResponseOK) {
                Stderr (client.getResponse);
                 break;
            }
            subClient.read(&cat, subClient.getResponseHeaders.getInt(CL));
            foreach (segment; patterns(cast(char[])tmp, "=={{header|")) i++;
            --i;
            if (i) --i;
            Stdout.formatln ("{0,-40} - {}", n.value, i);
            total += i;
            tmp = null;
            i = 0;
        }
        Stdout("total examples: ", total).newline;
    } else {
        Stderr (client.getResponse);
    }
}
 

[edit] Haskell

Library: HTTP XML from HackageDB

import Network.Browser
import Network.HTTP
import Network.URI
import Data.List
import Data.Maybe
import Text.XML.Light
import Control.Arrow
 
justifyR w = foldl ((.return).(++).tail) (replicate w ' ')
showFormatted t n = t ++ ": " ++ justifyR 4 (show n)
 
getRespons url = do
    rsp <- Network.Browser.browse $ do
      setAllowRedirects True
      setOutHandler $ const (return ())     -- quiet
      request $ getRequest url
    return $ rspBody $ snd rsp
 
getNumbOfExampels p = do
  let pg = intercalate "_" $ words p
  rsp <- getRespons $ "http://www.rosettacode.org/w/index.php?title=" ++ pg ++ "&action=raw"
  let taskPage = rsp
      countEx = length $ filter (=="=={{header|") $ takeWhile(not.null) $ unfoldr (Just. (take 11 &&& drop 1)) taskPage
  return countEx
 
progTaskExamples = do 
  rsp <- getRespons "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
 
  let xmls = onlyElems $ parseXML $ rsp
      tasks = concatMap (map (fromJust.findAttr (unqual "title")). filterElementsName (== unqual "cm")) xmls
 
  taskxx <- mapM getNumbOfExampels tasks
  let ns = taskxx
      tot = sum ns
 
  mapM_ putStrLn $ zipWith showFormatted tasks ns
  putStrLn $ ("Total: " ++) $ show tot

some output:

*Main> progTaskExamples                                                    
100 doors:   56                                                            
24 game:   11                                                              
24 game Player:    9                                                       
99 Bottles of Beer:   73                                                   
Abstract type:   23                                                        
Ackermann Function:   61                                                   
Active object:    9
...
Total: 9156

[edit] J

Solution:
Using getCategoryMembers from Find unimplemented tasks.

require 'web/gethttp'
 
getAllTaskSolnCounts=: monad define
  tasks=.  getCategoryMembers 'Programming_Tasks'
  counts=. getTaskSolnCounts &> tasks
  tasks;counts
)
 
getTaskSolnCounts=: monad define
  makeuri=. 'http://www.rosettacode.org/w/index.php?title=' , ,&'&action=raw'
  wikidata=. gethttp makeuri urlencode y
  ([: +/ '{{header|'&E.) wikidata
)
 
formatSolnCounts=: monad define
  'tasks counts'=. y
  tasks=. tasks , &.>':'
  res=. ;:^:_1 tasks ,. (8!:0 counts) ,. <'examples.'
  res , 'Total examples: ' , ": +/counts
)

Example Usage:

   formatSolnCounts getAllTaskSolnCounts ''
100 doors: 61 examples.                                      
24 game: 15 examples.                                        
24 game Player: 11 examples.                                 
99 Bottles of Beer: 76 examples.                             
...

[edit] Java

Works with: Java version 1.5+

 
import java.util.ArrayList;
import java.util.Iterator;
import ScreenScrape;
 
public class CountProgramExamples {
	private static final String baseURL = "http://rosettacode.org/wiki/";
	private static final String rootURL = "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml";
	private static final String taskBegin = "title=\"";
	private static final String taskEnd = "\"";
	private static final String exmplBegin = "<span class=\"tocnumber\">";
	private static final String exmplEnd = "</span>";
	private static final String editBegin = "<span class=\"editsection\">";
 
	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// Setup variables
		int exTotal = 0;
		int exSubTot = 0;
		String title = "";
		String taskPage = "";
		int startPos = 0;
		String countStr = "";
		try {
			// Get root query results
			ArrayList<String> tasks = new ArrayList<String>();
			ScreenScrape ss = new ScreenScrape();
			String rootPage = ss.read(rootURL);
			while(rootPage.contains(taskBegin)){
				rootPage = rootPage.substring(rootPage.indexOf(taskBegin)+taskBegin.length());
				title = rootPage.substring(0, rootPage.indexOf(taskEnd));
				if (!title.contains("Category:")) {
					tasks.add(title);
				}
				rootPage = rootPage.substring(rootPage.indexOf(taskEnd));
			}
			// Loop through each task and print count
			Iterator<String> itr = tasks.iterator();
			while(itr.hasNext()) {
				title = itr.next().replaceAll("&#039;","'");
				taskPage = ss.read(baseURL+title.replaceAll(" ", "_"));
				if (taskPage.contains(exmplBegin)) {
					startPos = taskPage.lastIndexOf(exmplBegin)+exmplBegin.length();
					countStr = taskPage.substring(startPos,	taskPage.indexOf(exmplEnd, startPos));
					exSubTot = Integer.parseInt(countStr.contains(".") ?
							countStr.substring(0,countStr.indexOf(".")) : countStr);
				}else{
					exSubTot = 0;
					while(taskPage.contains(editBegin)) {
						taskPage = taskPage.substring(taskPage.indexOf(editBegin)+editBegin.length());
						exSubTot++;
					}
				}
				exTotal += exSubTot;
				System.out.println(title+": "+exSubTot+" examples.");
			}
			// Print total
			System.out.println("\nTotal: "+exTotal+" examples.");
		}catch(Exception e){
			System.out.println(title);
			System.out.println(startPos+":"+taskPage.indexOf(exmplEnd, startPos));
			System.out.println(taskPage);
			e.printStackTrace(System.out);
		}
	}
}
 

ScreenScrape class

[edit] OCaml

Library: ocamlnet

Library: xml-light

execute with:

ocaml str.cma unix.cma  -I +pcre pcre.cma  -I +netsys netsys.cma  -I +equeue equeue.cma \
  -I +netstring netstring.cma  -I +netclient netclient.cma  -I +xml-light xml-light.cma  countex.ml

or with the findlib package one can compile with:

ocamlfind opt -linkpkg -package str,unix,xml-light,netclient  countex.ml -o countex.opt

open Http_client.Convenience
 
let repl_quote s =
  let reg = Str.regexp_string "'" in
  (Str.global_replace reg "%27" s)
 
let repl_space s =
  let s = String.copy s in
  for i = 0 to pred(String.length s) do
    if s.[i] = ' ' then s.[i] <- '_'
  done;
  (s)
 
let count_ex s =
  let pat = Str.regexp_string "=={{header|" in
  let rec aux n p =
    try
      let p = Str.search_forward pat s p in
      aux (n+1) (p+1)
    with Not_found -> (n)
  in
  aux 0 0
 
 
let () =
  let url = "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&\
               cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml" in
 
  let xml = Xml.parse_string (http_get url) in
 
  let total = ref 0 in
  at_exit (fun () -> Printf.printf "\n Total: %d\n" !total);
 
  let f = function
  | Xml.Element ("cm", attrs, _) ->
      let _title = List.assoc "title" attrs in
      let title = repl_quote (repl_space _title) in
      let url = "http://www.rosettacode.org/w/index.php?title="^ title ^"&action=raw" in
      let n = count_ex (http_get url) in
      Printf.printf "%s: %d\n%!" _title n;
      total := n + !total;
  | _ -> ()
  in
 
  match xml with
  | Xml.Element ("api", [],
     [Xml.Element ("query", [],
       [Xml.Element ("categorymembers", [], cms)])]) -> List.iter f cms
  | _ -> ()

outputs:

100 doors: 56
24 game: 11
24 game Player: 9
99 Bottles of Beer: 73
Abstract type: 23
Ackermann Function: 61

...

XML Reading: 22
XML and XPath: 18
Xiaolin Wu's line algorithm: 3
Y combinator: 23
Yuletide Holiday: 32
Zig Zag: 29

 Total: 9106

[edit] Oz

Library: OzHttpClient

declare
  [HTTPClient] = {Module.link ['x-ozlib://mesaros/net/HTTPClient.ozf']}
  [XMLParser] = {Module.link ['x-oz://system/xml/Parser.ozf']}
  [StringX] = {Module.link ['x-oz://system/String.ozf']}
  [Regex] = {Module.link ['x-oz://contrib/regex']}
 
  AllTasksUrl = "http://rosettacode.org/mw/api.php?action=query&list="#
  "categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
 
  proc {Main}
     AllTasks = {Parse {GetPage AllTasksUrl}}
     TaskTitles = {GetTitles AllTasks}
     Total = {NewCell 0}
  in
     for Task in TaskTitles do
	TaskPage = {GetPage {TaskUrl Task}}
	RE = {Regex.compile "{{header\\|" [extended newline icase]}
	NumMatches = {Length {Regex.allMatches RE TaskPage}}
     in
	{System.showInfo Task#": "#NumMatches#" examples."}
	Total := @Total + NumMatches
     end
     {System.showInfo "Total: "#@Total#" examples."}
  end
 
  fun {TaskUrl Task}
     "http://rosettacode.org/mw/index.php?"#
     "title="#{StringX.replace Task " " "_"}#
     "&action=raw"
  end
 
  %% GetPage
  local
     Client = {New HTTPClient.urlGET init(inPrms(toFile:false toStrm:true) _)}
  in
     fun {GetPage RawUrl}
        Url = {VirtualString.toString RawUrl}
        OutParams
        HttpResponseParams
     in
        {Client getService(Url ?OutParams ?HttpResponseParams)}
        OutParams.sOut
     end
  end
 
  %% Parse
  local
     Parser = {New XMLParser.parser init} 
  in
     fun {Parse Xs} {Parser parseVS(Xs $)} end
  end
 
  fun {GetTitles Doc}
     CMs = Doc.2.1.children.1.children.1.children
     fun {Attributes element(attributes:As ...)} As end
     fun {IsTitle attribute(name:N ...)} N == title end
  in
     {Map {Filter {Flatten {Map CMs Attributes}} IsTitle}
      fun {$ A} {Atom.toString A.value} end}
  end
in
  {Main}

Example output:

100 doors: 59 examples.
24 game: 14 examples.
...
Yuletide Holiday: 33 examples.
Zig Zag: 31 examples.
Total: 9849 examples.

[edit] Perl

 
#!/usr/bin/perl -w
use strict ;
use LWP::UserAgent ;
use HTML::Parser ;
use constant DOCROOT => "http://www.rosettacode.org/wiki" ;
use constant SOLUTIONROOT => "http://www.rosettacode.org/w/index.php?title=" ;
my %tasklist = ( ) ; #key: last part of solution list URL, value: title of solution
my $ua = new LWP::UserAgent ;
my $url = DOCROOT . "/Category:Programming_Tasks" ;
my $request = HTTP::Request->new( 'GET' => "$url" ) ; 
my $response = $ua->request( $request ) ; 
my $counted = 0 ; 
my $total_examples = 0 ; 
my $solresponse ;
my $p = HTML::Parser->new( api_version => 3 ) ; #parser for list of tasks
my $q = HTML::Parser->new( api_version => 3 ) ; #parser for solutions by task
$p->handler( start => \&process , "tagname , attr" ) ; 
$q->handler( text => \&langfinder, "text" ) ; 
 
if ( $response->is_success( ) ) { 
   $p->parse( $response->content( ) ) ; 
   foreach my $task( keys %tasklist ) { 
      $request->uri( SOLUTIONROOT . "$task" . "&action=edit" ) ; 
      $solresponse = $ua->request( $request ) ;
      if ( $solresponse->is_success( )) {
         $q->parse( $solresponse->content( ) ) ;
         if ( $tasklist{$task} ) {
             print "$tasklist{$task} : $counted examples!\n" ;
         }
         $counted = 0 ;
         $q->eof( ) ;
      }
      else {
         print "Error: " . $solresponse->code( ) . " " . $solresponse->message( ) . "\n" ;
      }
   }
   $p->eof( ) ;
   print "\nTotal: $total_examples examples.\n" ;
}
else {
   print "Error " . $response->code( )  . " " . $response->message( ) . "\n" ;
}
sub process( ) {
   return if shift ne "a" ;
   my $props = shift ;
   if ( $props->{href} && $props->{href} =~ m,/wiki/([^:]+), ) {
      if ( $1 !~ /Category/ ) {
         $tasklist{ $1 } = $props->{title} ;
      }
   }
}
sub langfinder( ) {
   my $text = shift ;
   while ( $text =~ /header\|.+\}/g ) {
      $counted++ ;
      $total_examples++ ;
   }
}
 

[edit] Python

This example is incorrect. Problem with task "Catmull–Clark subdivision surface" because it contains unicode in the title. See talkpage for traceback. Please fix the code and remove this message.

import urllib, xml.dom.minidom
 
x = urllib.urlopen("http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml")
 
tasks = []
for i in xml.dom.minidom.parseString(x.read()).getElementsByTagName("cm"):
    t = i.getAttribute('title').replace(" ", "_")
    y = urllib.urlopen("http://www.rosettacode.org/w/index.php?title=%s&action=raw" % t)
    tasks.append( y.read().lower().count("{{header|") )
    print t.replace("_", " ") + ": %d examples." % tasks[-1]
 
print "\nTotal: %d examples." % sum(tasks)

[edit] R

Library: XML (R)

Library: RCurl

 
library(XML)
library(RCurl)
doc <- xmlInternalTreeParse("http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml")
nodes <- getNodeSet(doc,"//cm")
titles = as.character( sapply(nodes, xmlGetAttr, "title") )
headers <- list()
counts <- list()
for (i in 1:length(titles)){
	headers[[i]] <- getURL( paste("http://rosettacode.org/mw/index.php?title=", gsub(" ", "_", titles[i]), "&action=raw", sep="") )
	counts[[i]] <- strsplit(headers[[i]],split=" ")[[1]]
	counts[[i]] <- grep("\\{\\{header", counts[[i]])
	cat(titles[i], ":", length(counts[[i]]), "examples\n")
}
cat("Total: ", length(unlist(counts)), "examples\n")
 

[edit] Ruby

Library: REXML First, a RosettaCode module, saved as rosettacode.rb:

require 'open-uri'
require 'rexml/document'
 
module RosettaCode
 
  def self.rc_url(page, query)
    url = "http://www.rosettacode.org/w/%s?%s" % [
      URI.escape(page),
      URI.escape(query.map {|k,v| "%s=%s" % [k,v]}.join("&"))
    ]
    url.gsub(/\+/, '%2B')
  end
 
  def self.rc_tasks(category)
    query = {
      "action" => "query",
      "list" => "categorymembers",
      "cmtitle" => "Category:#{category}",
      "format" => "xml",
      "cmlimit" => 500,
    }
    while true
      url = rc_url "api.php", query
      doc = REXML::Document.new open(url)
 
      REXML::XPath.each(doc, "//cm") do |task|
        yield task.attribute("title").value
      end
 
      continue = REXML::XPath.first(doc, "//query-continue")
      break if continue.nil?
      cm = REXML::XPath.first(continue, "categorymembers")
      query["cmcontinue"] = cm.attribute("cmcontinue").value
    end
  end
 
end

Then, we implement the task with:

require 'rosettacode'
 
total_examples = 0
 
RosettaCode.rc_tasks("Programming_Tasks") do |task|
  url = RosettaCode.rc_url("index.php", {"action" => "raw", "title" => task})
  examples = open(url).read.scan("=={{header").length
  puts "#{task}: #{examples}"
  total_examples += examples
end
 
puts
puts "Total: #{total_examples}"

[edit] Scala

This was writen for Scala 2.8, but Scala 2.7 can be used with slight modifications to the IO library.

Different than the example for other languages, it parallelizes the reading and counting, and it also encode the URL, because some URLs are now causing problems. These modifications are minor, though.

It was written in script style.

import java.net.{URL, URLEncoder}
import scala.io.Source.fromURL
 
val allTasksURL = "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
val allTasks = xml.parsing.XhtmlParser(fromURL(new URL(allTasksURL)))
 
val regexExpr = "(?i)==\\{\\{header\\|".r
def oneTaskURL(title: String) = "http://www.rosettacode.org/w/index.php?title=%s&action=raw" format URLEncoder.encode(title.replace(' ', '_'), "UTF-8")
def count(title: String) =  regexExpr findAllIn fromURL(new URL(oneTaskURL(title)))(io.Codec.UTF8).mkString length
 
val counts = for (task <- allTasks \\ "cm" \\ "@title" map (_.text)) yield scala.actors.Futures.future((task, count(task)))
 
counts map (_.apply) map Function.tupled("%s: %d examples." format (_, _)) foreach println
println("\nTotal: %d examples." format (counts map (_.apply._2) sum))
 

[edit] Tcl

Using the json package from Library: tcllib

package require Tcl 8.5
package require http
package require json
 
fconfigure stdout -buffering none
 
proc get_tasks {category} {
    set start [clock milliseconds]
    puts -nonewline "getting $category members..."
    set base_url http://www.rosettacode.org/w/api.php
    set query {action query list categorymembers cmtitle Category:%s format json cmlimit 500}
    set this_query [dict create {*}[split [format $query $category]]]
    set tasks [list]
 
    while {1} {
        set url [join [list $base_url [http::formatQuery {*}$this_query]] ?]
        set response [http::geturl $url]
        if {[set s [http::status $response]] ne "ok" || [http::ncode $response] != 200} {
            error "Oops: url=$url\nstatus=$s\nhttp code=[http::code $response]"
        }
        set data [json::json2dict [http::data $response]]
        http::cleanup $response
 
        # add tasks to list
        foreach task [dict get $data query categorymembers] {
            lappend tasks [dict get [dict create {*}$task] title]
        }
 
        if {[catch {dict get $data query-continue categorymembers cmcontinue} continue_task] != 0} {
            # no more continuations, we're done
            break
        }
        dict set this_query cmcontinue $continue_task
    }
    puts " found [llength $tasks] tasks in [expr {[clock milliseconds] - $start}] milliseconds"
    return $tasks
}
 
# This proc can be replaced by a single regexp command:
#     set count [regexp -all "***=$needle" $haystack]
# However this proc is more efficient -- we're dealing with plain strings only.
proc count_substrings {needle haystack} {
    set count 0
    set idx 0
    while {[set idx [string first $needle $haystack $idx]] != -1} {
        incr count
        incr idx
    }
    return $count
}
 
set total 0
foreach task [get_tasks Programming_Tasks] {
    set url [format "http://www.rosettacode.org/w/index.php?title=%s&action=raw" [string map {{ } _} $task]]
    set response [http::geturl $url]
    if {[set s [http::status $response]] ne "ok" || [http::ncode $response] != 200} {
        error "Oops: url=$url\nstatus=$s\nhttp code=[http::code $response]"
    }
    set count [count_substrings "\{\{header|" [http::data $response]]
    puts [format "%3d examples in %s" $count $task]
    http::cleanup $response
    incr total $count
}
 
puts "\nTotal: $total examples"