User:ImplSearchBot/Code: Difference between revisions

our $editor;

our $starttime = time;

our $pagesedited = 0;

our $categorypulls = 0;

our $cachehits = 0;

our $task_count_change;

our $alltasks;

our %impldiff;

our %createdomitcategories;

our @initialcache;

our $alllanguages;

my $name = shift;

my $body = shift;

return {$name => $body};

my $impldiff = shift;

my $language = shift;

my $unimplisting = "";

foreach my $taskname (@$alltasks)

{

# If it's a category task, the task name will be slightly different.

my $baretaskname = $taskname;

$baretaskname =~ s/^Category://;

my $implpage = $taskname;

$implpage =~ s/^Category:(.*)/$1\/$language/;

my $escapedImplPage = $implpage;

$escapedImplPage =~ s/ /_/g;

my $link;

# if($taskname eq $baretaskname)

# {

$link = "[http://rosettacode.org/mw/index.php?action=edit&title=$escapedImplPage $implpage]";

# }

# else

# {

# $link = "[[:$taskname|$baretaskname]]";

# }

# Add the task to the unimplemented list, if it's unimplemented, and if it's not in the omit list.

unless(exists $impldiff->{$language}->{'impl'}->{$implpage} or exists $impldiff->{$language}->{'omit'}->{$implpage})

{

$unimplisting .= "* $link\n";

}

}

return &build_posting("Template:" . &get_unimpl_template_name($language), $unimplisting);

my $implediff = shift;

my $language = shift;

my $omitlisting = "";

my $omittemplatename = &get_omit_template_name($language);

foreach my $taskname (@$alltasks)

{

# We want the task name, not the fully-qualified wiki name.

my $baretaskname = $taskname;

$baretaskname =~ s/^Category://;

# Add the task to the omission list, if it's omitted.

my $escapedTaskName = $baretaskname;

$escapedTaskName =~ s/ /_/g;

$omitlisting .= "* [http://rosettacode.org/mw/index.php?action=edit&title=$escapedTaskName $baretaskname]\n"

if(exists $impldiff{$language}->{'omit'}->{$taskname})

}

# Note that there's no data in the template.

$implediff->{$language}->{'omit'} = 0

if( "" eq $omitlisting );

return &build_posting("Template:$omittemplatename", $omitlisting);

my $impldiff = shift;

my $language = shift;

my $targetcount = $impldiff->{$language}->{'target_count'};

my $unimpcount = $impldiff->{$language}->{'unimp_count'};

# Prepare template fields

my $langfield = "|$language";

my $unimpfield = "|$unimpcount";

my $tcfield = "|$targetcount";

my $impperccalc = 0;

$impperccalc = (($targetcount - $unimpcount) / $targetcount) * 100

my $imppercfield = sprintf "|%u", $impperccalc;

# Prepare the listing page format.

my $pagedata = '{{unimpl_header' . $langfield . $unimpfield . $tcfield . $imppercfield . '}}';

$pagedata .= "{{" . &get_unimpl_template_name($language) . "$unimpfield}}";

if (0 != $impldiff->{$language}->{'omit'})

{

$pagedata .= "{{omit_header" . "$langfield}}";

$pagedata .= "{{" . &get_omit_template_name($language) ."}}";

}

$pagedata .= "{{unimpl_footer$langfield}}";

return &build_posting(&get_listing_name($language), $pagedata);

my $implediff = shift;

my $language = shift;

return &build_posting(&get_omit_cat_name($language), "{{omit_cat|$language}}");

my $language = shift;

return "Tasks not implemented in $language";

my $language = shift;

return "unimp_omit_body_$language";

my $language = shift;

return "unimp_body_$language";

my $language = shift;

return "Category:$language/Omit";

my $implediff = shift;

my $language = shift;

my $unimpl = &prep_unimp_posting($implediff, $language);

my $omit = &prep_omit_posting($implediff, $language);

my $listing = &prep_listing_posting($implediff, $language);

my %langpostings = ( %$unimpl, %$omit, %$listing );

unless ( exists $createdomitcategories{&get_omit_cat_name($language)} )

{

my $omit_cat = &prep_omit_cat_posting($implediff, $language);

%langpostings = ( %langpostings, %$omit_cat );

}

return \%langpostings;

}

sub wikitxt_pathname {

my $page_name = $_[0];

"test/" . sanitizenamefs($page_name) . ".wikitxt";

my $implediff = shift;

my %work;

# Prepare all our work.

foreach my $language (keys %$implediff)

{

# &out("Preparing data for:$language\n", 4);

my $workitem = &process_language($implediff, $language);

%work = ( %work, %$workitem );

}

# Now that we've prepared all our work, commit it.

foreach my $pagename (keys %work)

{

&postpage($pagename, $work{$pagename}, "Updating $pagename", 0);

}

# Return the Mediawiki editor object.

# If we're not posting, and we're only drawing from cache

# We don't actually need to pull from the wiki.

if( exists $options{'cacheonly'} )

{

return undef

unless exists $options{'post'};

}

# Handles interaction with the wiki.

# Note that I had to modify HTTP::Message to make it work

# HTTP::Message silently failed when presented by MW

# with an encoding type of "application/json" or some such.

&out("Creating editor\n", 3);

my $editor = MediaWiki::Bot->new('ImpleSearchBot');

$editor->{debug} = 1;

# Tell the editor to edit Rosetta Code. I'm sure Wikipedia didn't like

# my initial attempts from before I added this line.

&out("Trying to set wiki.\n",3);

$editor->set_wiki('rosettacode.org','mw');

# If we're not posting, we don't need to log in if we're pulling from cache.

# Otherwise, attempt to log in.

&out("Trying to log in.\n", 3);

my $loginres = $editor->login($options{'username'}, $options{'password'});

die "Unable to login: " . $loginres

unless $loginres == 0;

return $editor;

my $string = shift;

my $loglevel = shift;

chomp $string;

if($options{'verbosity'} >= $loglevel)

{

if($loglevel == 1)

{

warn $string; # use stderr.

}

else

{

print "$string\n";

}

}

my $pagename = shift;

$pagename =~ tr/:\//__/;

return $pagename;

my $first = shift;

my $second = shift;

my %firsthash = map { $_, 1 } @$first;

my @new = ();

foreach my $secondelement (@$second)

{

push @new, $secondelement

unless exists $firsthash{$secondelement};

}

return @new;

my $first = shift;

my $second = shift;

my @onlyinfirst = &diffcat_simple($second, $first);

my @onlyinsecond = &diffcat_simple($first, $second);

return (\@onlyinfirst, \@onlyinsecond);

}

sub slurp_file {

my $pathname = $_[0];

open(my $stream, "<", $pathname) or do {warn"Can't open '$pathname': $!\n";

return};

my $ret;

while (1) {

my $len = read($stream, my $buf, 1024);

if (!defined($len)) { die "I/O error while reading '$pathname': $!"; }

elsif ($len == 0) { last }

else { $ret .= $buf }

}

$ret;

my $pagename = shift;

my $pagedata = shift;

my $remark = shift;

my $minoredit = shift;

# MediaWiki won't let us create blank, empty pages.

# But since we don't want to query (or cache) to see if

# the page already exists, we'll just add an HTML

# non-breaking-space entity if the page is truly empty.

$pagedata = "&nbsp;"

if(0 == length $pagedata);

++$pagesedited;

unless( exists $options{'post'} )

{

$pagename = wikitxt_pathname($pagename);

# save it to disk, and out of the way.

&out("Saving: $pagename\n", 2);

open my $outfile, '>', $pagename

or &out("Failed to open $pagename: $!", 1);

return unless defined $outfile;

print $outfile $pagedata;

close $outfile;

}

else {

&out("Posting $pagename\n", 2);

$editor->edit($pagename, $pagedata, "ImplSearchBot:$remark", $minoredit)

or &out("Failed to post page: " . $editor->{'errstr'}, 1);

}

# &reportcatchanges($categoryname, $old, $new);

sub getopt {

#I don't care to pollute my global scope.

my $wiki = 'rosettacode.org';

my $username;

my $password;

my $verbosity = 2; # verbosity level. 0 is silent. 1 is error only. 2 is updates. 3 is process, more is noisy.

my $post; # Is this an actual run?

my $cachepath = "/tmp/";

my $cacheonly; # Don't query the wiki for data. Just pull from cache.

my $nosvn;

my $rebuild_all;

my $opt_matrix = {

"wiki=s" => \$wiki,

"username=s" => \$username,

"password=s" => \$password,

"verbosity=s" => \$verbosity,

"post" => \$post,

"cacheonly" => \$cacheonly,

"nosvn" => \$nosvn,

"cachepath=s" => \$cachepath,

"rebuildall" => \$rebuild_all };

my $result = GetOptions( %$opt_matrix );

$options{'wiki'} = $wiki;

$options{'nosvn'} = $nosvn

if defined $nosvn;

$options{'username'} = $username

if defined $username;

$options{'password'} = $password

if defined $password;

$options{'post'} = $post

if defined $post;

$options{'verbosity'} = $verbosity;

$cachepath .= '/'

if('/' ne substr($cachepath, -1, 1));

$options{'cachepath'} = $cachepath;

$options{'cacheonly'} = $cacheonly

if defined $cacheonly;

$options{'rebuild_all'} = $rebuild_all

if defined $rebuild_all;

my $usage = "Usage: $0 (options)\n The available options are:\n";

$usage .= "\t--$_\n"

foreach (keys %$opt_matrix);

$usage .= "Username and password are required if you need to pull data from the wiki. Wiki defaults to Rosetta Code.\n";

my $wikineeded;

$wikineeded = "yes"

unless ( exists $options{'cacheonly'} );

$wikineeded = "yes"

if ( exists $options{'post'} );

if(defined $wikineeded)

{

unless(exists $options{'username'} and exists $options{'password'})

{

die $usage;

}

}

}

sub main {

unless (@ARGV) { @ARGV = @_; }

getopt();

#Statistic tracking.

# Tracking for svn checkin at end.

# We *should* be the only ones writing to the cache path for now.

# Anyone else should have their own checkout of the data,

# and test runs are done with --cacheonly and --nosvn.

opendir(my $cachedir, $options{'cachepath'})

or die "Unable to open cache directory";

@initialcache = readdir($cachedir);

closedir($cachedir);

&out(scalar @initialcache . " categories initially cached\n", 4);

# Get our editor

$editor = &geteditor();

# Get a complete listing of the tasks.

&out("Getting tasks\n", 3);

$alltasks = &getcategory('Category:Programming Tasks');

# Get a complete listing of the languages.

&out("Getting the languages.\n", 3);

$alllanguages= &getcategory('Category:Programming Languages');

# Quick check. Did we add or lose any tasks? If so, we've got to recalc *all*

# of the pages. :-/

unless(exists $options{'rebuild_all'}) {

my $cacheddata = &getcacheddata('Category:Programming Tasks');

my ($added, $removed) = &diffcat($alltasks, $cacheddata);

$task_count_change = scalar @$added + scalar @$removed;

}

else {

$task_count_change = 1; # It just needs to be nonzero...

}

# We want the language name, not the fully-qualified wiki name.

$_ =~ s/^Category:// foreach (@$alllanguages);

# Get a list of the languages for which we've already provided bodies for the related omit categories.

# Store it as a hash, so the lookup will be faster.

my $omitcatcontents = &getcategory('Category:Maintenance/OmitCategoriesCreated');

my %createdomitcategories = map {$_, 1} @$omitcatcontents;

&out("Identifying work to do\n", 3);

foreach my $lang (@$alllanguages) {

my $val = &getlangwork($lang);

next unless defined $val;

$impldiff{$lang} = $val;

}

&processimplediff(\%impldiff);

my $runtime = time() - $starttime;

my $statsdata = "Pages edited last run: $pagesedited<br/>Time to post all per-language updates: $runtime seconds<br/>Category pulls: $categorypulls<br/>Cache hits: $cachehits<br/>";

&out("Updating stats page. Runtime ($runtime), Pages edited ($pagesedited)\n", 2);

&postpage("User:ImplSearchBot/Stats", $statsdata, "Updating stats data", 0);

&out("Updating bot code page\n", 4);

open my $sourcefile, '<', $0

or die "Finished without updating bot source page";

my $botsource;

$botsource .= $_ while <$sourcefile>;

close $sourcefile;

my $tag = "lang";

&postpage("User:ImplSearchBot/Code", "<$tag perl>$botsource</$tag>", 0);

&out("Updating cache\n", 4);

&commitcache();

&out("Done\n", 3);

exit 0

}

if (!caller) { exit main }