Anonymous user
User:ImplSearchBot/Code: Difference between revisions
m
ImplSearchBot:0
m (ImplSearchBot:0) |
m (ImplSearchBot:0) |
||
Line 4:
use MediaWiki::Bot;
use JSON qw/to_json from_json/;
our $editor;
our $starttime = time;
our $pagesedited = 0;
our $categorypulls = 0;
our $cachehits = 0;
our $task_count_change;
our $alltasks;
our %impldiff;
our %createdomitcategories;
our @initialcache;
our $alllanguages;
my %options;
#---------------------
Line 189 ⟶ 29:
sub build_posting
{
}
# Prepares the template body for the unimplemented data.
sub prep_unimp_posting
{
{
my $escapedImplPage = $implpage;
$escapedImplPage =~ s/ /_/g;
my $link;
# if($taskname eq $baretaskname)
# {
$link = "[http://rosettacode.org/mw/index.php?action=edit&title=$escapedImplPage $implpage]";
# }
# else
# {
# $link = "[[:$taskname|$baretaskname]]";
# }
# Add the task to the unimplemented list, if it's unimplemented, and if it's not in the omit list.
unless(exists $impldiff->{$language}->{'impl'}->{$implpage} or exists $impldiff->{$language}->{'omit'}->{$implpage})
{
$unimplisting .= "* $link\n";
}
}
return &build_posting("Template:" . &get_unimpl_template_name($language), $unimplisting);
}
Line 237 ⟶ 80:
sub prep_omit_posting
{
{
$escapedTaskName =~ s/ /_/g;
$omitlisting .= "* [http://rosettacode.org/mw/index.php?action=edit&title=$escapedTaskName $baretaskname]\n"
if(exists $impldiff{$language}->{'omit'}->{$taskname})
}
# Note that there's no data in the template.
$implediff->{$language}->{'omit'} = 0
return &build_posting("Template:$omittemplatename", $omitlisting);
}
Line 262 ⟶ 107:
sub prep_listing_posting
{
unless ($targetcount == 0);
{
}
}
Line 297 ⟶ 142:
sub prep_omit_cat_posting
{
}
Line 305 ⟶ 150:
sub get_listing_name
{
}
Line 312 ⟶ 157:
sub get_omit_template_name
{
}
Line 319 ⟶ 164:
sub get_unimpl_template_name
{
}
Line 326 ⟶ 171:
sub get_omit_cat_name
{
}
Line 333 ⟶ 178:
sub process_language
{
{
}
return \%langpostings;
}
sub wikitxt_pathname {
my $page_name = $_[0];
"test/" . sanitizenamefs($page_name) . ".wikitxt";
}
Line 356 ⟶ 206:
sub processimplediff
{
{
}
# Now that we've prepared all our work, commit it.
foreach my $pagename (keys %work)
{
&postpage($pagename, $work{$pagename}, "Updating $pagename", 0);
}
}
# Return the Mediawiki editor
sub geteditor
{
{
}
}
Line 413 ⟶ 262:
sub out
{
{
{
}
{
}
}
}
Line 436 ⟶ 285:
sub sanitizenamefs
{
}
Line 444 ⟶ 293:
sub diffcat_simple
{
{
}
}
Line 462 ⟶ 311:
sub diffcat
{
}
sub slurp_file {
my $pathname = $_[0];
open(my $stream, "<", $pathname) or do {warn"Can't open '$pathname': $!\n";
return};
my $ret;
while (1) {
my $len = read($stream, my $buf, 1024);
if (!defined($len)) { die "I/O error while reading '$pathname': $!"; }
elsif ($len == 0) { last }
else { $ret .= $buf }
}
$ret;
}
Line 472 ⟶ 335:
sub postpage
{
{
$pagename = wikitxt_pathname($pagename);
# save it to disk, and out of the way.
&out("Saving: $pagename\n", 2);
open my $outfile, '>', $pagename
or &out("Failed to open $pagename: $!", 1);
return unless defined $outfile;
print $outfile $pagedata;
close $outfile;
}
else {
$editor->edit($pagename, $pagedata, "ImplSearchBot:$remark", $minoredit)
or &out("Failed to post page: " . $editor->{'errstr'}, 1);
}
}
# This allows us to compare site state between now and when we
# last ran, so that we can learn to avoid doing unnecessary work.
Line 537 ⟶ 398:
return;
}
print $outfile to_json($data);
close $outfile;
&out(scalar $data . " members cached to $filename.\n", 5);
}
# Return data we cached previously.
sub getcacheddata
Line 555 ⟶ 416:
return [];
}
my $jsondata;
$jsondata .= $_ while <$infile>;
close $infile;
my $cacheddata = from_json($jsondata);
&out(scalar @$cacheddata . " cache members retrieved from $filename\n", 5);
++$cachehits;
return $cacheddata;
}
# Report the changes between two categories.
# More interesting than "x added, y removed"
Line 574 ⟶ 435:
my $old = shift;
my $new = shift;
my ($removed, $added) = &diffcat($old, $new);
my $out = "";
$out .= "Removed from $category:\n"
if( scalar @$removed > 0 );
$out .= "$_\n"
foreach (@$removed);
$out .= "Added to $category:\n"
if( scalar @$added > 0 );
$out .= "$_\n"
foreach (@$added);
&out($out, 2)
if("" ne $out);
}
# Pull the category data, or cached data if we're not pulling from the wiki.
sub getcategory
Line 598 ⟶ 459:
my $categoryname = shift;
&out("Getting category contents for $categoryname...", 4);
my $old;
my $new;
if( exists $options{'cacheonly'} )
{
Line 615 ⟶ 476:
sleep 1;
}
# &reportcatchanges($categoryname, $old, $new);
&cachedata("$categoryname", $new);
&out(scalar @$new . " members returned for $categoryname\n", 5);
return $new;
}
# Find if this category changed, report its contents if it has.
sub getwork
Line 631 ⟶ 492:
&out("Getting work for $categoryname.\n",4);
my $cacheddata = &getcacheddata($categoryname);
# If we're on a cache-only basis, we'll just say we have no old data,
# and that our cached data is our new data.
Line 646 ⟶ 507:
$newdata = &getcategory($categoryname);
}
my ($removed, $added) = &diffcat($olddata, $newdata);
my $impl_count_change = scalar @$removed + @$added;
Line 652 ⟶ 513:
$work = $newdata
if(($impl_count_change + $task_count_change) > 0);
&out(scalar @$work . " items to process for $categoryname.\n", 3);
return @$work;
}
# If changes occurred, the info is more important than if they didn't.
sub getloglevelfromdiff
Line 668 ⟶ 529:
return $base + 1;
}
# Find all the work items for a given language.
sub getlangwork
Line 684 ⟶ 545:
$omit{$workitem} = 1;
}
# No work to do?
return undef
if( ( 0 == scalar keys %omit) && (0 == scalar keys %impl));
# Language metadata
my $unimpcount = scalar @$alltasks - scalar keys %impl;
my $targetcount = (scalar @$alltasks - scalar keys %omit);
# work to do.
return {'impl' => \%impl,
Line 700 ⟶ 561:
'target_count' => $targetcount };
}
# Commit the cache.
sub commitcache
Line 707 ⟶ 568:
my $cachepath = $options{'cachepath'};
opendir(CACHEDIR, $cachepath);
my @current = readdir(CACHEDIR);
close(CACHEDIR);
# We need to run svn adds if we've created any new files.
# Maybe we'll use SVN::Client some day. Not right now.
my ($added, $removed) = &diffcat(\@current, \@initialcache);
&out("Detected " . scalar @$added . " new cache files and " . scalar @$removed . " removed\n",&getloglevelfromdiff(2, $added, $removed));
if ((scalar @$added + scalar @$removed ) > 0)
{
Line 723 ⟶ 584:
&svn('add', $cachepath . $cachefile);;
}
foreach my $cachefile (@$removed)
{
Line 729 ⟶ 590:
}
}
&svn('ci', '--message="ImplSearchBot run"', $cachepath);
&svn('update', $cachepath);
}
# Wrap svn commands so we can log them.
sub svn
Line 741 ⟶ 602:
if(exists $options{'nosvn'});
my @args = @_;
my $string = "system 'svn'";
$string .= ", '$_'"
foreach (@args);
$string .= "\n";
&out($string, 3);
system 'svn', @args
if(exists $options{'post'});
}
sub getopt {
#I don't care to pollute my global scope.
my $wiki = 'rosettacode.org';
my $username;
my $password;
my $verbosity = 2; # verbosity level. 0 is silent. 1 is error only. 2 is updates. 3 is process, more is noisy.
my $post; # Is this an actual run?
my $cachepath = "/tmp/";
my $cacheonly; # Don't query the wiki for data. Just pull from cache.
my $nosvn;
my $rebuild_all;
my $opt_matrix = {
"wiki=s" => \$wiki,
"username=s" => \$username,
"password=s" => \$password,
"verbosity=s" => \$verbosity,
"post" => \$post,
"cacheonly" => \$cacheonly,
"nosvn" => \$nosvn,
"cachepath=s" => \$cachepath,
"rebuildall" => \$rebuild_all };
my $result = GetOptions( %$opt_matrix );
$options{'wiki'} = $wiki;
$options{'nosvn'} = $nosvn
if defined $nosvn;
$options{'username'} = $username
if defined $username;
$options{'password'} = $password
if defined $password;
$options{'post'} = $post
if defined $post;
$options{'verbosity'} = $verbosity;
$cachepath .= '/'
if('/' ne substr($cachepath, -1, 1));
$options{'cachepath'} = $cachepath;
$options{'cacheonly'} = $cacheonly
if defined $cacheonly;
$options{'rebuild_all'} = $rebuild_all
if defined $rebuild_all;
my $usage = "Usage: $0 (options)\n The available options are:\n";
$usage .= "\t--$_\n"
foreach (keys %$opt_matrix);
$usage .= "Username and password are required if you need to pull data from the wiki. Wiki defaults to Rosetta Code.\n";
my $wikineeded;
$wikineeded = "yes"
unless ( exists $options{'cacheonly'} );
$wikineeded = "yes"
if ( exists $options{'post'} );
if(defined $wikineeded)
{
unless(exists $options{'username'} and exists $options{'password'})
{
die $usage;
}
}
}
sub main {
unless (@ARGV) { @ARGV = @_; }
getopt();
#Statistic tracking.
# Tracking for svn checkin at end.
# We *should* be the only ones writing to the cache path for now.
# Anyone else should have their own checkout of the data,
# and test runs are done with --cacheonly and --nosvn.
opendir(my $cachedir, $options{'cachepath'})
or die "Unable to open cache directory";
@initialcache = readdir($cachedir);
closedir($cachedir);
&out(scalar @initialcache . " categories initially cached\n", 4);
# Get our editor
$editor = &geteditor();
# Get a complete listing of the tasks.
&out("Getting tasks\n", 3);
$alltasks = &getcategory('Category:Programming Tasks');
# Get a complete listing of the languages.
&out("Getting the languages.\n", 3);
$alllanguages= &getcategory('Category:Programming Languages');
# Quick check. Did we add or lose any tasks? If so, we've got to recalc *all*
# of the pages. :-/
unless(exists $options{'rebuild_all'}) {
my $cacheddata = &getcacheddata('Category:Programming Tasks');
my ($added, $removed) = &diffcat($alltasks, $cacheddata);
$task_count_change = scalar @$added + scalar @$removed;
}
else {
$task_count_change = 1; # It just needs to be nonzero...
}
# We want the language name, not the fully-qualified wiki name.
$_ =~ s/^Category:// foreach (@$alllanguages);
# Get a list of the languages for which we've already provided bodies for the related omit categories.
# Store it as a hash, so the lookup will be faster.
my $omitcatcontents = &getcategory('Category:Maintenance/OmitCategoriesCreated');
my %createdomitcategories = map {$_, 1} @$omitcatcontents;
&out("Identifying work to do\n", 3);
foreach my $lang (@$alllanguages) {
my $val = &getlangwork($lang);
next unless defined $val;
$impldiff{$lang} = $val;
}
&processimplediff(\%impldiff);
my $runtime = time() - $starttime;
my $statsdata = "Pages edited last run: $pagesedited<br/>Time to post all per-language updates: $runtime seconds<br/>Category pulls: $categorypulls<br/>Cache hits: $cachehits<br/>";
&out("Updating stats page. Runtime ($runtime), Pages edited ($pagesedited)\n", 2);
&postpage("User:ImplSearchBot/Stats", $statsdata, "Updating stats data", 0);
&out("Updating bot code page\n", 4);
open my $sourcefile, '<', $0
or die "Finished without updating bot source page";
my $botsource;
$botsource .= $_ while <$sourcefile>;
close $sourcefile;
my $tag = "lang";
&postpage("User:ImplSearchBot/Code", "<$tag perl>$botsource</$tag>", 0);
&out("Updating cache\n", 4);
&commitcache();
&out("Done\n", 3);
exit 0
}
if (!caller) { exit main }
</lang>
|