View source for User:PeerInfinity/Scripts/SyncArticleLinks.php

<?php

// SyncArticleLinks.php
// synchronize the list of article links on the concept pages with the list of concepts on the "All Articles" pages

// for the latest version of this script's output, see:

//[[/stderr output]] - the stderr output of this script

//[[/SyncArticleLinksOutput.txt]] - the output written to the text file, containing the Sync results

//<pre>

$stderr = fopen( "php://stderr", "wt" );

//*

$XMLfile = "daily_XML_dump.xml";


// the arrays for the data read from the All Articles pages:

// it would be more proper to create a struct for this, but for now I'll just do it the quick and dirty way

$NextAllArticleIndex    = 0;

// these arrays are indexed by $NextAllArticleIndex

$ArrayAllArticleTitle              = array();
$ArrayAllArticleLink               = array();
$ArrayAllArticleIndexedConcepts    = array(); // this is an array of arrays!
$ArrayAllArticleFoundConcepts      = array(); // this is an array of arrays!
$ArrayAllArticleNotIndexedConcepts = array(); // this is an array of arrays!
$ArrayAllArticleAuthor             = array();
$ArrayAllArticleDate               = array(); // currently unused, This would need to be read by following the link to the article, which might be a good idea to implement eventually
$ArrayAllArticleOfficialSummaries  = array(); // this is an array of arrays!  //this one is indexed by article name!
$ArrayAllArticleUsedSummaries      = array(); // this is an array of arrays!


// the array for all Concepts found

// these arrays are indexed by concept title

$ConceptFound = array();
$ConceptNotInIndex = array();
$ConceptThatAreRedirects = array();
$ConceptSeeAlso = array(); // this is an array of arrays!

// these arrays are indexed by concept title

$PagesWithOvercomingBiasLinks = array();
$PagesWithComments = array();
$PagesWithOvercomingBiasArticlesHeader = array();
$PagesWithExternalReferences = array();
$PagesWithSeeAlso = array();
$PagesWithExternalAuthorLinks = array();
$PagesWithNewlineAfterWikiLink = array();
$PagesWithSeeAlsoBeforeBlogPosts = array();

// ugh... this is ugly:

// these arrays are indexed by the index variables below

$ArticleLinksWithoutEndingSlash = array();
$ArticleLinksWithWrongTitle = array();
$ArticleLinksWithoutAuthor = array();
$ArticleLinksWithAvailableSummary = array();

$ArticleLinksWithoutEndingSlashConcept = array();
$ArticleLinksWithWrongTitleConcept = array();
$ArticleLinksWithoutAuthorConcept = array();
$ArticleLinksWithAvailableSummaryConcept = array();

$ArticleLinksWithoutEndingSlashNextIndex = 0;
$ArticleLinksWithWrongTitleNextIndex = 0;
$ArticleLinksWithoutAuthorConceptNextIndex = 0;
$ArticleLinksWithAvailableSummaryIndex = 0;


$CurrentTitle = "";
$CurrentBody = "";

$IsReadingTitle      = false;
$IsReadingText       = false;

$IsReadingAllArticlesPages = false;
$IsReadingSummariesPages   = false;
$IsReadingConceptPages     = false;

$IsReadingOneAllArticlesPage = false;
$IsReadingOneSummariesPage   = false;
$IsReadingOneConceptPage     = false;

$FirstYearToRead = 2006;
$FinalYearToRead = 2010; //todo - update this in 2011!!!

$NextYearToRead = $FirstYearToRead;

$YearCurrentlyBeingRead = $FirstYearToRead;

$SuccessfullyReadOneAllArticlesPage = false;
$SuccessfullyReadOneSummariesPage   = false;
$FailedToReadAllArticlesPage        = false;

$SuccessfullyReadOneConceptPage = false;
$FailedToReadConceptPage        = false;

$PagesRead = 0;

$Debug = true;


// for doxygen
$fontsize = 12;

// to make the script easier to read, pad each section to a specific number of characters, to make everything line up
$PaddingValue1 = 100;
$PaddingValue2 = 50;




//open a text file for the output of this script
//the script will send its output both to this text file and to stdout (or was it stderr?)

$fp = fopen('SyncArticleLinksOutput.txt', 'w');





// now for some code that I still haven't figured out a good way to untangle
// this processing currently needs to be done in these functions, and can't be moved to a more sensible place


function startElement($parser, $name, $attribs)
{
	global $CurrentTitle ;
	global $CurrentBody  ;
	
	global $IsReadingTitle      ;
	global $IsReadingText       ;
	
	global $stderr;
	
	// remember what element we're reading, so that we know what to do in characterData()
	
	if( $name == "TITLE" )
	{
		$IsReadingTitle = true;
	}
	else
	{
		$IsReadingTitle = false;
	}
	
	if( $name == "TEXT" )
	{
		$IsReadingText = true;
	}
	else
	{
		$IsReadingText = false;
	}
}

function characterData($parser, $data)
{
	global $stderr;

	global $XMLfile;
	
	global $NextAllArticleIndex;

	global $ArrayAllArticleTitle             ;
	global $ArrayAllArticleLink              ;
	global $ArrayAllArticleIndexedConcepts   ;
	global $ArrayAllArticleFoundConcepts     ;
	global $ArrayAllArticleNotIndexedConcepts;
	global $ArrayAllArticleAuthor            ;
	global $ArrayAllArticleDate              ;
	global $ArrayAllArticleOfficialSummaries ;
	global $ArrayAllArticleUsedSummaries     ;

	global $ConceptFound;
	global $ConceptNotInIndex;
	global $ConceptThatAreRedirects;
	global $ConceptSeeAlso;
	
	global $PagesWithOvercomingBiasLinks;
	global $PagesWithComments;
	global $PagesWithOvercomingBiasArticlesHeader;
	global $PagesWithExternalReferences;
	global $PagesWithSeeAlso;
	global $PagesWithExternalAuthorLinks;
	global $PagesWithNewlineAfterWikiLink;
	global $PagesWithSeeAlsoBeforeBlogPosts;

	global $ArticleLinksWithoutEndingSlash;
	global $ArticleLinksWithWrongTitle;
	global $ArticleLinksWithoutAuthor;
	global $ArticleLinksWithAvailableSummary;

	global $ArticleLinksWithoutEndingSlashConcept;
	global $ArticleLinksWithWrongTitleConcept;
	global $ArticleLinksWithoutAuthorConcept;
	global $ArticleLinksWithAvailableSummaryConcept;

	global $ArticleLinksWithoutEndingSlashNextIndex;
	global $ArticleLinksWithWrongTitleNextIndex;
	global $ArticleLinksWithoutAuthorNextIndex;
	global $ArticleLinksWithAvailableSummaryNextIndex;

	global $CurrentTitle;
	global $CurrentBody  ;

	global $IsReadingTitle;
	global $IsReadingText ;

	global $IsReadingAllArticlesPages;
	global $IsReadingSummariesPages  ;
	global $IsReadingConceptPages    ;

	global $IsReadingOneAllArticlesPage;
	global $IsReadingOneSummariesPage  ;
	global $IsReadingOneConceptPage    ;

	global $FirstYearToRead;
	global $FinalYearToRead;

	global $NextYearToRead;

	global $YearCurrentlyBeingRead;

	global $SuccessfullyReadOneAllArticlesPage;
	global $SuccessfullyReadOneSummariesPage  ;
	global $FailedToReadAllArticlesPage       ;

	global $SuccessfullyReadOneConceptPage;
	global $FailedToReadConceptPage;
	
	global $PagesRead;
	
	global $Debug;





	//open a text file for the output of this script
	//the script will send its output both to this text file and to stdout (or was it stderr?)

	$fp = fopen('SyncArticleLinksOutput.txt', 'w');



	
	
	
	// if we're reading the title, then remember the title, and check if we want to read the page content
	
	if( $IsReadingTitle )
	{
		$CurrentBody = "";
		
		$CurrentTitle = $data;
		$IsReadingTitle = false;

		$IsReadingOneAllArticlesPage = false;
		$IsReadingOneSummariesPage   = false;
		$IsReadingOneConceptPage     = false;
		
		// skip category pages, template pages, etc.

		if( substr_count($CurrentTitle, "Category:") > 0 )
		{
		}
		else if( substr_count($CurrentTitle, "Template:") > 0 )
		{
		}
		else if( substr_count($CurrentTitle, "Talk:") > 0 )
		{
		}
		else if( substr_count($CurrentTitle, "Category talk:") > 0 )
		{
		}
		else if
			(
				$CurrentTitle == "Catch Phrases" ||
				$CurrentTitle == "Categories" ||
				$CurrentTitle == "Chat Logs/2009-04-11" ||
				$CurrentTitle == "Disagreements on Less Wrong" ||
				$CurrentTitle == "Series" ||
				$CurrentTitle == "Using the wiki" ||
				$CurrentTitle == "Acronyms used on Less Wrong" ||
				$CurrentTitle == "Less Wrong/Errors from moving Eliezer's posts from OB to LW" ||
				false
			)
		{
		}
		else
		{
			if( $IsReadingAllArticlesPages && $NextYearToRead <= $FinalYearToRead )
			{
				// check if the page title is the next year we want to read
				if
					(
						( $NextYearToRead == 2006 && substr_count($CurrentTitle, "Less Wrong/2006 Articles") > 0 ) ||
						( $NextYearToRead == 2007 && substr_count($CurrentTitle, "Less Wrong/2007 Articles") > 0 ) ||
						( $NextYearToRead == 2008 && substr_count($CurrentTitle, "Less Wrong/2008 Articles") > 0 ) ||
						( $NextYearToRead == 2009 && substr_count($CurrentTitle, "Less Wrong/2009 Articles") > 0 ) ||
						( $NextYearToRead == 2010 && substr_count($CurrentTitle, "Less Wrong/2010 Articles") > 0 )
						//todo - update this in 2011!!!
					)
				{
					$YearCurrentlyBeingRead = $NextYearToRead;
					$NextYearToRead++;
					$IsReadingOneAllArticlesPage = true;

					fwrite( $stderr, "Processing the wikipage for $YearCurrentlyBeingRead \n\n" );
				}
			}
			else if( $IsReadingSummariesPages && $NextYearToRead <= $FinalYearToRead )
			{
				// check if the page title is the next year we want to read
				if
					(
						( $NextYearToRead == 2006 && substr_count($CurrentTitle, "Less Wrong/2006 Articles/Summaries") > 0 ) ||
						( $NextYearToRead == 2007 && substr_count($CurrentTitle, "Less Wrong/2007 Articles/Summaries") > 0 ) ||
						( $NextYearToRead == 2008 && substr_count($CurrentTitle, "Less Wrong/2008 Articles/Summaries") > 0 ) ||
						( $NextYearToRead == 2009 && substr_count($CurrentTitle, "Less Wrong/2009 Articles/Summaries") > 0 ) ||
						( $NextYearToRead == 2010 && substr_count($CurrentTitle, "Less Wrong/2010 Articles/Summaries") > 0 )
						//todo - update this in 2011!!!
					)
				{
					$YearCurrentlyBeingRead = $NextYearToRead;
					$NextYearToRead++;
					$IsReadingOneSummariesPage = true;

					fwrite( $stderr, "Processing the summaries page for $YearCurrentlyBeingRead \n\n" );
				}
			}
			else if( $IsReadingConceptPages )
			{
				// don't process the All Articles pages!
				if
					(
						substr_count($CurrentTitle, "Less Wrong/2006 Articles") <= 0 &&
						substr_count($CurrentTitle, "Less Wrong/2007 Articles") <= 0 &&
						substr_count($CurrentTitle, "Less Wrong/2008 Articles") <= 0 &&
						substr_count($CurrentTitle, "Less Wrong/2009 Articles") <= 0 &&
						substr_count($CurrentTitle, "Less Wrong/2010 Articles") <= 0
						//todo - update this in 2011!!!
					)
				{
					// no special processing here, just remember the title

					$IsReadingOneConceptPage     = true;


					if( $PagesRead % 100 == 0 )
					{
						fwrite( $stderr, "Pages read: $PagesRead\n" );
					}

					//fwrite( $stderr, "Found the article $CurrentTitle \n\n" );

					$PagesRead++;
				}
			}
			else if( $IsReadingAllArticlesPages )
			{
				//fwrite( $stderr, "reading All Articles pages, past the final year\n\n" );
			}
			else if( $IsReadingSummariesPages )
			{
				//fwrite( $stderr, "reading summary pages, past the final year\n\n" );
			}
			else
			{
				fwrite( $stderr, "Error: not reading all articles, summaries, or concept pages\n\n" );
			}
		}
	}
	
	// if we're reading the text, then store the content of the article
	// we'll process the data in endElement()
	
	// there is probably a more efficient way to do this
	
	if( $IsReadingText )
	{
		if( $IsReadingOneAllArticlesPage )
		{
			$CurrentBody .= $data;
		}
		
		if( $IsReadingOneSummariesPage )
		{
			$CurrentBody .= $data;
		}
		
		if( $IsReadingOneConceptPage )
		{
			$CurrentBody .= $data;
		}
	}
}

function endElement($parser, $name)
{
	global $stderr;

	global $XMLfile;
	
	global $NextAllArticleIndex;

	global $ArrayAllArticleTitle             ;
	global $ArrayAllArticleLink              ;
	global $ArrayAllArticleIndexedConcepts   ;
	global $ArrayAllArticleFoundConcepts     ;
	global $ArrayAllArticleNotIndexedConcepts;
	global $ArrayAllArticleAuthor            ;
	global $ArrayAllArticleDate              ;
	global $ArrayAllArticleOfficialSummaries ;
	global $ArrayAllArticleUsedSummaries     ;

	global $ConceptFound;
	global $ConceptNotInIndex;
	global $ConceptThatAreRedirects;
	global $ConceptSeeAlso;
	
	global $PagesWithOvercomingBiasLinks;
	global $PagesWithComments;
	global $PagesWithOvercomingBiasArticlesHeader;
	global $PagesWithExternalReferences;
	global $PagesWithSeeAlso;
	global $PagesWithExternalAuthorLinks;
	global $PagesWithNewlineAfterWikiLink;
	global $PagesWithSeeAlsoBeforeBlogPosts;

	global $ArticleLinksWithoutEndingSlash;
	global $ArticleLinksWithWrongTitle;
	global $ArticleLinksWithoutAuthor;
	global $ArticleLinksWithAvailableSummary;

	global $ArticleLinksWithoutEndingSlashConcept;
	global $ArticleLinksWithWrongTitleConcept;
	global $ArticleLinksWithoutAuthorConcept;
	global $ArticleLinksWithAvailableSummaryConcept;

	global $ArticleLinksWithoutEndingSlashNextIndex;
	global $ArticleLinksWithWrongTitleNextIndex;
	global $ArticleLinksWithoutAuthorNextIndex;
	global $ArticleLinksWithAvailableSummaryNextIndex;

	global $CurrentTitle;
	global $CurrentBody  ;

	global $IsReadingTitle;
	global $IsReadingText ;

	global $IsReadingAllArticlesPages;
	global $IsReadingSummariesPages  ;
	global $IsReadingConceptPages    ;

	global $IsReadingOneAllArticlesPage;
	global $IsReadingOneSummariesPage  ;
	global $IsReadingOneConceptPage    ;

	global $FirstYearToRead;
	global $FinalYearToRead;

	global $NextYearToRead;

	global $YearCurrentlyBeingRead;

	global $SuccessfullyReadOneAllArticlesPage;
	global $SuccessfullyReadOneSummariesPage ;
	global $FailedToReadAllArticlesPage      ;

	global $SuccessfullyReadOneConceptPage;
	global $FailedToReadConceptPage;
	
	global $PagesRead;
	
	global $Debug;
	
	
	// now process all the text that we read
	
	if( $name == "TEXT" )
	{
		// this check is probably unnecessary
		if( $IsReadingText )
		{
			if( $IsReadingOneAllArticlesPage )
			{
				//fwrite( $stderr, "-------------------------------------------reading body of year $YearCurrentlyBeingRead , $CurrentTitle \n\n" );

				//fwrite( $stderr, $CurrentBody );
				
				// now parse the data from this page and store it into the arrays
				
				$CurrentGroupStartPos = 0;
				
				// skip to the first '|-'
				$NextGroupStartPos = strpos( $CurrentBody, "|-", $CurrentGroupStartPos+1 );
				
				// keep going until there are no more groups
				while( $NextGroupStartPos !== FALSE && $CurrentGroupStartPos != $NextGroupStartPos )
				{
					// find the start and end of teh current group
					$CurrentGroupStartPos = $NextGroupStartPos;
					
					$NextGroupStartPos = strpos( $CurrentBody, "|-", $CurrentGroupStartPos+1 );
					
					// if there is no next group, then set the end of this group to the end of the whole string
					if( $NextGroupStartPos === FALSE )
					{
						$NextGroupStartPos = strlen( $CurrentBody ) - 1;
					}
					
					// now find and store the parts
					
					// there is probably a much simpler way to do all this using regexes
					
					$DataIsValid = true;
					
					$CurrentArticleConceptArray = array();
					
					$TokenStartPos = strpos($CurrentBody, "[", $CurrentGroupStartPos) + 1;
					$TokenEndPos = strpos($CurrentBody, " ", $TokenStartPos);
					
					$CurrentArticleLink = substr( $CurrentBody, $TokenStartPos, $TokenEndPos-$TokenStartPos );
					
					if( substr_count( $CurrentArticleLink, "http://" ) <= 0 )
					{
						$DataIsValid = false;
					}


					$TokenStartPos = $TokenEndPos + 1;
					$TokenEndPos = strpos($CurrentBody, "]", $TokenStartPos);
					
					$CurrentArticleTitle = substr( $CurrentBody, $TokenStartPos, $TokenEndPos-$TokenStartPos );
					
					$CurrentArticleTitle = str_replace("&quot;", "\"", $CurrentArticleTitle);

					
					$NextLineStartPos = strpos($CurrentBody, "|", $TokenEndPos);
					$NextLineEndPos = strpos($CurrentBody, "|", $NextLineStartPos+1);
					
					$ConceptLine = substr( $CurrentBody, $NextLineStartPos, $NextLineEndPos-$NextLineStartPos );

					//fwrite( $stderr, "concept line start: $NextLineStartPos \n concept line end: $NextLineEndPos \n concept line: $ConceptLine\n\n" );
					
					$FinishedFindingConcepts = false;
					
					$ConceptIndex = 0;

					$TokenStartPos = 0;
					$TokenEndPos = 0;
					
					while( !$FinishedFindingConcepts )
					{
						$TokenStartPos = strpos($ConceptLine, "[[", $TokenEndPos);
						$TokenEndPos = strpos($ConceptLine, "]]", $TokenStartPos);
						
						$CurrentConcept = substr( $ConceptLine, $TokenStartPos+2, $TokenEndPos-$TokenStartPos-2 );

						if
							(
								$TokenEndPos > $TokenStartPos &&
								substr_count($CurrentConcept, "[[") <= 0 &&
								substr_count($CurrentConcept, "]]") <= 0
								)
						{
							//fwrite( $stderr, "found concept: $CurrentConcept\n\n" );
							
							$CurrentArticleConceptArray[$ConceptIndex] = $CurrentConcept;
							
							$ConceptIndex++;
							
							$FinishedFindingConcepts = false;
						}
						else
						{
							$FinishedFindingConcepts = true;
						}

						//fwrite( $stderr, "concept start: $TokenStartPos \n concept end: $TokenEndPos \n concept index: $ConceptIndex \n concept line: $ConceptLine\n\n" );
					}

					$NextLineStartPos = strpos($CurrentBody, "|", $NextLineEndPos);
					$NextLineEndPos = strpos($CurrentBody, "|", $NextLineStartPos+1);

					$TokenStartPos = strpos($CurrentBody, "[", $NextLineStartPos);
					$TokenEndPos = strpos($CurrentBody, "]", $TokenStartPos);

					$CurrentArticleAuthor = substr( $CurrentBody, $TokenStartPos, $TokenEndPos-$TokenStartPos+1 );
					
					if( $DataIsValid )
					{
						$ArrayAllArticleTitle              [$NextAllArticleIndex] = $CurrentArticleTitle;

						$ArrayAllArticleLink               [$NextAllArticleIndex] = $CurrentArticleLink;

						//$CurrentArticleConceptArray[] = "concept1";

						$ArrayAllArticleIndexedConcepts    [$NextAllArticleIndex] = $CurrentArticleConceptArray;

						$ArrayAllArticleFoundConcepts      [$NextAllArticleIndex] = array();

						$ArrayAllArticleNotIndexedConcepts [$NextAllArticleIndex] = array();

						$ArrayAllArticleAuthor             [$NextAllArticleIndex] = $CurrentArticleAuthor;

						$ArrayAllArticleDate               [$NextAllArticleIndex] = "date";

						$ArrayAllArticleUsedSummaries      [$NextAllArticleIndex] = array();

						//$TempString = print_r($ArrayAllArticleIndexedConcepts[$NextAllArticleIndex]);

						//fwrite( $stderr, "start: $CurrentGroupStartPos \n end: $NextGroupStartPos \n index: $NextAllArticleIndex \n $ArrayAllArticleTitle[$NextAllArticleIndex] \n $ArrayAllArticleLink[$NextAllArticleIndex] \n $TempString \n $ArrayAllArticleAuthor[$NextAllArticleIndex] \n $ArrayAllArticleDate[$NextAllArticleIndex] \n\n" );

						$NextAllArticleIndex++;
					}
				}

				$SuccessfullyReadOneAllArticlesPage = true;
			}
			
			if( $IsReadingOneSummariesPage )
			{
				//fwrite( $stderr, "reading summaries page for $YearCurrentlyBeingRead \n\n" );

				$NextSummaryTitleStart = 0;
				$NextSummaryTitleEnd = 0;
				$NextSummaryTextBlockStart = 0;
				$NextSummaryTextBlockEnd = 0;
				$NextSummaryTextStart = 0;
				$NextSummaryTextEnd = 0;
				
				$AlternateSummaryCount = 0;
				
				$FinishedReadingSummaries = false;

				while( !$FinishedReadingSummaries )
				{
					$NextSummaryTitleStart = strpos($CurrentBody, "=====[", $NextSummaryTitleEnd);

					//fwrite( $stderr, "CurrentBody: $CurrentBody \n\n" );
					//fwrite( $stderr, "NextSummaryTitleStart: $NextSummaryTitleStart \n\n" );
				
					if( $NextSummaryTitleStart === FALSE )
					{
						$FinishedReadingSummaries = true;
					}
					else
					{
						$NextSummaryTitleStart += 6;
						
						$NextSummaryTitleEnd = strpos($CurrentBody, "]=====", $NextSummaryTitleStart);
						
						$FirstSpacePos = strpos($CurrentBody, " ", $NextSummaryTitleStart);
						
						$CurrentSummaryLink = substr( $CurrentBody, $NextSummaryTitleStart, $FirstSpacePos-$NextSummaryTitleStart );
						
						$CurrentSummaryTitle = substr( $CurrentBody, $FirstSpacePos+1, $NextSummaryTitleEnd-$FirstSpacePos-1 );

						$CurrentSummaryTitle = str_replace("&quot;", "\"", $CurrentSummaryTitle);

						//fwrite( $stderr, "reading summary of $CurrentSummaryTitle \n\n" );
						
						$NextSummaryTextBlockStart = $NextSummaryTitleEnd + 8;
						
						$NextSummaryTextBlockEnd = strpos($CurrentBody, "=====[", $NextSummaryTitleEnd);
						
						if( $NextSummaryTextBlockEnd === FALSE )
						{
							$NextSummaryTextBlockEnd = strlen($CurrentBody);
						}
						
						$NextSummaryTextBlock = substr( $CurrentBody, $NextSummaryTextBlockStart, $NextSummaryTextBlockEnd-$NextSummaryTextBlockStart );

						$AlternateSummaryCount = substr_count( $NextSummaryTextBlock, "(alternate summary:)" );

						$NextSummaryTextEnd = 0;
						
						$ArrayAllArticleOfficialSummaries[$CurrentSummaryTitle] = array();
						
						for( $AlternateSummaryNum = 0; $AlternateSummaryNum < $AlternateSummaryCount; $AlternateSummaryNum++ )
						{
							$NextSummaryTextStart = $NextSummaryTextEnd;
							$NextSummaryTextEnd = strpos( $NextSummaryTextBlock, "\n\n(alternate summary:)\n\n", $NextSummaryTextStart );
							
							$SummaryToAdd = substr( $NextSummaryTextBlock, $NextSummaryTextStart, $NextSummaryTextEnd-$NextSummaryTextStart );
							
							$ArrayAllArticleOfficialSummaries[$CurrentSummaryTitle][$AlternateSummaryNum] = $SummaryToAdd;
							
							$NextSummaryTextEnd += strlen( "\n\n(alternate summary:)\n\n" );
						}

						$NextSummaryTextStart = $NextSummaryTextEnd;
						
						$SummaryToAdd = substr( $NextSummaryTextBlock, $NextSummaryTextStart );
						
						$SummaryToAdd = trim( $SummaryToAdd, "\n" );
						
						if( strlen( $SummaryToAdd ) > 0 )
						{
							if( strpos( $SummaryToAdd, "__NOTOC__" ) !== FALSE )
							{
								//todo - decide how to deal with this case!
							}
							else
							{
								$ArrayAllArticleOfficialSummaries[$CurrentSummaryTitle][$AlternateSummaryCount] = $SummaryToAdd;
							}
						}
					}
				}



				$SuccessfullyReadOneSummariesPage = true;
			}

			if( $IsReadingOneConceptPage )
			{
				//fwrite( $stderr, "------reading body of $CurrentTitle \n\n" );
				
				// don't bother processing the page unless it has a "Blog posts" section

				$TokenStartPos = 0;
				$TokenEndPos = 0;

	
				// keep track of which pages have links to OvercomingBias.com articles
				
				if( substr_count( $CurrentBody, "http://www.overcomingbias.com/20" ) > 0 )
				{
					// the following pages are "allowed" to have OvercomingBias.com articles - these pages were already checked manually for links that should point to lesswrong.com instead
					if
						(
							$CurrentTitle == "Aumann's agreement theorem" ||
							$CurrentTitle == "Bias" ||
							$CurrentTitle == "Bite the bullet" ||
							$CurrentTitle == "Black swan" ||
							$CurrentTitle == "Catch Phrases" ||
							$CurrentTitle == "Chat Logs/2009-04-11" ||
							$CurrentTitle == "Cognitive style" ||
							$CurrentTitle == "Coherence" ||
							$CurrentTitle == "Connotation" ||
							$CurrentTitle == "Consistency" ||
							$CurrentTitle == "Cryonics" ||
							$CurrentTitle == "Dark arts" ||
							$CurrentTitle == "Disagreement" ||
							$CurrentTitle == "Disagreements on Less Wrong" ||
							$CurrentTitle == "Emotion" ||
							$CurrentTitle == "Extraordinary evidence" ||
							$CurrentTitle == "Forecast" ||
							$CurrentTitle == "Hypocrisy" ||
							$CurrentTitle == "Impossible world" ||
							$CurrentTitle == "Intellectual roles" ||
							$CurrentTitle == "Likelihood ratio" ||
							$CurrentTitle == "Meme lineage" ||
							$CurrentTitle == "Modesty argument" ||
							$CurrentTitle == "Near/far thinking" ||
							$CurrentTitle == "Overcoming Bias" ||
							$CurrentTitle == "Overconfidence" ||
							$CurrentTitle == "Prediction market" ||
							$CurrentTitle == "Scales of justice fallacy" ||
							$CurrentTitle == "Series" ||
							$CurrentTitle == "Signaling" ||
							$CurrentTitle == "Signalling" ||
							$CurrentTitle == "Status" ||
							$CurrentTitle == "Stereotype" ||
							false
						)
					{
						// do nothing
					}
					else
					{
						$PagesWithOvercomingBiasLinks[$CurrentTitle] = true;
					}
				}

				if( substr_count( $CurrentBody, "<!--" ) > 0 )
				{
					// the following pages are "allowed" to have comments
					if
						(
							$CurrentTitle == "LessWrong Wiki"
						)
					{
						// do nothing
					}
					else
					{
						$PagesWithComments[$CurrentTitle] = true;
					}
				}

				if( substr_count( $CurrentBody, "==Overcoming Bias Articles==" ) > 0 )
				{
					$PagesWithOvercomingBiasArticlesHeader[$CurrentTitle] = true;
				}

				if( substr_count( $CurrentBody, "External references" ) > 0 )
				{
					$PagesWithExternalReferences[$CurrentTitle] = true;
				}

				if( substr_count( $CurrentBody, "See Also" ) > 0 )
				{
					$PagesWithSeeAlso[$CurrentTitle] = true;
				}
				
				if( substr_count( $CurrentBody, "by [http" ) > 0 )
				{
					$PagesWithExternalAuthorLinks[$CurrentTitle] = true;
				}
				
				if( substr_count( $CurrentBody, "wikilink}}\n\n" ) > 0 )
				{
					$PagesWithNewlineAfterWikiLink[$CurrentTitle] = true;
				}
				
				$SeeAlsoPos = strpos($CurrentBody, "==See also==");
				$BlogPostsPos = strpos($CurrentBody, "==Blog posts==");
				
				if( $SeeAlsoPos !== FALSE && $BlogPostsPos !== FALSE )
				{
					if( $SeeAlsoPos < $BlogPostsPos )
					{
						$PagesWithSeeAlsoBeforeBlogPosts[$CurrentTitle] = true;
					}
				}
				

				if( substr_count( $CurrentBody, "#REDIRECT" ) > 0 )
				{
					if( array_key_exists( $CurrentTitle, $ConceptFound ) )
					{
						$ConceptThatAreRedirects[$CurrentTitle] = true;
					}
				}


				//todo - reconsider this!!!
				//if( substr_count( $CurrentArticleLink, "==Blog posts==" ) > 0 )
				if( substr_count( $CurrentBody, "http://lesswrong.com/lw/" ) > 0 )
				{
					// first check if the concept exists in the index

					$ConceptIsInIndex = false;

					if( array_key_exists( $CurrentTitle, $ConceptFound ) )
					{
						$ConceptIsInIndex = true;
					}

					if( $ConceptIsInIndex )
					{
						// if the concept exists in the index, then mark the concept as found
						
						$ConceptFound[$CurrentTitle] = true;
						
						//fwrite( $stderr, "++++++found: $CurrentTitle\n\n" );
					}
					else
					{
						// if the concept doesn't exist in the index, then mark the concept as not found
						
						$ConceptNotInIndex[$CurrentTitle] = true;
					}
					
					
					// now keep track of the "See also" links
					if( array_key_exists($CurrentTitle, $ConceptFound ) )
					{
						if( $ConceptFound[$CurrentTitle] == true )
						{
							$SeeAlsoStartPos = strpos($CurrentBody, "==See also==", 0);

							if( $SeeAlsoStartPos !== FALSE )
							{
								$SeeAlsoStartPos += 12;
							}
							else
							{
								$SeeAlsoStartPos = strpos($CurrentBody, "==Related concepts==", 0);

								if( $SeeAlsoStartPos !== FALSE )
								{
									$SeeAlsoStartPos += 20;
								}
								else
								{
								}
							}

							if( $SeeAlsoStartPos !== FALSE )
							{
								$SeeAlsoEndPos = strpos($CurrentBody, "==", $SeeAlsoStartPos);
								
								if( $SeeAlsoEndPos === FALSE )
								{
									$SeeAlsoEndPos = strlen($CurrentBody);
								}

								$TokenStartPos = $SeeAlsoStartPos;
								$TokenEndPos = $SeeAlsoStartPos;

								$DoneSeeAlso = false;

								$EntriesFound = 0;

								$ConceptSeeAlso[$CurrentTitle] = array();

								//fwrite( $stderr, "Concept: " . $CurrentTitle . "\n" );

								while( !$DoneSeeAlso )
								{
									$TokenStartPos = strpos($CurrentBody, "[[", $TokenEndPos);
									$TokenEndPos = strpos($CurrentBody, "]]", $TokenStartPos);

									if
										(
											$TokenStartPos < $SeeAlsoEndPos &&
											$TokenStartPos >= $SeeAlsoStartPos &&
											$TokenStartPos !== FALSE &&
											$TokenEndPos !== FALSE
										)
									{
										$CurrentSeeAlso = substr( $CurrentBody, $TokenStartPos+2, $TokenEndPos-$TokenStartPos-2 );
										
										if( substr_count( $CurrentSeeAlso, "Category:") <= 0 )
										{
											$ConceptSeeAlso[$CurrentTitle][$EntriesFound] = $CurrentSeeAlso;

											//fwrite( $stderr, $ConceptSeeAlso[$CurrentTitle][$EntriesFound] . "\n" );

											$EntriesFound++;
										}
									}
									else
									{
										$DoneSeeAlso = true;
									}

									//fwrite( $stderr, "SeeAlsoStartPos: $SeeAlsoStartPos SeeAlsoEndPos: $SeeAlsoEndPos TokenStartPos: $TokenStartPos TokenEndPos $TokenEndPos" . "\n" );
								}
							}
						}
					}
					


					$TokenStartPos = strpos($CurrentBody, "http://lesswrong.com/lw/", $TokenEndPos);
					$TokenEndPos = strpos($CurrentBody, " ", $TokenStartPos);

					$NextTokenStartPos = strpos($CurrentBody, "http://lesswrong.com/lw/", $TokenEndPos);
					
					if( $NextTokenStartPos === FALSE )
					{
						$NextTokenStartPos = strlen($CurrentBody);
					}

					$CurrentLink = substr( $CurrentBody, $TokenStartPos, $TokenEndPos-$TokenStartPos );
					
					$FinishedFindingLinks = false;

					while( !$FinishedFindingLinks )
					{
						// check if the link appears before or after the first header
						
						$FirstHeaderPos = strpos($CurrentBody, "==", 0);

						$LinkIsAfterHeader = false;

						if( $FirstHeaderPos === FALSE )
						{
							$LinkIsAfterHeader = false;
						}
						else
						{
							if( $FirstHeaderPos < $TokenStartPos )
							{
								$LinkIsAfterHeader = true;
							}
							else
							{
								$LinkIsAfterHeader = false;
							}
						}



						// ignore the following links:
						
						if
							(
								substr_count( $CurrentLink, "http://lesswrong.com/lw/b1/persuasiveness_vs_soundness/789") <= 0 &&
								substr_count( $CurrentLink, "http://lesswrong.com/lw/14v/the_usefulness_of_correlations/11iu") <= 0
							)

						{
							// check if the link ends with a /

							$LastCharInLink = substr( $CurrentLink, -1 );
							
							$SlashCountBeforeAdd = substr_count( $CurrentLink, "/");

							if
								(
									strcmp( $LastCharInLink, "/" ) != 0
								)
							{
								$CurrentLink .= "/";
							}



							// check if the link's title is correct

							$TitleStartPos = $TokenEndPos+1;
							$TitleEndPos = strpos($CurrentBody, "]", $TitleStartPos);

							$CurrentArticleTitle = substr( $CurrentBody, $TitleStartPos, $TitleEndPos-$TitleStartPos );

							$CorrectArticleTitle = "(not found)";
							$CorrectArticleAuthor = "(not found)";

							$SearchResult = array_search( $CurrentLink, $ArrayAllArticleLink );

							if( $SearchResult !== FALSE )
							{
								$CorrectArticleTitle  = $ArrayAllArticleTitle [$SearchResult];
								$CorrectArticleAuthor = $ArrayAllArticleAuthor[$SearchResult];
								
								
								
								//don't do any processing for summaries unless the link we're looking at is after the first header
								if( $LinkIsAfterHeader )
								{
									$LinkEndPos = strpos($CurrentBody, "]", $TokenStartPos) + 1;

									$NewLinePos = strpos($CurrentBody, "\n", $LinkEndPos);

									$NextArticleSummary = "";

									if( $NewLinePos === FALSE )
									{
										$NextArticleSummary = substr( $CurrentBody, $LinkEndPos );
									}
									else
									{
										//$NextArticleSummary .= "\nsummary:\n" + substr( $CurrentBody, $TokenStartPos, $NextTokenStartPos-$TokenStartPos );
										$NextArticleSummary = substr( $CurrentBody, $LinkEndPos, $NewLinePos-$LinkEndPos );
									}

									// search through the array of official summaries for this article, to see if any of them are found in the current article
									// if a match is found, then don't add the summary to the array of used summaries

									$SummaryExistsForThisArticle = false;
									$FoundAMatchingSummary = false;
									$FirstFoundSummary = "";

									if( array_key_exists( $CorrectArticleTitle, $ArrayAllArticleOfficialSummaries ) )
									{
										$SummaryExistsForThisArticle = true;

										foreach( $ArrayAllArticleOfficialSummaries[$CorrectArticleTitle] as $key => $val )
										{
											if( substr_count( $CurrentBody, $val ) > 0 )
											{
												if( !$FoundAMatchingSummary )
												{
													$FirstFoundSummary = $val;
												}

												$FoundAMatchingSummary = true;
											}
										}
									}

									$AddThisSummaryToTheArray = false;

									if( strpos( $NextArticleSummary, "by [[" ) > 0 )
									{
										//todo - consider adding a check for improperly formatted author links

										//todo - consider adding a check for an author link followed by a summary

										//todo - consider adding a check for a summary that doesn't appear until after the newline
									}
									else if( strlen( $NextArticleSummary ) > 0 )
									{
										if( $FoundAMatchingSummary )
										{
											// if we found a matching summary, then we don't need to do anything here
										}
										else
										{
											$AddThisSummaryToTheArray = true;
										}
									}
									else if( $SummaryExistsForThisArticle )
									{
										// a summary is available for this article, but the summary isn't on this page.
										// report this to 

										//$ArticleLinksWithAvailableSummary[$ArticleLinksWithAvailableSummaryNextIndex] = "*[$CurrentLink $CorrectArticleTitle] - " . $ArrayAllArticleOfficialSummaries[$CorrectArticleTitle][0];
										//$ArticleLinksWithAvailableSummary[$ArticleLinksWithAvailableSummaryNextIndex] = "*[$CurrentLink $CorrectArticleTitle] - " . $FirstFoundSummary;

										// just accessing the array directly doesn't work, and neither did that $FirstFoundSummary trick.  for some reason, I need to use a foreach

										//fwrite( $stderr, "$FirstFoundSummary\n" );
										//fwrite( $stderr, "$ArrayAllArticleOfficialSummaries[$CorrectArticleTitle][0]\n" );

										$IsFirstIteration = true;

										foreach( $ArrayAllArticleOfficialSummaries[$CorrectArticleTitle] as $key => $val )
										{
											if( $IsFirstIteration )
											{
												$ArticleLinksWithAvailableSummary[$ArticleLinksWithAvailableSummaryNextIndex] = "*[$CurrentLink $CorrectArticleTitle] - " . $val;

												$IsFirstIteration = false;
											}
											//fwrite( $stderr, "$val\n" );
										}

										$ArticleLinksWithAvailableSummaryConcept[$ArticleLinksWithAvailableSummaryNextIndex] = $CurrentTitle;

										$ArticleLinksWithAvailableSummaryNextIndex++;
									}

									if( $AddThisSummaryToTheArray )
									{
										// ignore known invalid summaries
										if( strlen( $NextArticleSummary ) > 10 )
										{
											$NextArticleSummary = trim( $NextArticleSummary, " ,-—'" );

											if
												(
													substr_count( $NextArticleSummary, "'' and ''[http://lesswrong.com/lw/hm/new_improved_lottery/ New Improved Lottery]''" ) > 0 ||
													substr_count( $NextArticleSummary, "(and [http://lesswrong.com/lw/ht/beware_the_unsurprised/ Beware the Unsurprised])" ) > 0 ||
													substr_count( $NextArticleSummary, "In particular, the [[Litany of Tarski]]." ) > 0 ||
													substr_count( $NextArticleSummary, "(but first read: [http://lesswrong.com/lw/m2/the_litany_against_gurus/ The Litany Against Gurus])" ) > 0 ||
													substr_count( $NextArticleSummary, ", [http://lesswrong.com/lw/" ) > 0 ||
													substr_count( $NextArticleSummary, "and [http://lesswrong.com/lw/" ) > 0 ||
													substr_count( $NextArticleSummary, "'' (prerequisite: [http://lesswrong.com/lw" ) > 0 ||
													substr_count( $NextArticleSummary, "'' and ''[http://lesswrong.com/lw" ) > 0 ||
													strcmp( $NextArticleSummary, "setting up the problem." ) == 0 ||
													strcmp( $NextArticleSummary, "[[Eliezer Yudkowsky]]" ) == 0 ||
													strcmp( $NextArticleSummary, "by Salamon and Rayhawk." ) == 0 ||
													strcmp( $NextArticleSummary, "by  [[Eliezer Yudkowsky]]" ) == 0 ||
													strcmp( $NextArticleSummary, "by talisman" ) == 0 ||
													strcmp( $NextArticleSummary, "(short story)" ) == 0 ||
													strcmp( $NextArticleSummary, "of a particular study design. Debiasing [http://lesswrong.com/lw/jk/burdensome_details/ won't be as simple] as practicing specific questions, it requires certain general habits of thought." ) == 0 ||
													strcmp( $NextArticleSummary, "as practicing specific questions, it requires certain general habits of thought." ) == 0 ||
													strcmp( $NextArticleSummary, "'' their single principle; but if they were ''really'' following ''only'' that single principle, they would [http://lesswrong.com/lw/kz/fake_optimization_criteria/ choose other acts to justify]." ) == 0 ||
													strcmp( $NextArticleSummary, "all of their complicated ''other'' preferences into their choice of ''exactly'' which acts they try to ''[http://lesswrong.com/lw/kq/fake_justification/ justify using]'' their single principle; but if they were ''really'' following ''only'' that single principle, they would [http://lesswrong.com/lw/kz/fake_optimization_criteria/ choose other acts to justify]." ) == 0 ||
													strcmp( $NextArticleSummary, "to this post tries to explain the cognitive twists whereby people [http://lesswrong.com/lw/ld/the_hidden_complexity_of_wishes/ smuggle] all of their complicated ''other'' preferences into their choice of ''exactly'' which acts they try to ''[http://lesswrong.com/lw/kq/fake_justification/ justify using]'' their single principle; but if they were ''really'' following ''only'' that single principle, they would [http://lesswrong.com/lw/kz/fake_optimization_criteria/ choose other acts to justify]." ) == 0 ||
													strcmp( $NextArticleSummary, "[http://lesswrong.com/lw/n1/allais_malaise/ followups]) - Offered choices between gambles, people make decision-theoretically inconsistent decisions." ) == 0 ||
													strcmp( $NextArticleSummary, ") - Offered choices between gambles, people make decision-theoretically inconsistent decisions." ) == 0 ||
													strcmp( $NextArticleSummary, "and ''[http://lesswrong.com/lw/oo/explaining_vs_explaining_away/ Explaining vs. Explaining Away]'' - elementary [[reductionism]]." ) == 0 ||
													strcmp( $NextArticleSummary, "\" which essentially answered \"Not on the present state of the Art\"" ) == 0 ||
													strcmp( $NextArticleSummary, "(and its [[Privileging the hypothesis | requisites]], like [[Locating the hypothesis]])" ) == 0 ||
													strcmp( $NextArticleSummary, "and ''[http://lesswrong.com/lw/hm/new_improved_lottery/ New Improved Lottery]" ) == 0 ||
													strcmp( $NextArticleSummary, "their single principle; but if they were ''really'' following ''only'' that single principle, they would [http://lesswrong.com/lw/kz/fake_optimization_criteria/ choose other acts to justify]." ) == 0 ||
													strcmp( $NextArticleSummary, "[http://lesswrong.com/lw/w6/recursion_magic/ ...Recursion, Magic]" ) == 0 ||
													strcmp( $NextArticleSummary, "[http://lesswrong.com/lw/wf/hard_takeoff/ Hard Takeoff]" ) == 0 ||
													strcmp( $NextArticleSummary, "[http://lesswrong.com/lw/wg/permitted_possibilities_locality/ Permitted Possibilities, & Locality]" ) == 0 ||
													strcmp( $NextArticleSummary, "(in the martial arts)" ) == 0 ||
													strcmp( $NextArticleSummary, "(in both psychotherapy and martial arts)" ) == 0 ||
													strcmp( $NextArticleSummary, "Description and account of the game." ) == 0 ||
													false
												)
											{
												// don't add the invalid summary
											}
											else
											{
												$ArrayAllArticleUsedSummaries[$SearchResult][] = $NextArticleSummary;
											}
										}
									}
								}
							}

							// for authors who have their own wikipages, link to the wikipage instead of the LW user page
							$CorrectArticleAuthor = str_replace("[http://lesswrong.com/user/Eliezer_Yudkowsky Eliezer_Yudkowsky]", "[[Eliezer Yudkowsky]]", $CorrectArticleAuthor);



							// check if the link ends with a /

							if
								(
									strcmp( $LastCharInLink, "/" ) != 0 &&
									$SlashCountBeforeAdd < 6
								)
							{
								$ArticleLinksWithoutEndingSlash[$ArticleLinksWithoutEndingSlashNextIndex] = "*[$CurrentLink $CorrectArticleTitle] by $CorrectArticleAuthor";

								$ArticleLinksWithoutEndingSlashConcept[$ArticleLinksWithoutEndingSlashNextIndex] = $CurrentTitle;

								$ArticleLinksWithoutEndingSlashNextIndex++;
							}



							if( strcmp($CurrentArticleTitle, $CorrectArticleTitle) != 0 && $LinkIsAfterHeader )
							{
								// don't report an error for the following links:
								if
									(
										substr_count( $CurrentArticleTitle, "ranges over anything, not just internal subjective experiences") <= 0 &&
										substr_count( $CurrentArticleTitle, "sequence leading up") <= 0 &&
										substr_count( $CurrentArticleTitle, "smuggle") <= 0 &&
										substr_count( $CurrentArticleTitle, "justify using") <= 0 &&
										substr_count( $CurrentArticleTitle, "choose other acts to justify") <= 0 &&
										substr_count( $CurrentArticleTitle, "Timeless decision theory") <= 0 &&
										substr_count( $CurrentArticleTitle, "philosophical majoritarianism") <= 0 &&
										substr_count( $CurrentArticleTitle, "critical comments") <= 0 &&
										substr_count( $CurrentArticleTitle, "Positive Bias") <= 0 &&
										substr_count( $CurrentArticleTitle, "Hindsight Bias") <= 0 &&
										substr_count( $CurrentArticleTitle, "not an isolated artifact") <= 0 &&
										substr_count( $CurrentArticleTitle, "won't be as simple") <= 0 &&
										substr_count( $CurrentArticleTitle, "Illusion of Transparency") <= 0 &&
										substr_count( $CurrentArticleTitle, "Affect Heuristic") <= 0 &&
										substr_count( $CurrentArticleTitle, "Evaluability") <= 0 &&
										substr_count( $CurrentArticleTitle, "Unbounded Scales, Huge Jury Awards, and Futurism") <= 0 &&
										substr_count( $CurrentArticleTitle, "subsequent") <= 0 &&
										substr_count( $CurrentArticleTitle, "followups") <= 0 &&
										substr_count( $CurrentArticleTitle, "Do We Believe <i>Everything</i> We're Told?") <= 0 &&
										substr_count( $CurrentArticleTitle, "Quantum Physics") <= 0 &&
										substr_count( $CurrentArticleTitle, "Shut Up and Do the Impossible") <= 0 &&
										substr_count( $CurrentArticleTitle, "You ''Can'' Face Reality") <= 0 &&
										substr_count( $CurrentArticleTitle, "Absence of Evidence ''Is'' Evidence of Absence") <= 0 &&
										substr_count( $CurrentArticleTitle, "Doublethink: Choosing to be Biased") <= 0 &&
										substr_count( $CurrentArticleTitle, "Anti-Epistemology") <= 0 &&
										substr_count( $CurrentArticleTitle, "Is Humanism a Religion-Substitute?") <= 0 &&
										substr_count( $CurrentArticleTitle, "Your Strength As A Rationalist") <= 0 &&
										substr_count( $CurrentArticleTitle, "Absence of Evidence '''is''' Evidence of Absence") <= 0 &&
										substr_count( $CurrentArticleTitle, "Reversed Stupidity is Not Intelligence") <= 0 &&
										substr_count( $CurrentArticleTitle, "A Human's Guide to Words") <= 0 &&
										substr_count( $CurrentArticleTitle, "here") <= 0 &&
										true
									)
								{
									$ArticleLinksWithWrongTitle[$ArticleLinksWithWrongTitleNextIndex] = "*[$CurrentLink $CorrectArticleTitle] by $CorrectArticleAuthor\n**(title was [$CurrentLink $CurrentArticleTitle])";

									$ArticleLinksWithWrongTitleConcept[$ArticleLinksWithWrongTitleNextIndex] = $CurrentTitle;

									$ArticleLinksWithWrongTitleNextIndex++;

									if( $Debug )
									{
										//fwrite( $stderr, "article title: $CurrentArticleTitle\ncorrect title: $CorrectArticleTitle\n\n" );
									}
								}
							}

							//fwrite( $stderr, "article title: $CurrentArticleTitle\n\n" );

							$ByCharacters = substr( $CurrentBody, $TitleEndPos, 6 );

							$FirstHeaderPos = strpos($CurrentBody, "==", 0);





							// check if the link has an author

							// ignore missing authors if the link appears before any headers
							if( strcmp($ByCharacters, "] by [") != 0 && $LinkIsAfterHeader )
							{
								$LineUpToAuthor = substr( $CurrentBody, $TokenStartPos-1, ($TitleEndPos+6)-($TokenStartPos-1) );
								$LineUpToAuthor = str_replace("\n", "\\n", $LineUpToAuthor);

								$ArticleLinksWithoutAuthor[$ArticleLinksWithoutAuthorNextIndex] = "*[$CurrentLink $CorrectArticleTitle] by $CorrectArticleAuthor\n**(was $LineUpToAuthor)";

								$ArticleLinksWithoutAuthorConcept[$ArticleLinksWithoutAuthorNextIndex] = $CurrentTitle;

								$ArticleLinksWithoutAuthorNextIndex++;

								if( $Debug )
								{
									//fwrite( $stderr, "Concept: $CurrentTitle, by characters: <$ByCharacters>\n\n" );
								}
							}

							//todo - change this to check if the author is correct!
						}
					
					
					
					
					
						//fwrite( $stderr, "link start: $TokenStartPos \n link end: $TokenEndPos \n current link: $CurrentLink\n\n" );
					
						// find the current link in the array, if it exists

						$SearchResult = array_search( $CurrentLink, $ArrayAllArticleLink );

						if( $SearchResult !== FALSE )
						{
							// find if this concept exists in the array
							if( in_array( $CurrentTitle, $ArrayAllArticleIndexedConcepts[$SearchResult] ) )
							{
								// mark the link as found
								$ArrayAllArticleFoundConcepts     [$SearchResult][$CurrentTitle] = true;
							}
							else
							{
								// add the concept to the array of unindexed concepts
								$ArrayAllArticleNotIndexedConcepts[$SearchResult][] = $CurrentTitle;
							}
						}
						
						$TokenStartPos = strpos($CurrentBody, "http://lesswrong.com/lw/", $TokenEndPos);
						$TokenEndPos = strpos($CurrentBody, " ", $TokenStartPos);

						$CurrentLink = substr( $CurrentBody, $TokenStartPos, $TokenEndPos-$TokenStartPos );
						
						if
							(
								$TokenEndPos <= $TokenStartPos ||
								$TokenStartPos === FALSE ||
								substr_count( $CurrentLink, "http://lesswrong.com/lw/" ) <= 0
							)
						{
							$FinishedFindingLinks = true;
						}
					}
				}
				
				
				//$IsReadingConceptPages = false;

				//fwrite( $stderr, $CurrentBody );

				//unfinished!!!

				//for each page:

				// first check if the page is in the $ConceptFound list

				// read the list of articles in the "Blog posts" section.
				// report any links that appear in the All Articles page, for that concept, but don't appear in the wiki page
				// report any links that appear in the wiki page, but don't appear in the All Articles page, for that concept


				$SuccessfullyReadOneConceptPage = true;
			}
		}
	}
}

function defaultHandler($parser, $data)
{
}

function new_xml_parser($file)
{
    global $parser_file;
	
    $xml_parser = xml_parser_create();
    xml_parser_set_option($xml_parser, XML_OPTION_CASE_FOLDING, 1);
    xml_set_element_handler($xml_parser, "startElement", "endElement");
    xml_set_character_data_handler($xml_parser, "characterData");
    xml_set_default_handler($xml_parser, "defaultHandler");
	
    if( !($fp = @fopen($file, "r")) )
    {
        return false;
    }
    if( !is_array($parser_file) )
    {
        settype($parser_file, "array");
    }
    $parser_file[$xml_parser] = $file;
    return array($xml_parser, $fp);
}



function ReadOnceThroughTheWholeXMLFile()
{
	global $XMLfile;

	// create the XML parser

	if( !(list($xml_parser, $fp) = new_xml_parser($XMLfile)) )
	{
		die("could not open XML input");
	}

	// read the XML file

	while( $data = fread($fp, 4096) )
	{
		if( !xml_parse($xml_parser, $data, feof($fp)) )
		{
			die( sprintf("XML error: %s at line %d\n",
				xml_error_string(xml_get_error_code($xml_parser)),
				xml_get_current_line_number($xml_parser)));
		}
	}

	// we're done with the XML file now, so close it

	fclose($fp);
}


fwrite( $stderr, "Reading through the All Articles pages\n\n" );

// first read through the All Articles pages

$IsReadingAllArticlesPages = true;

$NextYearToRead = $FirstYearToRead;

$FailedToReadAllArticlesPage = false;

// keep looping through the whole XML file
// abort when we've successfully read all of the All Articles pages,
// or if we've looped through the whole file without finding the next one

// there's probably a more efficient way than looping repeatedly through the whole XML file,
//  but that would probably make the code even more tangled than it is now.

// this looping should be unnecessary, because the All Articles pages should be in order in the XML file, but I'll leave this code as it is for now.

while
	(
		$NextYearToRead <= $FinalYearToRead &&
		! $FailedToReadAllArticlesPage
	)
{
	fwrite( $stderr, "processing the XML file, year $NextYearToRead \n\n" );
	
	$SuccessfullyReadOneAllArticlesPage = false;

	ReadOnceThroughTheWholeXMLFile();
	
	if( ! $SuccessfullyReadOneAllArticlesPage )
	{
		$FailedToReadAllArticlesPage = true;
		
		fwrite( $stderr, "error: failed to read the next All Articles page, year $NextYearToRead \n\n" );
	}
}

$IsReadingAllArticlesPages = false;



fwrite( $stderr, "Reading through the Summaries pages\n\n" );

// next read through the summaries pages

$IsReadingSummariesPages = true;

$NextYearToRead = $FirstYearToRead;

$FailedToReadSummariesPage = false;

// keep looping through the whole XML file
// abort when we've successfully read all of the Summaries pages,
// or if we've looped through the whole file without finding the next one

// there's probably a more efficient way than looping repeatedly through the whole XML file,
//  but that would probably make the code even more tangled than it is now.

// this looping should be unnecessary, because the Summaries pages should be in order in the XML file, but I'll leave this code as it is for now.

while
	(
		$NextYearToRead <= $FinalYearToRead &&
		! $FailedToReadSummariesPage
	)
{
	fwrite( $stderr, "processing the XML file for summaries, year $NextYearToRead \n\n" );
	
	$SuccessfullyReadOneSummariesPage = false;

	ReadOnceThroughTheWholeXMLFile();
	
	if( ! $SuccessfullyReadOneSummariesPage )
	{
		$FailedToReadSummariesPage = true;
		
		fwrite( $stderr, "error: failed to read the next summaries page, year $NextYearToRead \n\n" );
	}
}

$IsReadingSummariesPages = false;



fwrite( $stderr, "marking all concepts as not found yet\n\n" );

// now read through the array of concepts, and mark them all as unfound

foreach( $ArrayAllArticleIndexedConcepts as $key => $val )
{
	foreach( $val as $key2 => $val2 )
	{
		$ConceptFound[$val2] = false;
	}
}



fwrite( $stderr, "Reading through the Concept pages\n\n" );

// next read through the Concept pages

$IsReadingConceptPages     = true;

$FailedToReadConceptPage = false;

// just read once through the whole XML file
// there's no need to read through the concept pages in any particular order

ReadOnceThroughTheWholeXMLFile();

fwrite( $stderr, "Pages read: $PagesRead\n" );

if( ! $SuccessfullyReadOneConceptPage )
{
	$FailedToReadConceptPage = true;

	fwrite( $stderr, "error: failed to read any Concept page \n\n" );
}



fwrite( $stderr, "Doing final processing, step 1 of 3...\n\n" );


// now go through the $ConceptFound array

// for any concept that doesn't already have a wikipage, output a template for a blank page, with the "Blog posts" section filled in with all of the blog posts that mention this topic


if( false )
{
	// this section was added just to clean up the pages that use the old OB link for Eliezer's post instead of the new LW link
	// disabled now because it gives too many false positives
	fwrite( $fp, "\n\n==The following concept pages link to OvercomingBias.com articles:==\n\n" );

	foreach( $PagesWithOvercomingBiasLinks as $key => $val )
	{
		fwrite( $fp, "*[[$key]]\n" );
	}
}



fwrite( $fp, "\n\n==The following concept pages have comments:==\n\n" );

foreach( $PagesWithComments as $key => $val )
{
	fwrite( $fp, "*[[$key]]\n" );
}



// this section was added just to clean up the pages that use the old OB link for Eliezer's post instead of the new LW link
// disabled now because it gives too many false positives
fwrite( $fp, "\n\n==The following concept pages have the \"Overcoming Bias Articles\" header:==\n\n" );

foreach( $PagesWithOvercomingBiasArticlesHeader as $key => $val )
{
	fwrite( $fp, "*[[$key]]\n" );
}



fwrite( $fp, "\n\n==The following concept pages have \"External references\" instead of \"References\":==\n\n" );

foreach( $PagesWithExternalReferences as $key => $val )
{
	fwrite( $fp, "*[[$key]]\n" );
}



fwrite( $fp, "\n\n==The following concept pages have a miscapitalized \"See Also\" header:==\n\n" );

foreach( $PagesWithSeeAlso as $key => $val )
{
	fwrite( $fp, "*[[$key]]\n" );
}



fwrite( $fp, "\n\n==The following concept pages have an author link that links to an external site:==\n\n" );

foreach( $PagesWithExternalAuthorLinks as $key => $val )
{
	fwrite( $fp, "*[[$key]]\n" );
}



fwrite( $fp, "\n\n==The following concept pages have an extra newline after the wikilink template:==\n\n" );

foreach( $PagesWithNewlineAfterWikiLink as $key => $val )
{
	fwrite( $fp, "*[[$key]]\n" );
}



fwrite( $fp, "\n\n==The following concept pages have the See Also section before the Blog Posts section:==\n\n" );

foreach( $PagesWithSeeAlsoBeforeBlogPosts as $key => $val )
{
	fwrite( $fp, "*[[$key]]\n" );
}


// disabled because of a weird bug that was introduced when I added the "see also" checking
/*
fwrite( $fp, "\n\n==The following article links are missing the / at the end, or aren't in the index:==\n\n" );

$PreviousConcept = "";

foreach( $ArticleLinksWithoutEndingSlash as $key => $val )
{
	if( strcmp( $PreviousConcept, $ArticleLinksWithoutEndingSlashConcept[$key] ) != 0 )
	{
		fwrite( $fp, "\n*[[$ArticleLinksWithoutEndingSlashConcept[$key]]]\n" );
	}
	fwrite( $fp, "*$val\n" );
	
	$PreviousConcept = $ArticleLinksWithoutEndingSlashConcept[$key];
}
*/


fwrite( $fp, "\n\n==The following article links have a wrong or improperly formatted title:==\n\n" );

$PreviousConcept = "";

foreach( $ArticleLinksWithWrongTitle as $key => $val )
{
	if( strcmp( $PreviousConcept, $ArticleLinksWithWrongTitleConcept[$key] ) != 0 )
	{
		fwrite( $fp, "\n*[[$ArticleLinksWithWrongTitleConcept[$key]]]\n" );
	}
	fwrite( $fp, "*$val\n" );
	
	$PreviousConcept = $ArticleLinksWithWrongTitleConcept[$key];
}


fwrite( $fp, "\n\n==The following article links have a summary available that was not added to the page:==\n\n" );

$PreviousConcept = "";

foreach( $ArticleLinksWithAvailableSummary as $key => $val )
{
	if( strcmp( $PreviousConcept, $ArticleLinksWithAvailableSummaryConcept[$key] ) != 0 )
	{
		fwrite( $fp, "\n*[[$ArticleLinksWithAvailableSummaryConcept[$key]]]\n" );
	}
	fwrite( $fp, "*$val\n" );
	
	$PreviousConcept = $ArticleLinksWithAvailableSummaryConcept[$key];
}





if( false )
{
	/// this section was disabled because there were way too many false positives
	fwrite( $fp, "\n\n==The following article links have a missing or improperly formatted author:==\n\n" );

	$PreviousConcept = "";

	foreach( $ArticleLinksWithoutAuthor as $key => $val )
	{
		if( strcmp( $PreviousConcept, $ArticleLinksWithoutAuthorConcept[$key] ) != 0 )
		{
			fwrite( $fp, "\n*[[$ArticleLinksWithoutAuthorConcept[$key]]]\n" );
		}
		fwrite( $fp, "*$val\n" );

		$PreviousConcept = $ArticleLinksWithoutAuthorConcept[$key];
	}
}



fwrite( $fp, "\n\n==The following concepts don't have wikipages with links to LessWrong.com articles yet:==\n\n" );

foreach( $ConceptFound as $key => $val )
{
	if( ! $val )
	{
		fwrite( $fp, "*[[$key]]\n" );
	}
}



fwrite( $fp, "\n\n==The following concepts are not in the All Articles pages:==\n\n" );

foreach( $ConceptNotInIndex as $key => $val )
{
	fwrite( $fp, "*[[$key]]\n" );
}



fwrite( $fp, "\n\n==The following concepts are in the All Articles page, but are redirects:==\n\n" );

foreach( $ConceptThatAreRedirects as $key => $val )
{
	fwrite( $fp, "*[[$key]]\n" );
}




fwrite( $stderr, "Doing final processing, step 2 of 3...\n\n" );



// now output the list of which articles in the All Articles index are missing an entry:

fwrite( $fp, "\n\n==The following articles in the [[Less Wrong/All Articles|All Articles]] index are missing an entry:==\n" );

foreach( $ArrayAllArticleNotIndexedConcepts as $key => $val )
{
	if( count( $val ) > 0 )
	{
		fwrite( $fp, "\n*[$ArrayAllArticleLink[$key] $ArrayAllArticleTitle[$key]] is missing the following concepts:\n" );
		
		foreach( $val as $key2 => $val2 )
		{
			fwrite( $fp, "**[[$val2]]\n" );
		}
	}
}



fwrite( $stderr, "Doing final processing, step 3 of 3...\n\n" );


// now output the list of which article links need to be added to the concept pages:

fwrite( $fp, "\n\n==The following article links need to be added to the concept pages:==\n" );

// for each concept
foreach( $ConceptFound as $key => $val )
{
	
	// make an array to store the missing links
	$MissingLinks = array();

	// for each article
	foreach( $ArrayAllArticleIndexedConcepts as $key2 => $val2 )
	{
		// check if the article contains the concept
		if( array_search($key, $val2) )
		{
			//fwrite( $stderr, "concept: $key \n link: $ArrayAllArticleLink[$key2] \n title: $ArrayAllArticleTitle[$key2]\n\n" );
			
			// check if the concept's link was not found
			if( ! array_key_exists($key, $ArrayAllArticleFoundConcepts[$key2]) )
			{
				// add it to the list of concepts whose links were not found
				$MissingLinks[] = "**[$ArrayAllArticleLink[$key2] $ArrayAllArticleTitle[$key2]] by $ArrayAllArticleAuthor[$key2]";
				
				//fwrite( $stderr, "concept: $key \n link: $ArrayAllArticleLink[$key2] \n title: $ArrayAllArticleTitle[$key2]\n\n" );
			}
		}
	}
	
	// if there are any not found article links, then output the list
	if( count($MissingLinks) > 0 )
	{
		fwrite( $fp, "\n*[[$key]] is missing the following article links:\n" );
		
		foreach( $MissingLinks as $key3 => $val3 )
		{
			fwrite( $fp, "$val3\n" );
		}
	}
}


// now output the list of which article links need to be added to the concept pages:

fwrite( $fp, "\n\n==The following See Also links only go one way:==\n" );

// for each concept
foreach( $ConceptSeeAlso as $key => $val )
{
	foreach( $val as $key2 => $val2 )
	{
		$Concept1 = $key;
		$Concept2 = $val2;
		
		$MatchFound = false;

		foreach( $ConceptSeeAlso as $key3 => $val3 )
		{
			foreach( $val3 as $key4 => $val4 )
			{
				$Concept3 = $key3;
				$Concept4 = $val4;
				
				if( $Concept1 == $Concept4 && $Concept2 == $Concept3 )
				{
					$MatchFound = true;
				}
			}
		}
		
		if( !$MatchFound )
		{
			fwrite( $fp, "\n*[[$Concept1]] -> [[$Concept2]]" );
		}
	}
}



fwrite( $fp, "\n\n==The following is a list of all concept pages:==\n\n" );

$SortedConcepts = $ConceptFound;

ksort($SortedConcepts);

foreach( $SortedConcepts as $key => $val )
{
	if( $val )
	{
		fwrite( $fp, "*[[$key]]\n" );
	}
}


fwrite( $fp, "\n\n==Links to the All Articles pages:==\n" );
fwrite( $fp, "*[[Less Wrong/All Articles]]\n" );
fwrite( $fp, "*[[Less Wrong/2006 Articles]]\n" );
fwrite( $fp, "*[[Less Wrong/2007 Articles]]\n" );
fwrite( $fp, "*[[Less Wrong/2008 Articles]]\n" );
fwrite( $fp, "*[[Less Wrong/2009 Articles]]\n" );
fwrite( $fp, "*[[Less Wrong/2010 Articles]]\n" );


fwrite( $fp, "\n\n==Links to the Summaries pages:==\n" );
fwrite( $fp, "*[[Less Wrong/Article summaries]]\n" );
fwrite( $fp, "*[[Less Wrong/2006 Articles/Summaries]]\n" );
fwrite( $fp, "*[[Less Wrong/2007 Articles/Summaries]]\n" );
fwrite( $fp, "*[[Less Wrong/2008 Articles/Summaries]]\n" );
fwrite( $fp, "*[[Less Wrong/2009 Articles/Summaries]]\n" );
fwrite( $fp, "*[[Less Wrong/2010 Articles/Summaries]]\n" );



fclose($fp);


$fp = fopen('ConceptGraph.dot.txt', 'w');

//{_COPYBLOCK1
fwrite($fp, "digraph G {\n\nnode [fontsize=\"$fontsize\"]\n\n");

// output the data for each concept
foreach( $ConceptSeeAlso as $key => $val )
{
	$curLine = "";
	
	$NameWithSpaces = $key;
	$NameWithoutSpaces = str_replace(" ", "_", $NameWithSpaces);

	$curName  = "\"" . $NameWithoutSpaces . "\"";
	$curName  = str_pad( $curName, $PaddingValue1 );
	$curLine .= $curName;

	$curLine .= "[label=\"";

	$curName  = $NameWithSpaces;
	$curName .= "\"";
	$curName  = str_pad( $curName, $PaddingValue2 );
	$curLine .= $curName;

//	$curLine .= ", color=";
//
//	$curLine .= $NewArrayScenarioOutlineColors[$key];
//
//	$curLine .= ", shape=";
//
//	$curLine .= $NewArrayScenarioShapes[$key];
//
//	$curLine .= ", style=";
//
//	$curLine .= $NewArrayScenarioStyles[$key];
//
//	$curLine .= ", fillcolor=\"#";
//
//	$curLine .= $NewArrayScenarioFillColors[$key];
//
//	$curLine .= "\"";

	//if( strcmp($NewArrayScenarioNamesWithoutSpaces[$key], $ArrayScenarioNamesWithoutSpaces[$basekey]) == 0 )
	//{
	//	$curLine .= ", peripheries=3";
	//}

	$newURL = "http://wiki.lesswrong.com/wiki/" . str_replace(" ", "_", $NameWithSpaces);

	$curLine .= ", URL=\"$newURL\"";

	$curLine .= "];\n";

	fwrite($fp, $curLine);
}

fwrite($fp, "\n\n");

// output the data for the edges
foreach( $ConceptSeeAlso as $key => $val )
{
	foreach( $val as $key2 => $val2 )
	{
		$curLine = "";

		$Name1WithSpaces = $key;
		$Name1WithoutSpaces = str_replace(" ", "_", $Name1WithSpaces);

		$Name2WithSpaces = $val2;
		$Name2WithoutSpaces = str_replace(" ", "_", $Name2WithSpaces);

		$curName  = "\"" . $Name1WithoutSpaces . "\"";
		$curName  = str_pad( $curName, $PaddingValue1 );
		$curLine .= $curName;

		$curLine .= "-> ";

		$curLine .= "\"" . $Name2WithoutSpaces . "\"";

		$curLine .= "\n";

		fwrite($fp, $curLine);

	}
}

fwrite($fp, "\n\n}");

//}_COPYBLOCK1


fclose($fp);



$fp = fopen('AllArticles.txt', 'w');


fwrite( $stderr, "Outputting new All Articles page...\n\n" );


$ProgressCounter = 0;

foreach( $ArrayAllArticleTitle as $key => $val )
{
	$ConceptString = "";
	
	$TempConceptArray = array();

	foreach( $ArrayAllArticleIndexedConcepts[$key] as $key2 => $val2 )
	{
		if( ! in_array($val2, $TempConceptArray) )
		{
			$ConceptString .= "[[$val2]], ";
			$TempConceptArray[] = $val2;
		}
	}
	
	foreach( $ArrayAllArticleNotIndexedConcepts[$key] as $key2 => $val2 )
	{
		if( ! in_array($val2, $TempConceptArray) )
		{
			$ConceptString .= "[[$val2]], ";
			$TempConceptArray[] = $val2;
		}
	}
	
	if( strlen($ConceptString) > 2 )
	{
		$ConceptString = substr($ConceptString, 0, -2);
	}

	fwrite( $fp, "|-valign=\"top\"\n" );
	fwrite( $fp, "| [$ArrayAllArticleLink[$key] $ArrayAllArticleTitle[$key]]\n" );
	fwrite( $fp, "| $ConceptString\n" );
	fwrite( $fp, "| $ArrayAllArticleAuthor[$key]\n" );

	if( $ProgressCounter % 100 == 0 )
	{
		//fwrite( $stderr, "Progress: $ProgressCounter\n" );
	}
	
	$ProgressCounter++;
}

fwrite( $stderr, "Progress: $ProgressCounter\n" );



fclose($fp);




$fp = fopen('ArticleSummaries.txt', 'w');


fwrite( $stderr, "Outputting article summaries...\n\n" );


$ProgressCounter = 0;

foreach( $ArrayAllArticleTitle as $key => $val )
{
	$ConceptString = "";
	
	$TempConceptArray = array();

	$IsFirstIteration = true;

	foreach( $ArrayAllArticleUsedSummaries[$key] as $key2 => $val2 )
	{
		if( !$IsFirstIteration )
		{
			fwrite( $fp, "\n\n(alternate summary:)\n\n" );
		}
		else
		{
			fwrite( $fp, "\n\n=====[$ArrayAllArticleLink[$key] $ArrayAllArticleTitle[$key]]=====\n\n" );
	
			$IsFirstIteration = false;
		}
		
		fwrite( $fp, $val2 );
	}

	if( $ProgressCounter % 100 == 0 )
	{
		//fwrite( $stderr, "Progress: $ProgressCounter\n" );
	}
	
	$ProgressCounter++;
}

fwrite( $stderr, "Progress: $ProgressCounter\n" );



fclose($fp);




$fp = fopen('ArticleSummaries2.txt', 'w');


fwrite( $stderr, "Outputting article summaries...\n\n" );


$ProgressCounter = 0;

foreach( $ArrayAllArticleTitle as $key => $val )
{
	$ConceptString = "";
	
	$TempConceptArray = array();

	fwrite( $fp, "\n\n=====[$ArrayAllArticleLink[$key] $ArrayAllArticleTitle[$key]]=====\n\n" );
	
	$IsFirstIteration = true;
	
	if( array_key_exists( $ArrayAllArticleTitle[$key], $ArrayAllArticleOfficialSummaries ) )
	{
		foreach( $ArrayAllArticleOfficialSummaries[$ArrayAllArticleTitle[$key]] as $key2 => $val2 )
		{
			if( !$IsFirstIteration )
			{
				fwrite( $fp, "\n\n(alternate summary:)\n\n" );
			}
			else
			{
				$IsFirstIteration = false;
			}

			fwrite( $fp, $val2 );
		}
	}

	foreach( $ArrayAllArticleUsedSummaries[$key] as $key2 => $val2 )
	{
		if( !$IsFirstIteration )
		{
			fwrite( $fp, "\n\n(alternate summary:)\n\n" );
		}
		else
		{
			$IsFirstIteration = false;
		}
		
		fwrite( $fp, $val2 );
	}

	if( $ProgressCounter % 100 == 0 )
	{
		//fwrite( $stderr, "Progress: $ProgressCounter\n" );
	}
	
	$ProgressCounter++;
}

fwrite( $stderr, "Progress: $ProgressCounter\n" );



fclose($fp);


//*/


fwrite( $stderr, "Processing Recent Post List...\n\n" );



$handle = fopen('http://lesswrong.com/recentposts', 'r');

$RecentPostRawData = "";

while( ( $buf = fread( $handle, 8192 ) ) != '' )
{
	$RecentPostRawData .= $buf;
}

if( $buf === FALSE )
{
	return "error reading file";
}

fclose($handle);



$fp = fopen('RecentPosts.txt', 'w');

// read the data from the recent posts page, then output the data in reverse order

$ArrayRecentPostLines = array();


$PostLineStartPos  = strpos( $RecentPostRawData, "<h3>", 0);
$PostLineEndPos    = $PostLineStartPos;

$EndPos = strpos( $RecentPostRawData, "<p class=\"nextprev\">View more:", 0);


$NextArticleTitle = "";
$IterationCount = 0;
$PastTheEnd = false;


while
	(
//		! in_array($NextArticleTitle, $ArrayAllArticleTitle) &&
		$IterationCount < 1000 &&
		! $PastTheEnd &&
		true
	)
{
	$PostLineStartPos  = strpos( $RecentPostRawData, "<a href=\"", $PostLineEndPos ) + 9;
	$PostLineEndPos    = strpos( $RecentPostRawData, "\"", $PostLineStartPos );

	$NextArticleLink   = substr( $RecentPostRawData, $PostLineStartPos, $PostLineEndPos-$PostLineStartPos );

	$PostLineStartPos  = strpos( $RecentPostRawData, ">", $PostLineEndPos ) + 1;
	$PostLineEndPos    = strpos( $RecentPostRawData, "<", $PostLineStartPos );

	$NextArticleTitle  = substr( $RecentPostRawData, $PostLineStartPos, $PostLineEndPos-$PostLineStartPos );
	$NextArticleTitle  = str_replace("&quot;", "\"", $NextArticleTitle);

	$PostLineStartPos  = strpos( $RecentPostRawData, "<a href=\"", $PostLineEndPos ) + 9;
	$PostLineEndPos    = strpos( $RecentPostRawData, "\"", $PostLineStartPos );

	$NextArticleAuthorLink = substr( $RecentPostRawData, $PostLineStartPos, $PostLineEndPos-$PostLineStartPos );

	$PostLineStartPos  = strpos( $RecentPostRawData, ">", $PostLineEndPos ) + 1;
	$PostLineEndPos    = strpos( $RecentPostRawData, "<", $PostLineStartPos );

	$NextArticleAuthorName = substr( $RecentPostRawData, $PostLineStartPos, $PostLineEndPos-$PostLineStartPos );
	
	
	if( $PostLineStartPos > $EndPos )
	{
		$PastTheEnd = true;
	}
	else
	{
		$CurrentLineString = "";


		$CurrentLineString .= "|-valign=\"top\"\n";
		$CurrentLineString .= "| [http://lesswrong.com$NextArticleLink $NextArticleTitle]\n";
		$CurrentLineString .= "| \n";
		$CurrentLineString .= "| [$NextArticleAuthorLink $NextArticleAuthorName]\n";

		//fwrite( $fp, $CurrentLineString );

		$ArrayRecentPostLines[$IterationCount] = $CurrentLineString;

		$IterationCount++;
	}
}

//fwrite( $fp, "\n\n\n\n\n\n\n\n----------------------------------------\n\n\n\n\n\n\n\n" );


for( $index = $IterationCount-1; $index >= 0; $index-- )
//foreach( $ArrayRecentPostLines as $key => $val )
{
	fwrite( $fp, $ArrayRecentPostLines[$index] );
}



fwrite( $stderr, "Done" );


fclose($fp);


if( $Debug )
{
	if( true )
	{
		$fp = fopen('Debug.txt', 'w');

		fwrite( $fp, "\n\n\n\n\n\n\n\n\n\n ConceptFound \n\n\n\n\n\n\n\n\n\n" );

		foreach( $ConceptFound as $key => $val ) { $data = $key . "\n" . print_r($val, true) . "\n"; fwrite( $fp, $data ); }

		fwrite( $fp, "\n\n\n\n\n\n\n\n\n\n ArrayAllArticleIndexedConcepts \n\n\n\n\n\n\n\n\n\n" );

		foreach( $ArrayAllArticleIndexedConcepts as $key => $val ) { $data = $key . "\n" . print_r($val, true); fwrite( $fp, $data ); }

		fwrite( $fp, "\n\n\n\n\n\n\n\n\n\n ArrayAllArticleFoundConcepts \n\n\n\n\n\n\n\n\n\n" );

		foreach( $ArrayAllArticleFoundConcepts as $key => $val ) { $data = $key . "\n" . print_r($val, true); fwrite( $fp, $data ); }

		fwrite( $fp, "\n\n\n\n\n\n\n\n\n\n ArrayAllArticleNotIndexedConcepts \n\n\n\n\n\n\n\n\n\n" );

		foreach( $ArrayAllArticleNotIndexedConcepts as $key => $val ) { $data = $key . "\n" . print_r($val, true); fwrite( $fp, $data ); }

		fwrite( $fp, "\n\n\n\n\n\n\n\n\n\n ConceptSeeAlso \n\n\n\n\n\n\n\n\n\n" );

		foreach( $ConceptSeeAlso as $key => $val ) { $data = $key . "\n" . print_r($val, true); fwrite( $fp, $data ); }

		fwrite( $fp, "\n\n\n\n\n\n\n\n\n\n ArrayAllArticleOfficialSummaries \n\n\n\n\n\n\n\n\n\n" );

		foreach( $ArrayAllArticleOfficialSummaries as $key => $val ) { $data = $key . "\n" . print_r($val, true); fwrite( $fp, $data ); }

		fwrite( $fp, "\n\n\n\n\n\n\n\n\n\n ArrayAllArticleUsedSummaries \n\n\n\n\n\n\n\n\n\n" );

		foreach( $ArrayAllArticleUsedSummaries as $key => $val ) { $data = $ArrayAllArticleTitle[$key] . "\n" . print_r($val, true); fwrite( $fp, $data ); }


		fclose($fp);
	}
}



fclose($stderr);



//</pre>
?>