Index: /documentation/trunk/php/common.php
===================================================================
--- /documentation/trunk/php/common.php	(revision 25026)
+++ /documentation/trunk/php/common.php	(revision 25026)
@@ -0,0 +1,174 @@
+<?php
+
+$base_path = getcwd();
+
+// The dokuwiki directory is always in the same place, relative to the PHP
+// directory.
+$dokuwiki_path = fileCat(array($base_path,'..','packages','dokuwiki'));
+$dokuwiki_url = "../packages/dokuwiki";
+
+// Similarly, the xml-source directory is also relative to the PHP directory.
+$xml_source_path = fileCat(array($base_path, '..', 'manuals', 'xml-source'));
+
+// Keep track of the id's we've seen to ensure when generating new ones they
+//  are unique.
+$seen_ids = array();
+
+/**
+ */
+function fileCat($raw_parts)
+{
+  // 1. Expand out all of the parts first (as some may contain nested
+  //    directories)
+  $long_path = array();
+  foreach ($raw_parts as $raw_part)
+  {
+    $long_path = array_merge($long_path, preg_split('/[\\\\\/]/', $raw_part));
+  }
+  // 2. Resolve any '..' in the path
+  $path = array();
+  foreach ($long_path as $part)
+  {
+    if ($part == '..')
+    {
+      array_pop($path);
+    }
+    else
+    {
+      $path[] = $part;
+    }
+  }
+  return implode('/', $path);
+}
+/** fileCat() **/
+
+/** Given a dokuwiki namespace use the wikis metadata to determine the file
+ *  path to the most recently approved version of the page (which may not be
+ *  the active page in the 'pages' directory, but an older version in the
+ *  attic somewhere).
+ *  @param $namespace  A namespace in the form part:part:part
+ */
+function getApprovedVersionPath($namespace)
+{
+  global $dokuwiki_path;
+  $path = '#ERROR#';
+  $dokuwiki_data_path = $dokuwiki_path . '/data';
+  $namespace_path = str_replace(':','/',$namespace);
+  // 1. Build the path to the pages metadata file
+  $metadata_path = $dokuwiki_data_path . '/meta/' . $namespace_path . '.meta';
+  if (file_exists($metadata_path))
+  {
+    // 2. Deserialize the file
+    $page_metadata = unserialize(file_get_contents($metadata_path));
+
+    // 3. Look up the 'current' versions timestamp
+    $current_timestamp = $page_metadata['current']['date']['modified'];
+
+    // 4. Now look up the timestamp of the most recently approved version
+    $approved_timestamp = $current_timestamp; // default to the 'live' page
+    // - note that if a page have never been approved the 'approval' array
+    //   will not exist
+    if (isset($page_metadata['current']['approval']) && !empty($page_metadata['current']['approval']))
+    {
+      $approved_timestamp = max(array_keys($page_metadata['current']['approval']));
+    }
+
+    // 5. If they are the same, we can simply return the path to the 'live'
+    //    page in the 'pages' directory. Otherwise we construct a path to
+    //    the 'approved' version, which lurks in the mirrored location in
+    //    the 'attic' directory
+    if ($current_timestamp == $approved_timestamp)
+    {
+      $path = $dokuwiki_data_path . '/pages/' . $namespace_path . '.txt';
+    }
+    else
+    {
+      $path = $dokuwiki_data_path . '/attic/' . $namespace_path . '.' . $approved_timestamp . '.txt';
+    }
+  }
+  ///cho "[debug] Approved path: $path\n";
+  return $path;
+}
+/** getApprovedVersionPath() **/
+
+/** Given a text string generate a unique identifier.
+ */
+function generateID($text)
+{
+  global $seen_ids;
+  $text = preg_replace('/(<!--.*?-->|\(.*?\))/', '', $text);
+  $text = preg_replace('/\'s/', '', $text);
+  $text = strtolower(trim($text));
+  $poss_id = preg_replace('/\s+/', '_', $text);
+  $id = $poss_id;
+  $counter = 1;
+  while (isset($seen_ids[$id]))
+  {
+    $id = $poss_id . '_' . $counter;
+    $counter++;
+  }
+  $seen_ids[$id] = true;
+  return $id;
+}
+/** generateID($text) **/
+
+function mkAllDir($destination_dir, $mode)
+{
+  ///cho '<p> * Make all directories: ' . $destination_dir . '</p>' . "\n";
+  $dirs_to_create = array();
+  $dir = $destination_dir;
+  while (!empty($dir) && !file_exists($dir))
+  {
+    array_unshift($dirs_to_create, $dir);
+    $dir = substr($dir, 0, strrpos($dir, '/'));
+  }
+  foreach ($dirs_to_create as $dir)
+  {
+    mkdir($dir, 0755);
+  }
+  if (!file_exists($destination_dir))
+  {
+    printError('Failed to create directory: ' . $destination_dir);
+  }
+}
+
+/**
+ */
+function parseCLIArguments()
+{
+  if ($_SERVER['argc'])
+  {
+    for ($i = 1; $i < $_SERVER['argc'] && !$unknown_argument; $i++)
+    {
+      if (preg_match('/^\-+m/i', $_SERVER['argv'][$i]))
+      {
+        $i++;
+        $_REQUEST['m'] = $_SERVER['argv'][$i];
+      }
+      else
+      {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+/** parseCLIArguments() **/
+
+
+/**
+ */
+function printError($message, $is_fatal=true)
+{
+  if ($is_fatal)
+  {
+    exit('<p><b>Error!</b> ' . $message . "</p>\n");
+  }
+  else
+  {
+    echo '<p><b>Error!</b> ' . $message . "</p>\n";
+  }
+}
+/** printError($message) **/
+
+?>
Index: /documentation/trunk/php/gs-manual-export.php
===================================================================
--- /documentation/trunk/php/gs-manual-export.php	(revision 25026)
+++ /documentation/trunk/php/gs-manual-export.php	(revision 25026)
@@ -0,0 +1,1220 @@
+<?php
+
+require_once('common.php');
+$debug = 1;
+
+// There are some elements that are, in my opinion, incorrectly 'resolved' in
+// code blocks just because they are valid HTML. For instance, &lt;i&gt; is
+// resolved to <i> in code blocks, while something like &lt;Metadata&gt; is
+// not. Set this to true to allow such abominations (for the purpose of
+// comparing before and after versions of the XML). [jmt12]
+$allow_bad_codeblocks = true;
+
+/** @file gs-manual-export.php
+ *  This script transforms the series of dokuwiki pages that make up a certain
+ *  manual (as specified by the 'm' argument) in a certain language ('l') into
+ *  the XML format required by the rest of the Greenstone manual generation
+ *  scripts.
+ */
+
+// 0. Initialization
+$xml_source_path = '/research/jmt12/gsdl-docs/temp';
+// - we have a counter to assign identifiers to text blocks etc without ids
+$text_id_counter = 1;
+// - we need an array of all the footnotes
+$footnotes = array();
+
+// Defaults
+if (!isset($_REQUEST['l']) || empty($_REQUEST['l']))
+{
+  $_REQUEST['l'] = 'en';
+}
+if (!isset($_REQUEST['m']) || empty($_REQUEST['m']))
+{
+  //$_REQUEST['m'] = 'user';
+  //$_REQUEST['m'] = 'install';
+  $_REQUEST['m'] = 'develop';
+  //$_REQUEST['m'] = 'paper';
+}
+if (!isset($_REQUEST['v']) || empty($_REQUEST['v']))
+{
+  $_REQUEST['v'] = 'draft';
+}
+if (!isset($_REQUEST['a']) || !preg_match('/^(download|store)$/', $_REQUEST['a']))
+{
+  $_REQUEST['a'] = 'store'; // Try to store the file to disk
+}
+
+echo '<html>' . "\n";
+echo '<head>' . "\n";
+echo '<title>GS Manual Export</title>' . "\n";
+echo '</head>' . "\n";
+echo '<body>' . "\n";
+
+// - validate arguments before we use them (security)
+if (!preg_match('/^(develop|install|paper|user)$/',$_REQUEST['m']))
+ {
+   printError('Unknown manual type requested: ' . htmlspecialchars($_REQUEST['m']));
+ }
+
+if (!preg_match('/^(ar|en|es|fr|pt-br|ru)$/',$_REQUEST['l']))
+ {
+   printError('Unknown language requested: ' . htmlspecialchars($_REQUEST['l']));
+ }
+
+echo '<h2>Generating Greenstone Manual XML</h2>' . "\n";
+echo '<p><b>Manual:</b> ' . $_REQUEST['m'] . ' <b>Language:</b> ' . $_REQUEST['l'] . "</p>\n<hr/>\n";
+// 1. Create the XML output file handle
+$xml_file_path = '';
+if ($_REQUEST['a'] == 'download')
+{
+  $xml_file_path = fileCat(array($base_path, 'var', ucfirst($_REQUEST['m']) . '_' . $_REQUEST['l'] . '.xml'));
+}
+else
+{
+  // - construct the path using the information we've been provided as arguments
+  $xml_file_dir = $xml_source_path . '/' . $_REQUEST['l'];
+  mkAllDir($xml_file_dir, 0755);
+  $xml_file_path = $xml_file_dir . '/' . ucfirst($_REQUEST['m']) . '_' . $_REQUEST['l'] . '.xml';
+  // - backup any existing file
+  if (file_exists($xml_file_path))
+  {
+    $xml_backup_file_path = $xml_file_path . '.bak';
+    if (!rename($xml_file_path, $xml_backup_file_path))
+    {
+      printError('Failed to rename existing manual file for backup');
+    }
+  }
+}
+// - and create a handle to the new file
+$xml_out = fopen($xml_file_path, 'w');
+
+// 2. Read in the top level page - this will give configuration data for the
+//    manual and cover page, as well as specifying the order for the other
+//    pages in the manual
+echo "<p><b>Frontmatter:</b><br/>\n";
+// - by reading this page we hope to populate an array of metadata, and also
+//   extract the sequence of other pages within this manual
+$cover_metadata = array();
+$pages_in_order = array();
+// - we now need to consider if the user has asked for a draft version (i.e.
+//   includes the latest version of pages regardless of approval) or if only
+//   the approved versions of pages should be included
+// - only necessary for english version of manual, as those are the only pages
+//   editable
+$top_page_path = '';
+if ($_REQUEST['v'] == 'draft' || $_REQUEST['l'] != 'en')
+{
+  // - again, we can construct the path to the top level page given the arguments
+  //   provided
+  $top_page_path = $dokuwiki_path . '/data/pages/' . $_REQUEST['l'] . '/manuals/' . $_REQUEST['m'] . '.txt';
+}
+else
+{
+  $top_page_path = getApprovedVersionPath('en:manuals:' . $_REQUEST['m']);
+}
+
+if (!file_exists($top_page_path))
+ {
+   printError('Failed to locate top level page for manual');
+ }
+// - we're going to open a handle to the file, then read it in line-by-line
+//   watching for the lines we are interested in (using pattern matching)
+$top_page_in = fopen($top_page_path, 'r');
+if (!$top_page_in)
+ {
+   printError('Failed to open top level page for reading');
+ }
+$in_contents = false;
+while (($line = fgets($top_page_in)) !== false)
+ {
+   // - if we are capturing page order, and we encounter something that looks
+   //   like a bulletpoint item pointing to a wiki page, then we append the
+   //   name of that page to our pages in order array
+   if ($in_contents && preg_match('/^\s+\*\s+\[\[.:' . $_REQUEST['m'] . ':(.+?)\|(.*?)\]\]\s*$/', $line, $matches))
+   {
+     array_push($pages_in_order, $matches[1]);
+   }
+   // - metadata is all encoded within dokuwiki tables
+   elseif (preg_match('/^\^\s+([^\s]+)\s+\|\s+(.+?)\s+\|\s*$/', $line, $matches))
+   {
+     $field = $matches[1];
+     $value = $matches[2];
+     $values = array();
+     if (isset($cover_metadata[$field]))
+     {
+       $values = $cover_metadata[$field];
+     }
+     array_push($values, $value);
+     $cover_metadata[$field] = $values;
+   }
+   // - watch for the heading 'Contents' to begin extracting page order
+   //   information
+   elseif (preg_match('/^=+\s(.+)\s=+$/', $line, $matches))
+   {
+     if ($matches[1] == 'Contents')
+     {
+       $in_contents = true;
+     }
+     // - any other title means we aren't capturing page order (anymore)
+     else
+     {
+       $in_contents = false;
+     }
+   }
+ }
+if (!feof($top_page_in))
+ {
+   printError('Unexpected fgets() fail when reading top page');
+ }
+fclose($top_page_in);
+// - ensure we have the required metadata
+$required_metadata = array('Heading','Title','Affiliation','Version','Date');
+foreach ($required_metadata as $required_field)
+{
+  if (!isset($cover_metadata[$required_field]))
+  {
+    printError('Missing required metadata: ' . $required_field);
+  }
+}
+// - now we can use the metadata to construct the XML header and the cover page.
+//   This follows a pretty set recipe with only elements that can repeat---like
+//   Author, SupplementaryText etc---are at all tricky
+fwrite($xml_out, '<?xml version="1.0" encoding="UTF-8"?>' . "\n");
+fwrite($xml_out, '<!DOCTYPE Manual [' . "\n");
+if (isset($cover_metadata['ENTITY']))
+ {
+   foreach ($cover_metadata['ENTITY'] as $entity)
+   {
+     fwrite($xml_out, "\t" . '<!ENTITY ' . $entity . '>' . "\n");
+   }
+ }
+fwrite($xml_out, ']>' . "\n");
+fwrite($xml_out, '<Manual id="' . ucfirst($_REQUEST['m']) . '" lang="' . $_REQUEST['l'] . '">' . "\n");
+
+///cho "<p>[Debug] metadata: " . print_r($cover_metadata, true) . "</p>\n\n";
+
+outputMetadataSingle($xml_out, $cover_metadata, 'Heading');
+outputMetadataSingle($xml_out, $cover_metadata, 'Title');
+outputMetadataSingle($xml_out, $cover_metadata, 'Author');
+outputMetadataSingle($xml_out, $cover_metadata, 'Affiliation');
+outputMetadataMultiple($xml_out, $cover_metadata, 'SupplementaryText');
+outputMetadataMultiple($xml_out, $cover_metadata, 'Text');
+outputMetadataMultiple($xml_out, $cover_metadata, 'Comment');
+outputMetadataSingle($xml_out, $cover_metadata, 'Version');
+outputMetadataSingle($xml_out, $cover_metadata, 'Date');
+
+// 3. Process each page listed in the contents of the top level page in order
+foreach ($pages_in_order as $page)
+{
+  processPage($xml_out, $page);
+}
+
+// 4. Output out list of footnotes (if any)
+if (!empty($footnotes))
+ {
+   fwrite($xml_out, '<FootnoteList>'. "\n");
+   foreach ($footnotes as $footnote=>$footnote_id)
+   {
+     ///cho '[debug] footnotes: (' . $footnote_id . ') ' . $footnote . '<br />'. "\n";
+     outputMetadataSingle($xml_out, $footnote, 'Footnote', $footnote_id);
+   }
+   fwrite($xml_out, '</FootnoteList>'. "\n");
+ }
+
+// 5. Finalize and close the XML output
+fwrite($xml_out, '</Manual>' . "\n");
+fclose($xml_out);
+
+// 6. Complete!
+echo '<p><b>Complete!</b></p>' . "\n<hr/>\n";
+if ($_REQUEST['a'] == 'download')
+{
+  echo '<p>Click <a href="var/' . ucfirst($_REQUEST['m']) . '_' . $_REQUEST['l'] . '.xml">here</a> to download XML file</p>' . "\n";
+}
+echo '<p>Click <a href="' . $dokuwiki_url . '/doku.php?id=' . $_REQUEST['l'] . ':manuals:' . $_REQUEST['m'] . '">here</a> to return to dokuwiki</p>' . "\n";
+echo '</body>' . "\n";
+echo '</html>';
+exit(0);
+
+/**
+ */
+function outputMetadataSingle($xml_out, $metadata, $field, $mid=false)
+{
+  echo '[metadata: ' . $field . "] \n";
+  if ($mid)
+  {
+    fwrite($xml_out, '<' . $field . ' id="' . $mid . '">' . "\n");
+  }
+  else
+  {
+    fwrite($xml_out, '<' . $field . '>' . "\n");
+  }
+  if (is_array($metadata))
+  {
+    if (isset($metadata[$field]) && isset($metadata[$field][0]))
+    {
+      outputTextBlock($xml_out, $metadata[$field][0]);
+    }
+    else
+    {
+      echo 'no such field or no metadata';
+    }
+  }
+  elseif (!empty($metadata))
+  {
+    outputTextBlock($xml_out, $metadata);
+  }
+  else
+  {
+    echo 'no such field or no metadata';
+  }
+  fwrite($xml_out, '</' . $field . '>' . "\n");
+}
+/** outputMetadataSingle() **/
+
+function outputMetadataList($xml_out, $metadata, $field, $separator = ',', $final_separator = false)
+{
+  echo '[metadata list: ' . $field . "] \n";
+  fwrite($xml_out, '<' . $field . '>' . "\n");
+  if (isset($metadata[$field]))
+  {
+    if (count($metadata[$field]) == 1)
+    {
+      outputTextBlock($xml_out, $metadata[$field][0]);
+    }
+    if (count($metadata[$field]) > 1)
+    {
+      $last_value = '';
+      if ($final_separator)
+      {
+        $last_value = array_pop($metadata[$field]);
+      }
+      $values = implode($separator, $metadata[$field]);
+      if ($final_separator)
+      {
+        $values .= $final_separator . $last_value;
+      }
+      outputTextBlock($xml_out, $values);
+    }
+  }
+  else
+  {
+    echo 'no such field or no metadata';
+  }
+  fwrite($xml_out, '</' . $field . '>' . "\n");
+}
+/** outputMetadataList() **/
+
+function outputMetadataMultiple($xml_out, $metadata, $field)
+{
+  echo '[metadata multiple: ' . $field . "] \n";
+  // - Text blocks don't need to be wrapped in Text element
+  if ($field != 'Text')
+  {
+    fwrite($xml_out, '<' . $field . '>' . "\n");
+  }
+  if (isset($metadata[$field]))
+  {
+    foreach ($metadata[$field] as $value)
+    {
+      outputTextBlock($xml_out, $value);
+    }
+  }
+  else
+  {
+    echo 'no such field or no metadata';
+  }
+  if ($field != 'Text')
+  {
+    fwrite($xml_out, '</' . $field . '>' . "\n");
+  }
+}
+
+function translateTableCodeline($text)
+{
+  // Escape any italic tags hidden in HTML comments
+  $text = str_replace('<!--i-->', '%!--i--%', $text);
+  $text = str_replace('<!--/i-->', '%!--/i--%', $text);
+  // Encode entities etc
+  $text = translateText($text, true);
+  // Restore any italics elements hidden above
+  $text = str_replace('%!--i--%', '<i>', $text);
+  $text = str_replace('%!--/i--%', '</i>', $text);
+  return $text;
+}
+/** translateTableCodeline() **/
+
+function translateText($text, $in_code_block=false)
+{
+  global $allow_bad_codeblocks;
+  $text = str_replace('&','&amp;',$text);
+  $text = str_replace('<','&lt;',$text);
+  $text = str_replace('>','&gt;',$text);
+  if ($in_code_block && $allow_bad_codeblocks)
+  {
+    ///cho "**** [debug] restoring bogus decoded tags in: |$text| ****\n";
+    $text = str_replace('&lt;i&gt;','<i>',$text);
+    $text = str_replace('&lt;/i&gt;','</i>',$text);
+    $text = str_replace('&lt;br/&gt;','<br/>',$text);
+  }
+  return $text;
+}
+
+function outputTextBlock($xml_out, $text, $type='', $in_code_block = false)
+{
+  global $cover_metadata;
+  global $text_id_counter;
+  global $footnotes;
+  global $allow_bad_codeblocks;
+
+  // - Start by dealing with any footnotes before anything else
+  while (preg_match('/\(\((.*?)\)\)/', $text, $matches))
+  {
+    $pattern = $matches[0];
+    $footnote = $matches[1];
+    $footnote_id = count($footnotes) + 1;
+    $footnotes[$footnote] = $footnote_id;
+    // - note that we have to escape the footnote reference as the following
+    //   code will convert any < and > to entities...
+    $footnote_reference = '%FootnoteRef id="' . $footnote_id . '"/%';
+    $text = str_replace($pattern, $footnote_reference, $text);
+  }
+
+  $text_id = '';
+  // - check whether the string begins with an explicit id
+  if (preg_match('/^\s*<!--\s*id:(.+?)\s*-->(.*)$/', $text, $matches))
+  {
+    $text_id = $matches[1];
+    $text = $matches[2];
+    if (is_numeric($text_id))
+    {
+      $text_id_counter = $text_id + 1;
+    }
+  }
+  else
+  {
+    $text_id = $text_id_counter;
+    $text_id_counter++;
+  }
+
+  // - protect the special case of an HTML comment being actually displayed
+  //   in the text
+  $text = preg_replace('/<!--([\s\.]+?)-->/','##lt##!--\1--##gt##',$text);
+
+  // - reformat dokuwiki syntax to HTML tag syntax
+  $text = preg_replace('/<!--.*?-->/', '', $text);
+
+  // we leave code blocks alone in terms of ampersands
+  if (!$in_code_block)
+  {
+    // - ampersands aren't safe in XML...
+    $text = str_replace('&', '&amp;', $text);
+    // ...except for the entities that we have registered as metadata
+    if (isset($cover_metadata['ENTITY']))
+    {
+      foreach ($cover_metadata['ENTITY'] as $entity)
+      {
+        if (preg_match('/([a-z]+)\s+"&#(\d+);"/', $entity, $matches))
+        {
+          $entity_name = $matches[1];
+          if ($entity_name != 'mdash')
+          {
+          $entity_character = html_entity_decode('&#'.$matches[2].';',ENT_NOQUOTES,'UTF-8');
+          $text = str_replace('&amp;' . $entity_name . ';', '&' . $entity_name . ';', $text);
+          // - we also convert any characters that match the entity char into
+          //   the entity
+          $text = str_replace($entity_character, '&' . $entity_name . ';', $text);
+          }
+        }
+      }
+    }
+    // - protect <br/> tags
+    $text = str_replace('<br/>','%%br/%%',$text);
+    // - encoding all of the < and > that appear in the text (rather than
+    //   true html formatting)
+    $text = str_replace('<','&lt;',$text);
+    $text = str_replace('>','&gt;',$text);
+    // - restore <br/> tags
+    $text = str_replace('%%br/%%','<br/>',$text);
+  }
+  else if ($type == 'code')
+  {
+    $text = str_replace('<','&lt;',$text);
+    $text = str_replace('>','&gt;',$text);
+  }
+
+  // - links, oh how I hate thee
+  // - external links are slightly easier
+  $text = preg_replace('/\[\[http:\/\/(.*?)\|(.*?)\]\]/', '<Link url="http://\1">\2</Link>', $text);
+  // - internals have to become the horrible <CrossRef> tags. We ignore any
+  //   number prefix on the page name as that is just used for ordering within
+  //   Dokuwiki
+  $text = preg_replace('/\[\[\.\:(.*?)\|[^\]]+\]\]/','<CrossRef target="Chapter" ref="\1"/>', $text);
+  // - internal links starting with hash must be on the same page
+  $text = preg_replace('/\[\[###(.*?)\|.*?\]\]/','<CrossRef target="Part" ref="\1"/>', $text);
+  $text = preg_replace('/\[\[##(.*?)\|.*?\]\]/','<CrossRef target="Subsection" ref="\1"/>', $text);
+  $text = preg_replace('/\[\[#(.*?)\|.*?\]\]/','<CrossRef target="Section" ref="\1"/>', $text);
+  // - 'external' internal wiki links are even worst - since we can't know what
+  //   the page order number for another manual's chapters might be, we instead
+  //   use a search
+  $text = preg_replace('/\[\[\?do\=search\&amp;id\=([^\s]+)\s+@([a-z]+):manuals:([a-z]+)\|.*?\]\]/i', '<CrossRef external="\3" lang="\2" target="Chapter" ref="\1"/>', $text);
+  // - references to images and tables
+  $text = preg_replace('/(?:<|&lt;)imgref\sfigure_(.+?)(?:>|&gt;)/','<CrossRef target="Figure" ref="\1"/>', $text);
+  $text = preg_replace('/(?:<|&lt;)tblref\stable_(.+?)(?:>|&gt;)/','<CrossRef target="Table" ref="\1"/>', $text);
+  // - explicitly convert URLs as they are a bit messy
+  // - first all the cases of URLs in italics, without protocol
+  $text = preg_replace('/\/\/\s([a-z]+\.[a-z0-9\-]+\.[a-z0-9\.\-]+(?:\/.*?)?)\s\/\//i','<i>\1</i>', $text);
+  $text = preg_replace('/\/\/\s([a-z0-9\-]+\.org(?:\/.*?)?)\s\/\//i','<i>\1</i>', $text);
+  $text = preg_replace('/\/\/\s(localhost(?:\/.*?)?)\s\/\//i','<i>\1</i>', $text);
+  // - now all the protocol ones (with care taken to protect // in protocol)
+  $text = preg_replace('/\/\/\shttp:\/\/([a-z]+\.[a-z0-9\-]+\.[a-z0-9\.\-]+(?:\/.*?)?)\s\/\//i','<i>http:##DOUBLESLASH##\1</i>', $text);
+  $text = preg_replace('/\/\/\shttp:\/\/([a-z0-9\-]+\.org(?:\/.*?)?)\s\/\//i','<i>http:##DOUBLESLASH##\1</i>', $text);
+  $text = preg_replace('/\/\/\shttp:\/\/(localhost(?:\/.*?)?)\s\/\//i','<i>http:##DOUBLESLASH##\1</i>', $text);
+  // - next we have the underlined URLs sans protocols
+  $text = preg_replace('/__\s([a-z]+\.[a-z0-9\-]+\.[a-z0-9\.\-]+(?:\/.*?)?)\s__/i','<u>\1</u>', $text);
+  $text = preg_replace('/__\s([a-z0-9\-]+\.org(?:\/.*?)?)\s__/i','<u>\1</u>', $text);
+  $text = preg_replace('/__\s(localhost(?:\/.*?)?)\s__/i','<u>\1</u>', $text);
+  // - and finally the protocol prefixed underlined URLs
+  $text = preg_replace('/__\shttp:\/\/([a-z]+\.[a-z0-9\-]+\.[a-z0-9\.\-]+(?:\/.*?)?)\s__/i','<u>http:##DOUBLESLASH##\1</u>', $text);
+  $text = preg_replace('/__\shttp:\/\/([a-z0-9\-]+\.org(?:\/.*?)?)\s__/i','<u>http:##DOUBLESLASH##\1</u>', $text);
+  $text = preg_replace('/__\shttp:\/\/(localhost(?:\/.*?)?)\s__/i','<u>http:##DOUBLESLASH##\1</u>', $text);
+  // - lets also protect any other protocols we find floating around
+  $text = preg_replace('/(file|ftp|http):\/\//i', '\1:##DOUBLESLASH##', $text);
+
+  // - italic formatting (taking care of protected double slashes)
+  $text = preg_replace('/%%\/\/%%/', '##DOUBLESLASH##', $text);
+  $text = preg_replace('/\/{5}/', '<i>/</i>', $text); // another special case
+  $text = preg_replace('/\/\/(\/.+?)\s*\/\//', '<i>\1</i>', $text); // another special case
+  $text = preg_replace('/\/\/\s*(.+?\/)\/\//', '<i>\1</i>', $text); // another special case
+  $text = preg_replace('/\/\/\s*(.+?)\s*\/\//', '<i>\1</i>', $text);
+  $text = preg_replace('/##DOUBLESLASH##/', '//', $text);
+  // - bold formatting
+  $text = preg_replace('/\*\*([^"]+?)\*\*/', '<b>\1</b>', $text);
+  // - underline formatting
+  $text = preg_replace('/__([^"]+?)__/', '<u>\1</u>', $text);
+
+  // - decode certain entities in codeblock (just because they are valid HTML,
+  //   derp).
+  if ($in_code_block && $allow_bad_codeblocks)
+  {
+    ///cho "**** [debug] restoring bogus decoded tags in: |$text| ****\n";
+    $text = str_replace('&lt;i&gt;','<i>',$text);
+    $text = str_replace('&lt;/i&gt;','</i>',$text);
+    //$text = str_replace('&lt;br/&gt;','<br/>',$text);
+  }
+  // - restore protected entities
+  $text = preg_replace('/##(gt|lt)##/','&\1;',$text);
+  // - restore protected comment blocks
+  $text = str_replace('%!--', '&lt;!--', $text);
+  $text = str_replace('--%', '--&gt;', $text);
+  // - restore protected footnote refs
+  $text = preg_replace('/%FootnoteRef id="([^"]+)"\/%/', '<FootnoteRef id="\1"/>', $text);
+  // output the text block
+  $text = trim($text);
+  if (empty($text))
+  {
+    fwrite($xml_out, '<Text id="' . $text_id . '"/>' . "\n");
+  }
+  else if (!empty($type))
+  {
+    fwrite($xml_out, '<Text type="' . $type . '" id="' . $text_id . '">' . $text . '</Text>' . "\n");
+  }
+  else
+  {
+    fwrite($xml_out, '<Text id="' . $text_id . '">' . $text . '</Text>' . "\n");
+  }
+}
+/** outputTextBlock($xml_out, $text) **/
+
+/**
+ */
+function processPage($xml_out, $page_name)
+{
+  global $dokuwiki_path;
+  global $seen_ids;
+  echo "</p>\n<p><b>Export Chapter:</b> " . $page_name . "<br/>\n";
+  // - locate the page in question (taking into account if the user asked for a
+  //   draft version or an approved version of the manual)
+  $page_path = '';
+  if ($_REQUEST['v'] == 'draft' || $_REQUEST['l'] != 'en')
+  {
+    $page_path = $dokuwiki_path . '/data/pages/' . $_REQUEST['l'] . '/manuals/' . $_REQUEST['m'] . '/' . $page_name . '.txt';
+  }
+  else
+  {
+    $page_path = getApprovedVersionPath('en:manuals:' . $_REQUEST['m'] . ':' . $page_name);
+  }
+  $page_in = @fopen($page_path, 'r');
+  if (!$page_in)
+  {
+    printError('Failed to open page for reading:' . $page_name, false);
+    return;
+  }
+  // - once again we read in line-by-line, but this time we are going to output
+  //   each line as we go through. We expect to encounter certain lines in a
+  //   predefined order, and should complain if we don't find what we expect.
+  $in_chapter = false;
+  $in_section = false;
+  $in_subsection = false;
+  $in_part = false;
+  $in_list = false;
+  $lists = array();
+  $previous_listitem_type = '';
+  $in_figure = false;
+  $in_table = false;
+  $column_widths = array();
+  $in_code_block = false;
+  while (($line = fgets($page_in)) !== false)
+  {
+    // remove newline character
+    $line = preg_replace('/\r?\n$/','',$line);
+    // - we need to know the 'depth' for the bulletpoint lists
+    $depth = 0;
+    while (strlen($line) > 2 && preg_match('/^\s+[\*\-]/', $line) && substr($line, 0, 2) == '  ')
+    {
+      $depth++;
+      $line = substr($line, 2);
+    }
+    $first_character = substr($line, 0, 1);
+    // - special case for the end of bullet lists
+    if ($in_list && ($first_character != "*" && $first_character != "-"))
+    {
+      while (count($lists) > 0)
+      {
+        $list_type = array_pop($lists);
+        if ($list_type == '*')
+        {
+          fwrite($xml_out, '</Bullet>' . "\n");
+          fwrite($xml_out, '</BulletList>' . "\n");
+        }
+        else
+        {
+          fwrite($xml_out, '</NumberedItem>' . "\n");
+          fwrite($xml_out, '</NumberedList>' . "\n");
+        }
+      }
+      $in_list = false;
+    }
+    // - special case for the end of tables
+    if ($in_table && $first_character != '^' && $first_character != '|')
+    {
+      fwrite($xml_out, '</TableContent>' . "\n");
+      fwrite($xml_out, '</Table>' . "\n");
+      $in_table = false;
+    }
+    // - special cases for premature closing of sections, subsections and parts
+    if (preg_match('/<!-- close:(section|subsection|part) -->/', $line, $matches))
+    {
+      // - we always try to do this (regardless of actual flag) as we must
+      //   always close the smallest 'granularity' first
+      if ($in_part)
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Part>' . "\n");
+        $in_part = false;
+      }
+      if ($in_subsection && ($matches[1] == 'section' || $matches[1] == 'subsection'))
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Subsection>' . "\n");
+        $in_subsection = false;
+      }
+      if ($in_section && $matches[1] == 'section')
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Section>' . "\n");
+        $in_section = false;
+      }
+    }
+
+    // - if this page is a chapter, then the first thing on the page should be
+    //   the chapter title (six equals)
+    if (preg_match('/====== (.+) ======/', $line, $matches))
+    {
+      $chapter_title = $matches[1];
+      $chapter_id = $page_name;
+      if (empty($chapter_id))
+      {
+        $chapter_id = generateID($chapter_title);
+      }
+      // - are we already processing a part? if so end it, end it now
+      if ($in_part)
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Part>' . "\n");
+        $in_part = false;
+      }
+      // - are we already processing a subsection? if so end it, end it now
+      if ($in_subsection)
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Subsection>' . "\n");
+        $in_subsection = false;
+      }
+      // - are we already processing a section? if so end it, end it now
+      if ($in_section)
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Section>' . "\n");
+        $in_section = false;
+      }
+      // - are we already processing a chapter? if so end it, end it now
+      if ($in_chapter)
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Chapter>' . "\n");
+        $in_chapter = false;
+      }
+      // - write out this chapter's header
+      fwrite($xml_out, '<Chapter id="' . $chapter_id . '">' . "\n");
+      outputMetadataSingle($xml_out, $chapter_title, 'Title');
+      fwrite($xml_out, '<Content>' . "\n");
+      $in_chapter = true;
+    }
+    // - the next likely thing to encounter is a section heading (five equals)
+    elseif (preg_match('/=====\s+(.+)\s+=====/', $line, $matches))
+    {
+      $section_title = $matches[1];
+      // - check for explicit section id
+      $section_id = '';
+      if (preg_match('/<!-- sid:(.+?) -->(.*)/', $section_title, $matches))
+      {
+        $section_id = $matches[1];
+        $section_title = $matches[2];
+      }
+      if (empty($section_id))
+      {
+        $section_id = generateID($section_title);
+      }
+      // - are we already processing a part? if so end it, end it now
+      if ($in_part)
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Part>' . "\n");
+        $in_part = false;
+      }
+      // - are we already processing a subsection? if so end it, end it now
+      if ($in_subsection)
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Subsection>' . "\n");
+        $in_subsection = false;
+      }
+      // - are we already processing a section? if so end it, end it now
+      if ($in_section)
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Section>' . "\n");
+        $in_section = false;
+      }
+      // - write out this section's header
+      fwrite($xml_out, '<Section id="' . $section_id . '">' . "\n");
+      outputMetadataSingle($xml_out, $section_title, 'Title');
+      fwrite($xml_out, '<Content>' . "\n");
+      $in_section = true;
+    }
+    // - similar for subsection heading (four equals)
+    elseif (preg_match('/==== (.+) ====/', $line, $matches))
+    {
+      $subsection_title = $matches[1];
+      // - check for explicit subsection id
+      $subsection_id = '';
+      if (preg_match('/<!-- sid:(.+?) -->(.*)/', $subsection_title, $matches))
+      {
+        $subsection_id = $matches[1];
+        $subsection_title = $matches[2];
+      }
+      if (empty($subsection_id))
+      {
+        $subsection_id = generateID($subsection_title);
+      }
+      // - are we already processing a part? if so end it, end it now
+      if ($in_part)
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Part>' . "\n");
+        $in_part = false;
+      }
+      // - are we already processing a subsection? if so end it, end it now
+      if ($in_subsection)
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Subsection>' . "\n");
+        $in_subsection = false;
+      }
+      // - write out this subsection's header
+      fwrite($xml_out, '<Subsection id="' . $subsection_id . '">' . "\n");
+      outputMetadataSingle($xml_out, $subsection_title, 'Title');
+      fwrite($xml_out, '<Content>' . "\n");
+      $in_subsection = true;
+    }
+    // - and part heading (three equals)
+    elseif (preg_match('/=== (.+) ===/', $line, $matches))
+    {
+      $part_title = $matches[1];
+      // - check for explicit part id
+      $part_id = '';
+      if (preg_match('/<!-- sid:(.+?) -->(.*)/', $part_title, $matches))
+      {
+        $part_id = $matches[1];
+        $part_title = $matches[2];
+      }
+      if (empty($part_id))
+      {
+        $part_id = generateID($part_title);
+      }
+      // - are we already processing a part? if so end it, end it now
+      if ($in_part)
+      {
+        fwrite($xml_out, '</Content>' . "\n");
+        fwrite($xml_out, '</Part>' . "\n");
+        $in_part = false;
+      }
+      // - write out this part's header
+      fwrite($xml_out, '<Part id="' . $part_id . '">' . "\n");
+      outputMetadataSingle($xml_out, '**//' . $part_title . '//**', 'Title');
+      fwrite($xml_out, '<Content>' . "\n");
+      $in_part = true;
+    }
+    // - Ignore 5th level heading - they are only used to allow more convenient
+    //   editing of figures and tables
+    elseif (preg_match('/== (.+) ==/', $line, $matches))
+    {
+    }
+    // - lists need special handling
+    elseif (preg_match('/^(\*|\-)\s+(.*)/', $line, $matches))
+    {
+      $list_type = $matches[1];
+      $list_text = $matches[2];
+      $list_depth = count($lists);
+      if (!$in_list)
+      {
+        if ($list_type == '*')
+        {
+          fwrite($xml_out, '<BulletList>' . "\n");
+        }
+        else
+        {
+          fwrite($xml_out, '<NumberedList>' . "\n");
+        }
+        $in_list = true;
+        array_push($lists, $list_type);
+      }
+      // - this bullet is at the same depth as previous - close the previous
+      //   point
+      elseif ($depth == $list_depth)
+      {
+        $previous_list_type = end($lists);
+        if ($previous_list_type == '*')
+        {
+          fwrite($xml_out, '</Bullet>' . "\n");
+        }
+        else
+        {
+          fwrite($xml_out, '</NumberedItem>' . "\n");
+        }
+        // - we don't match in type anymore... close the previous list and open
+        //   a new list of the appropriate type
+        if ($list_type != $previous_list_type)
+        {
+          if ($previous_list_type == '*')
+          {
+            fwrite($xml_out, '</BulletList>' . "\n");
+            fwrite($xml_out, '<NumberedList>' . "\n");
+          }
+          else
+          {
+            fwrite($xml_out, '</NumberedNumbered>' . "\n");
+            fwrite($xml_out, '<BulletList>' . "\n");
+          }
+          array_pop($lists);
+          array_push($lists, $list_type);
+        }
+      }
+      else
+      {
+        // - we have either got deeper...
+        if ($depth > $list_depth)
+        {
+          if ($list_type == '*')
+          {
+            fwrite($xml_out, '<BulletList>' . "\n");
+          }
+          else
+          {
+            fwrite($xml_out, '<NumberedList>' . "\n");
+          }
+          array_push($lists, $list_type);
+        }
+        // ... or shallower in the bullet listing
+        if ($depth < $list_depth)
+        {
+          $previous_list_type = array_pop($lists);
+          if ($previous_list_type == '*')
+          {
+            fwrite($xml_out, '</Bullet>' . "\n");
+            fwrite($xml_out, '</BulletList>' . "\n");
+          }
+          else
+          {
+            fwrite($xml_out, '</NumberedItem>' . "\n");
+            fwrite($xml_out, '</NumberedList>' . "\n");
+          }
+          // - we still have to close the last item too
+          $previous_listitem_type = end($lists);
+          if ($previous_listitem_type == '*')
+          {
+            fwrite($xml_out, '</Bullet>' . "\n");
+          }
+          else
+          {
+            fwrite($xml_out, '</NumberedItem>' . "\n");
+          }
+        }
+      }
+      if ($list_type == '*')
+      {
+        fwrite($xml_out, '<Bullet>' . "\n");
+      }
+      else
+      {
+        fwrite($xml_out, '<NumberedItem>' . "\n");
+      }
+      // Special Case: bullets that contain (start) a code block
+      if (preg_match('/^(.*)<code>\s*$/', $list_text, $matches))
+      {
+        $list_text = $matches[1];
+        $in_code_block = true;
+      }
+
+      outputTextBlock($xml_out, $list_text);
+
+      // - to make things clearer, we'll process any and all code blocks within
+      //   bullets here - especially as there may be more text block *after*
+      //   the code block finishes
+      if ($in_code_block)
+      {
+        $sub_line = '';
+        while ($in_code_block && ($sub_line = fgets($page_in)) !== false)
+        {
+          $sub_line = trim($sub_line);
+          // - closing code
+          if (preg_match('/^<\/code>(.*)$/', $sub_line, $matches))
+          {
+            $sub_line = $matches[1]; // may be empty string
+            $in_code_block = false;
+          }
+          // - output another plain codeline
+          else
+          {
+            fwrite($xml_out, '<CodeLine>' . $sub_line . "</CodeLine>\n");
+            $sub_line = '';
+          }
+        }
+        // - if sub_line still has anything in it, then add that content as a
+        //   text block
+        if (!empty($sub_line))
+        {
+          outputTextBlock($xml_out, $sub_line);
+        }
+      }
+    }
+    // - images start with an image caption 'element'
+    elseif (preg_match('/<imgcaption\s+figure_([a-z0-9_\-]+)\|(.+)>\s*<\/imgcaption>/', $line, $matches))
+    {
+      $figure_id = $matches[1];
+      $figure_title = $matches[2];
+      // - watch for the special withLineNumber flag
+      $class_attribute = '';
+      if (strpos($figure_title, '%!-- withLineNumber --%') != false)
+      {
+        $class_attribute = ' class="withLineNumber"';
+        $figure_title = str_replace('%!-- withLineNumber --%','',$figure_title);
+      }
+      fwrite($xml_out, '<Figure id="' . $figure_id . '"' . $class_attribute . '>' . "\n");
+      echo '[figure: ' . $figure_id . "] \n";
+      fwrite($xml_out, '<Title>' . "\n");
+      // - decode any comments in the title (used to store explicit id
+      //   information)
+      $figure_title = str_replace('%!--', '<!--', $figure_title);
+      $figure_title = str_replace('--%', '-->', $figure_title);
+      // - special case: the title may have a subtitle (as a prefix)
+      $figure_subtitle_id = '';
+      $figure_subtitle = '';
+      // - subtitle with explicit id
+      if (preg_match('/^(<!-- id:.+? -->\([a-z]\))\s*(.*)$/', $figure_title, $matches))
+      {
+        $figure_subtitle = $matches[1];
+        $figure_title = $matches[2];
+      }
+      // - subtitle without explicit id
+      else if (preg_match('/^(\([a-z]\))\s*(.*)$/', $figure_title, $matches))
+      {
+        $figure_subtitle = $matches[1];
+        $figure_title = $matches[2];
+      }
+      outputTextBlock($xml_out, $figure_title);
+      if (!empty($figure_subtitle))
+      {
+        fwrite($xml_out, '<SubTitle>' . "\n");
+        outputTextBlock($xml_out, $figure_subtitle);
+        fwrite($xml_out, '</SubTitle>' . "\n");
+      }
+      fwrite($xml_out, '</Title>' . "\n");
+      $in_figure = true;
+      // - record the id to prevent repeating
+      $seen_ids[$figure_id] = true;
+    }
+    // - tables start with a table caption 'element'
+    elseif (preg_match('/<tblcaption\s+table_([a-z0-9_\-]+)\|([^>]+)>\s*<\/tblcaption>/', $line, $matches))
+    {
+      $table_id = $matches[1];
+      $table_title = $matches[2];
+      if ($table_title == '##NOCAPTION##')
+      {
+        echo '[non-captioned table: ' . $table_id . "] \n";
+        // - watch for autogenerated ids... no point in outputting them
+        if (preg_match('/^table(_\d+)?$/', $table_id))
+        {
+          fwrite($xml_out, "<Table>\n");
+        }
+        else
+        {
+          fwrite($xml_out, '<Table id="' . $table_id . '">' . "\n");
+        }
+        fwrite($xml_out, '<Title/>' . "\n");
+      }
+      elseif ($table_title == '##HIDDEN##')
+      {
+        echo '[hidden table: ' . $table_id . "] \n";
+        // - watch for autogenerated ids... no point in outputting them
+        if (preg_match('/^table(_\d+)?$/', $table_id))
+        {
+          fwrite($xml_out, "<Table class=\"hidden\">\n");
+        }
+        else
+        {
+          fwrite($xml_out, '<Table class="hidden" id="' . $table_id . '">' . "\n");
+        }
+        fwrite($xml_out, '<Title/>' . "\n");
+      }
+      else
+      {
+        echo '[table: ' . $table_id . "] \n";
+        // - watch for autogenerated ids... no point in outputting them
+        if (preg_match('/^table(_\d+)?$/', $table_id))
+        {
+          fwrite($xml_out, "<Table>\n");
+        }
+        else
+        {
+          fwrite($xml_out, '<Table id="' . $table_id . '">' . "\n");
+        }
+        fwrite($xml_out, '<Title>' . "\n");
+        outputTextBlock($xml_out, $table_title);
+        fwrite($xml_out, '</Title>' . "\n");
+      }
+      fwrite($xml_out, '<TableContent>' . "\n");
+      $in_table = true;
+      // - record the id to prevent repeating
+      $seen_ids[$table_id] = true;
+    }
+    // - the second line in a table should be it's column width values
+    elseif (preg_match('/\|<\s-\s([0-9 ]+?)\s>\|/', $line, $matches))
+    {
+      $column_widths = explode(' ', $matches[1]);
+    }
+    // - then every row will be made of a number of cells
+    elseif (preg_match('/^\|(.*?)\|$/', $line, $matches))
+    {
+      $row_content = $matches[1];
+      $cell_contents = preg_split('/(\s+\||\|\s+)/', $row_content);
+      fwrite($xml_out, '<tr>' . "\n");
+      foreach ($cell_contents as $index=>$cell_content)
+      {
+        $cell_content = trim($cell_content);
+        $th_text = '';
+        if (isset($column_widths[$index]))
+        {
+          $th_text = '<th width="' . $column_widths[$index] . '"';
+        }
+        else
+        {
+          $th_text = '<th';
+        }
+        // - if the cell would be empty, we use the shorthand
+        if (empty($cell_content))
+        {
+          $th_text .= '/>' . "\n";
+          fwrite($xml_out, $th_text);
+        }
+        else
+        {
+          $th_text .= '>' . "\n";
+          fwrite($xml_out, $th_text);
+
+          // GAH - this is proving harder than a hard thing thats hard.
+          // The issue is that the most straightforward way of fixing this,
+          // namely using explicit newlines (\\) in the dokuwiki txt causes
+          // lots a legitimately translated <br/> to also be split up. I
+          // think the only way forward would be to maybe extend the HTML
+          // Comment plugin to also respect and process <br/> tags. Then I
+          // can avoid transforming them, and use the \\ sentinel to
+          // separate multi-line table cells.
+          $cell_content_lines = explode('\\\\', $cell_content);
+          foreach ($cell_content_lines as $cell_content)
+          {
+            // - watch out, as the content may be an image
+            if (preg_match('/\{\{.+?[^:?]+\?\d+x\d+(&direct)?\}\}/', $cell_content))
+            {
+              processImage($xml_out, $cell_content);
+            }
+            elseif (preg_match('/\'\'(.*)\'\'/', $cell_content, $matches))
+            {
+              fwrite($xml_out, '<CodeLine>' . translateTableCodeline($matches[1]) . '</CodeLine>' . "\n");
+            }
+            // - anything else it text
+            else
+            {
+              outputTextBlock($xml_out, $cell_content);
+            }
+          }
+          fwrite($xml_out, '</th>' . "\n");
+        }
+      }
+      fwrite($xml_out, '</tr>' . "\n");
+    }
+    // - links to image media in the wiki!
+    elseif (preg_match('/\{\{.+?[^:?]+\?\d+x\d+(&direct)?\}\}/', $line))
+    {
+      processImage($xml_out, $line);
+      // - if we were processing a figure, then now is a good time to close it
+      if ($in_figure)
+      {
+        fwrite($xml_out, '</Figure>' . "\n");
+        $in_figure = false;
+      }
+    }
+    // - if the line starts with a <code> block, then we have a tag
+    //   for that (which is special in that it get a unique text id)
+    elseif (preg_match('/^<code\s*\d*\s*>(.*?)(<\/code>)?$/', $line, $matches) || ($in_code_block && preg_match('/^(.*?)(<\/code>)?$/', $line, $matches)))
+    {
+      $payload = $matches[1];
+      $found_end = (isset($matches[2]));
+      $in_code_block = true;
+      // - be careful with empty lines
+      if (empty($payload))
+      {
+        // - as they may appear in the body of the code (in which case we need
+        //   to output them). The empty lines at the start or end of a code
+        //   block are just an unfortunate consequence of the support for code
+        //   line numbering.
+        if (!$found_end && strpos($line, '<code') === false)
+        {
+          fwrite($xml_out, "<CodeLine/>\n");
+        }
+      }
+      elseif (preg_match('/^<!-- id:([^\s]+) -->/', $payload, $matches))
+      {
+        $text_id = $matches[1];
+        outputTextBlock($xml_out, $payload, 'code', true);
+        // - record the id to prevent repeating
+        $seen_ids[$text_id] = true;
+      }
+      else
+      {
+        fwrite($xml_out, '<CodeLine>' . translateText($payload, true) . '</CodeLine>' . "\n");
+      }
+      // - if we didn't find an endtag we have to keep doing code mode until
+      //   we do
+      $in_code_block = (!$found_end);
+      if ($found_end)
+      {
+        // - if we were processing a figure, then now is a good time to close it
+        if ($in_figure)
+        {
+          fwrite($xml_out, '</Figure>' . "\n");
+          $in_figure = false;
+        }
+      }
+    }
+    // - entities on a line by themselves (i.e. references to external files)
+    //   go through verbatim
+    elseif (preg_match('/^\s*&[a-z0-9_-]+;\s*$/', $line))
+    {
+      fwrite($xml_out, $line . "\n");
+    }
+    // - lines starting with > are indented text blocks
+    elseif (preg_match('/^>(.*)$/', $line, $matches))
+    {
+      $payload = $matches[1];
+      fwrite($xml_out, "<Indented>\n");
+      outputTextBlock($xml_out, $payload);
+      fwrite($xml_out, "</Indented>\n");
+    }
+    // - everything else goes straight through as a text block
+    // - note that for code blocks, even empty lines count
+    elseif (!empty($line))
+    {
+      // - output the line of text having encoded entities etc
+      outputTextBlock($xml_out, $line, '', $in_code_block);
+    }
+  }
+  // Complete any open part
+  if ($in_part)
+  {
+    fwrite($xml_out, '</Content>' . "\n");
+    fwrite($xml_out, '</Part>' . "\n");
+    $in_part = false;
+  }
+  // Complete any open subsection
+  if ($in_subsection)
+  {
+    fwrite($xml_out, '</Content>' . "\n");
+    fwrite($xml_out, '</Subsection>' . "\n");
+    $in_subsection = false;
+  }
+  // Complete any open section
+  if ($in_section)
+  {
+    fwrite($xml_out, '</Content>' . "\n");
+    fwrite($xml_out, '</Section>' . "\n");
+    $in_section = false;
+  }
+  // Complete any open chapter
+  if ($in_chapter)
+  {
+    fwrite($xml_out, '</Content>' . "\n");
+    fwrite($xml_out, '</Chapter>' . "\n");
+    $in_chapter = false;
+  }
+}
+/** processPage($xml_out, $page_name) **/
+
+function processImage($xml_out, $text)
+{
+  global $dokuwiki_path;
+  global $xml_source_path;
+  if (preg_match('/\{\{.+?([^:?]+)\?(\d+)x(\d+)(&direct)?\}\}/', $text, $matches))
+  {
+    $filename = $matches[1];
+    $width = $matches[2];
+    $height = $matches[3];
+    // - copy the file into place
+    $image_source_path = $dokuwiki_path . '/data/media/' . $_REQUEST['l'] . '/manuals/images/' . strtolower($filename);
+    $image_destination_dir = $xml_source_path . '/' . $_REQUEST['l'] . '/images';
+    mkAllDir($image_destination_dir, 0755);
+    $image_destination_path = $image_destination_dir . '/' . $filename;
+    if (copy($image_source_path, $image_destination_path))
+    {
+      echo '[copying file: ' . $filename . "] \n";
+    }
+    else
+    {
+      printError('Failed to copy image into place: ' . $filename, false);
+    }
+    // - spit out the XML element
+    fwrite($xml_out, '<File width="' . $width . '" height="' . $height . '" url="images/' . $filename . '"/>' . "\n");
+  }
+}
Index: /documentation/trunk/php/gs-manual-import.php
===================================================================
--- /documentation/trunk/php/gs-manual-import.php	(revision 25026)
+++ /documentation/trunk/php/gs-manual-import.php	(revision 25026)
@@ -0,0 +1,1179 @@
+<?php
+
+require_once('common.php');
+
+/** @file gs-manual-export.php
+ *  This script transforms the single XML manual file required by the rest of
+ *  the Greenstone manual generation scripts into the series of dokuwiki pages
+ *  that make up a certain manual (as specified by the 'm' argument) in a
+ *  certain language ('l').
+ */
+
+if (!parseCLIArguments())
+ {
+   printError("Error! Failed to parse arguments...\nUsage: gs-manual-import.php -m [user|install|develop|paper]");
+ }
+
+// 0. Initialization
+if (!isset($_REQUEST['l']) || empty($_REQUEST['l']))
+{
+  $_REQUEST['l'] = 'en';
+}
+if (!isset($_REQUEST['m']) || empty($_REQUEST['m']))
+{
+  $_REQUEST['m'] = 'user';
+  //$_REQUEST['m'] = 'install';
+  //$_REQUEST['m'] = 'develop';
+  //$_REQUEST['m'] = 'paper';
+}
+
+// - validate arguments before we use them (security)
+if (!preg_match('/^(develop|install|paper|user)$/',$_REQUEST['m']))
+ {
+   printError('Unknown manual type requested: ' . htmlspecialchars($_REQUEST['m']));
+ }
+
+if (!preg_match('/^(ar|en|es|fr|pt-br|ru)$/',$_REQUEST['l']))
+ {
+   printError('Unknown language requested: ' . htmlspecialchars($_REQUEST['l']));
+ }
+
+echo '<h2>Importing Greenstone Manual XML</h2>' . "\n";
+echo '<p><b>XML Source Path:</b> ' . $xml_source_path . '<br/><b>Manual:</b> ' . $_REQUEST['m'] . '<br/><b>Language:</b> ' . $_REQUEST['l'] . "</p>\n<hr/>\n";
+echo "<p><b>Frontmatter: </b><br/>\n";
+// 1. By-and-large we're going to process all of this in a big state machine
+// - the top level page, containing cover page and chapter order information,
+//   needs to be created last, so we have to store it's information
+$manual_metadata = array();
+$entity_replacements = array();
+$footnotes = array();
+$page_order = array();
+$page_count = 2;
+$looking_for_metadata = '';
+$chapter_txt_out = false;
+$frontmatter_text = '';
+$in_section = false;
+$sections_page_name = '';
+$in_chapter = false;
+$chapter_id = '';
+$bullet_depth = 0;
+$is_numbered_list = true;
+$line_counter = 0;
+$in_code = false;
+$in_footnotes = false;
+$in_numbered_item = 0;
+$in_bullet_item = false;
+$seen_code_in_item = false;
+$in_indent = false;
+$is_code_linenumbered = false;
+// - construct the path using the information we've been provided as arguments
+$xml_file_path = $xml_source_path . '/' . $_REQUEST['l'] . '/' . ucfirst($_REQUEST['m']) . '_' . $_REQUEST['l'] . '.xml';
+$xml_in = fopen($xml_file_path, 'r');
+if (!$xml_in)
+ {
+   printError('Failed to locate top level page for manual');
+ }
+// - we also use this opportunity to read in any footnotes as we'll need to
+//   move them onto their appropriate page
+while (($line = fgets($xml_in)) !== false)
+ {
+   if (preg_match('/<Footnote id="(\d+)">/', $line, $matches))
+   {
+     $footnote_id = $matches[1];
+     $text_line = fgets($xml_in);
+     if (preg_match('/<Text id="([^"]+)">(.+?)<\/Text>/', $text_line, $matches))
+     {
+       $footnotes[$footnote_id] = '<!-- id:' . $matches[1] . ' -->' . translateText($matches[2]);
+     }
+     // - throw away </Footnote>
+     fgets($xml_in);
+   }
+ }
+fclose($xml_in);
+// - now reopen to parse it
+$xml_in = fopen($xml_file_path, 'r');
+if (!$xml_in)
+ {
+   printError('Failed to locate top level page for manual');
+ }
+while (($line = getLine($xml_in)) !== false)
+ {
+   // - Special Case: lingering code blocks, continue if next line also
+   //   contains code, otherwise we need an extra newline
+   if ($in_code)
+   {
+     $code_text = "\n";
+     if (strpos($line, '<CodeLine') === false && strpos($line, 'type="code"') === false)
+     {
+       $code_text .= '</code>';
+       // - codeblocks that appear inside numbered lists do not get their own
+       //   newlines as that would split the item. Instead newlines will be
+       //   added when </NumberedItem> encountered.
+       if (!$is_numbered_list)
+       {
+         $code_text .= "\n\n";
+       }
+       else
+       {
+         $seen_code_in_item = true;
+       }
+       $in_code = false;
+     }
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, $code_text);
+     }
+     else
+     {
+       $frontmatter_text .= $code_text;
+     }
+   }
+   // - some system metadata to watch for
+   if (preg_match('/<!ENTITY\s+([^>]+)>/', $line, $matches))
+   {
+     $entity = $matches[1];
+     addMetadata('ENTITY',$entity);
+     if (preg_match('/([a-z]+)\s+"&#(\d+);"/', $entity, $matches))
+     {
+       $entity_replacements[$matches[1]] = $matches[2];
+     }
+   }
+   // - we have an explicit list of cover metadata to watch for
+   elseif (!$in_section && !$in_chapter && preg_match('/<(Author|Affiliation|Comment|Date|Heading|SupplementaryText|Title|Version)>/', $line, $matches))
+   {
+     $looking_for_metadata = $matches[1];
+   }
+   elseif (!$in_section && !$in_chapter && preg_match('/<\/(Author|Affiliation|Comment|Date|Heading|SupplementaryText|Title|Version)>/', $line, $matches))
+   {
+     $looking_for_metadata = '';
+   }
+   // - found metadata we have!
+   elseif (!empty($looking_for_metadata) && preg_match('/<Text id="([^"]+)">(.+?)<\/Text>/', $line, $matches))
+   {
+     $text_id = $matches[1];
+     $text = '<!-- id:' . $text_id . ' -->' . translateText($matches[2]);
+     addMetadata($looking_for_metadata, $text);
+   }
+   // - any text we encounter outside of both sections and chapters also
+   //   belongs on the cover
+   elseif (!$in_section && !$in_chapter && !$in_footnotes && preg_match('/<Text id="([^"]+)">(.+?)<\/Text>/', $line, $matches))
+   {
+     // (for now I'll assume id's are persistent)
+     addMetadata('Text', '<!-- id:' . $matches[1] . ' -->' . translateText($matches[2]));
+   }
+   // - we will probably encounter the opening section (which is outside of a
+   //   chapter) first, so we have a special case for it
+   elseif (!$in_chapter && preg_match('/<Section id="([^"]+)">/', $line, $matches))
+   {
+     $section_id = $matches[1];
+     // - if this is the first non-chapter section we have encountered then it
+     //   gets the honor of having the page---that these sections will
+     //   eventually be printed out on---named after it. Typically this should
+     //   be "about_this_manual"
+     if (empty($frontmatter_text))
+     {
+       $sections_page_name = $section_id;
+     }
+     $in_section = true;
+     $title = getTitle($xml_in, 'section:' . $section_id);
+     if (empty($frontmatter_text))
+     {
+       array_unshift($page_order, $sections_page_name . '|' . noComments($title));
+     }
+     // - if the title, as is, wouldn't autogenerate the appropriate id, then
+     //   we have to include the id explicitly (as another html comment block)
+     if ($section_id != generateID($title))
+     {
+       $title = '<!-- sid:' . $section_id . ' -->' . $title;
+       $seen_ids[$section_id] = 1;
+     }
+     $frontmatter_text .= '===== ' . $title . ' =====' . "\n\n";
+     // - whew. Chapter's going to be just as bad though.
+   }
+   elseif ($in_section && preg_match('/<\/Section>/', $line))
+   {
+     $in_section = false;
+   }
+   elseif (preg_match('/<Chapter id="([^"]+)">/', $line, $matches))
+   {
+     $chapter_id = $matches[1];
+     echo "</p>\n<p><b>Import Chapter:</b>" . $chapter_id . '<br/>' . "\n";
+     $chapter_page_name = $chapter_id;
+     // - create a new file to store this chapter
+     $chapter_file_dir = $dokuwiki_path . '/data/pages/' . $_REQUEST['l'] . '/manuals/' . $_REQUEST['m'];
+     if (!file_exists($chapter_file_dir))
+     {
+       mkAllDir($chapter_file_dir, 0755);
+     }
+     $chapter_file_path = $chapter_file_dir . '/' . $chapter_page_name . '.txt';
+     // - backup existing file
+     if (file_exists($chapter_file_path))
+     {
+       $chapter_backup_file_path = $chapter_file_path . '.bak';
+       rename($chapter_file_path, $chapter_backup_file_path);
+     }
+     // - open new file for writing
+     $chapter_txt_out = fopen($chapter_file_path, 'w');
+     if (!$chapter_txt_out)
+     {
+       printError('Failed to open page file for writing: ' . $chapter_page_name);
+     }
+     $in_chapter = true;
+     $title = getTitle($xml_in, 'chapter: ' . $chapter_id);
+     fwrite($chapter_txt_out, '====== ' . $title . ' ======' . "\n\n");
+     array_push($page_order, $chapter_page_name . '|' . noComments($title));
+   }
+   elseif ($in_chapter && preg_match('/<\/Chapter>/', $line))
+   {
+     fclose($chapter_txt_out);
+     $chapter_txt_out = false;
+     $in_chapter = false;
+     $page_count++;
+   }
+   // - section, subsection and part titles within chapter
+   elseif ($in_chapter && preg_match('/<(Section|Subsection|Part)\sid="([^"]+)">/', $line, $matches))
+   {
+     $title_type = $matches[1];
+     $section_id = $matches[2];
+     $header_fix = '';
+     $title = getTitle($xml_in, 'heading: ' . $title_type);
+     if ($title_type == 'Section')
+     {
+       $header_fix = '=====';
+     }
+     if ($title_type == 'Subsection')
+     {
+       $header_fix = '====';
+     }
+     if ($title_type == 'Part')
+     {
+       $header_fix = '===';
+       // - remove b's and i's
+       $title = preg_replace('/<\/?(B|I)>/i', '', $title);
+     }
+     // - if the title, as is, wouldn't autogenerate the appropriate id, then
+     //   we have to include the id explicitly (as another html comment block)
+     if ($section_id != generateID($title))
+     {
+       $title = '<!-- sid:' . $section_id . ' -->' . $title;
+       $seen_ids[$section_id] = 1;
+     }
+     fwrite($chapter_txt_out, $header_fix . ' ' . $title . ' ' . $header_fix . "\n\n");
+   }
+   elseif ($in_chapter && (strpos($line, '</Section') !== false || strpos($line, '</Subsection') !== false  || strpos($line, '</Part') !== false))
+   {
+     // do nothing for now
+   }
+   // - figures (and their titles/captions)
+   elseif (preg_match('/<Figure id="([^"]+)"(.*?)>/', $line, $matches))
+   {
+     $figure_id = $matches[1];
+     $other_attributes = $matches[2];
+     echo '[adding figure: ' . $figure_id . "] \n";
+     // We need the title too
+     $caption = getTitle($xml_in, 'figure:' . $figure_id);
+     $caption = translateText(alternateComments($caption));
+     $txt = "<imgcaption figure_" . $figure_id . '|' . $caption . ' ';
+     // - we also check the other attributes to see if the XML has requested
+     //   any following codeblock be linenumbered
+     if (strpos($other_attributes, 'withLineNumber') !== false)
+     {
+       $is_code_linenumbered = true;
+       $txt .= '%!-- withLineNumber --%';
+     }
+     $txt .= '></imgcaption>' . "\n";
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, $txt);
+     }
+     else
+     {
+       $frontmatter_text .= $txt;
+     }
+   }
+   elseif (strpos($line, '</Figure>') !== false)
+   {
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, "\n\n");
+     }
+     else
+     {
+       $frontmatter_text .= "\n\n";
+     }
+     // - no longer required
+     $is_code_linenumbered = false;
+   }
+   elseif (preg_match('/<Table([^>]*).*?.*?>/', $line, $matches))
+   {
+     $attributes = $matches[1];
+     $table_txt = '';
+     $table_id = '';
+     if (preg_match('/id="([^"]+)"/', $attributes, $matches))
+     {
+       $table_id = $matches[1];
+     }
+     else
+     {
+       $table_id = generateID('table');
+     }
+     $hidden = false;
+     if (strpos($attributes, 'class="hidden"') !== false)
+     {
+       $hidden = true;
+     }
+     echo '[adding table: ' . $table_id . "] \n";
+     $table_caption = getTitle($xml_in, 'table: ' . $table_id);
+     if ($hidden)
+     {
+       $table_txt .= '<tblcaption table_' . $table_id . '|##HIDDEN##></tblcaption>' . "\n";
+     }
+     elseif (empty($table_caption))
+     {
+       $table_txt .= '<tblcaption table_' . $table_id . '|##NOCAPTION##></tblcaption>' . "\n";
+     }
+     else
+     {
+       $table_txt .= '<tblcaption table_' . $table_id . '|' . noComments($table_caption) . '></tblcaption>' . "\n";
+     }
+     // - in order to properly capture the table we're going to have to read in
+     //   the whole thing here, and take note of column widths
+     $have_output_widths = false;
+     $column_widths = array();
+     while (strpos($line, '</Table>') === false)
+     {
+       // - find the start of a row
+       while(!empty($line) && strpos($line, '<tr>') === false && strpos($line, '</Table>') === false)
+       {
+         $line = getLine($xml_in);
+       }
+       if (strpos($line, '<tr>') !== false)
+       {
+         $row_txt = '|';
+         $line = getLine($xml_in);
+         // - now we read in multiple cells
+         while (preg_match('/<th width="(\d+)"\/?>/', $line, $matches))
+         {
+           $cell_width = $matches[1];
+           if (!$have_output_widths)
+           {
+             array_push($column_widths, $cell_width);
+           }
+           // Ignore empty cells
+           // - adding another case for empty header cells (turned up in es
+           //   version of "From Paper")
+           if (preg_match('/<th width="\d+"\/>/', $line) || preg_match('/<th width="\d+">.*<\/th>/', $line))
+           {
+             $row_txt .= ' |';
+           }
+           else
+           {
+             $line = getLine($xml_in);
+             $first = true;
+             while (strpos($line, '</th>') === false)
+             {
+               if (!$first)
+               {
+                 $row_txt .= '\\\\';
+               }
+               // - we can have images or text in our tables
+               if (preg_match('/<File width="(\d+)" height="(\d+)" url="images\/([^"]+)"\/>/', $line, $matches))
+               {
+                 $image_txt = handleImage($matches[3], $matches[1], $matches[2]);
+                 $row_txt .= ' ' . $image_txt . ' ';
+               }
+               elseif (preg_match('/<Text id="([^"]+)">(.+?)<\/Text>/', $line, $matches))
+               {
+                 $row_txt .= ' <!-- id:' . $matches[1] . ' -->' . translateText($matches[2]) . ' ';
+               }
+               elseif (preg_match('/<CodeLine>(.*?)<\/CodeLine>/',$line,$matches))
+               {
+                 $row_txt .= ' \'\'' . translateTableCodeLine($matches[1]) . '\'\' ';
+               }
+               else
+               {
+                 printError('Warning! Unrecognized element in table: ' . $line);
+               }
+               $first = false;
+               // - next line
+               $line = getLine($xml_in);
+             }
+             // - close the cell
+             $row_txt .= '|';
+           }
+           // next!
+           $line = getLine($xml_in);
+         }
+         // - if we haven't already, output the width command
+         if (!$have_output_widths)
+         {
+           $table_txt .= '|< - ' . implode(' ', $column_widths) . ' >|' . "\n";
+           $have_output_widths = true;
+         }
+         $table_txt .= $row_txt . "\n";
+         // - throw away the closing </tr>
+         $line = getLine($xml_in);
+       }
+     }
+     $table_txt .= "\n";
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, $table_txt);
+     }
+     else
+     {
+       $frontmatter_text .= $table_txt;
+     }
+   }
+   // - copy and insert images
+   elseif (preg_match('/<File width="(\d+)" height="(\d+)" url="images\/([^"]+)"\/>/', $line, $matches))
+   {
+     $image_txt = handleImage($matches[3], $matches[1], $matches[2]);
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, $image_txt);
+     }
+     else
+     {
+       $frontmatter_text .= $image_txt;
+     }
+   }
+   // - bullet lists
+   elseif (preg_match('/<BulletList>/', $line))
+   {
+     echo "[adding bulletlist] \n";
+     if ($in_bullet_item || $in_numbered_item)
+     {
+       if ($in_chapter)
+       {
+         fwrite($chapter_txt_out, "\n");
+       }
+       elseif ($in_section)
+       {
+         $frontmatter_text .= "\n";
+       }
+     }
+     $bullet_depth++;
+     $is_numbered_list = false;
+   }
+   // - numbered lists
+   elseif (preg_match('/<NumberedList>/', $line))
+   {
+     echo "[adding numbered list] \n";
+     $bullet_depth++;
+     $is_numbered_list = true;
+     // - reset this flag that keeps track of whether an item (numbered or
+     //   otherwise) is legitimately split by a code block
+     $seen_code_in_item = false;
+   }
+   elseif (preg_match('/<\/BulletList>/', $line))
+   {
+     $bullet_depth--;
+     if ($bullet_depth == 0)
+     {
+       if ($in_chapter)
+       {
+         fwrite($chapter_txt_out, "\n");
+       }
+       elseif ($in_section)
+       {
+         $frontmatter_text .= "\n";
+       }
+       $is_numbered_list = false;
+     }
+     ///cho "[finished bulletlist] ";
+   }
+   elseif (preg_match('/<\/NumberedList>/', $line))
+   {
+     $bullet_depth--;
+     if ($bullet_depth == 0)
+     {
+       if ($in_chapter)
+       {
+         fwrite($chapter_txt_out, "\n");
+       }
+       elseif ($in_section)
+       {
+         $frontmatter_text .= "\n";
+       }
+       $is_numbered_list = false;
+     }
+     ///cho "[finished numbered list] ";
+   }
+   elseif (preg_match('/<NumberedItem>/', $line))
+   {
+     $in_numbered_item = 1;
+   }
+   elseif (preg_match('/<\/NumberedItem>/', $line))
+   {
+     $in_numbered_item = 0;
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, "\n");
+     }
+     else
+     {
+       $frontmatter_text .= "\n";
+     }
+   }
+   elseif (preg_match('/<Bullet>/', $line))
+   {
+     $in_bullet_item = true;
+   }
+   elseif (preg_match('/<\/Bullet>/', $line))
+   {
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, "\n");
+     }
+     else
+     {
+       $frontmatter_text .= "\n";
+     }
+     $in_bullet_item = false;
+   }
+   // TEXT HANDLING - this is the main case, but has disappeared into the mire
+   // of other cases.
+   elseif (!$in_footnotes && preg_match('/<Text id="([^"]+)">(.+?)$/', $line, $matches))
+   {
+     $id = $matches[1];
+     $str = $matches[2];
+     // - special case for those text elements split over multiple lines. We
+     //   keep concatenating lines until we find the closing text element or we
+     //   run out of lines!
+     $another_line = '';
+     while (strpos($str, '</Text>') === false && ($another_line = getLine($xml_in)) !== false)
+     {
+       $str .= ' ' . $another_line;
+     }
+     // - note that if we ran out of lines (eof) then we'll break out of this
+     //   block anyway, it's just there won't be a <\Text> at the end of this
+     //   block... despite this being a major validation issue in the XML it
+     //   shouldn't result in this script being vladed
+     // - now remove the </Text> from the end (hopefully) of str
+     $str = preg_replace('/<\/Text>\s*/', '', $str);
+     // - and prepend the id while translating the str into Dokuwiki format
+     $str = '<!-- id:' . $id . ' -->' . translateText($str);
+     if ($bullet_depth > 0)
+     {
+       if ($is_numbered_list)
+       {
+         // - special case for those text elements legimately split in two by
+         //   code blocks. They get no bullet of either type and are 'run-on'
+         //   immediately to the end of the code element in order to prevent
+         //   dokuwiki restarting numbering etc
+         if ($seen_code_in_item)
+         {
+           // - leave str as it is
+           // - reset flag just incase the item happens to contain another
+           //   code block
+           $seen_code_in_item = false;
+         }
+         else if ($in_numbered_item == 1)
+         {
+           $str = '- ' . $str;
+         }
+         // - superspecial case for the poorly formatted numberlists that
+         //   contain more than one text block per point. We'll nest them
+         //   as a bullet list as that preserves order, formatting and (I
+         //   hope) meaning.
+         else
+         {
+           if ($in_chapter)
+           {
+             fwrite($chapter_txt_out, "\n");
+           }
+           else
+           {
+             $frontmatter_text .= "\n";
+           }
+           $str = '  * ' . $str;
+         }
+         $in_numbered_item++;
+       }
+       else
+       {
+         $str = '* ' . $str;
+       }
+       for ($i = 0; $i < $bullet_depth; $i++)
+       {
+         $str = '  ' . $str;
+       }
+     }
+     else
+     {
+       // Indented text is preceeded by a >
+       if ($in_indent)
+       {
+         $str = '> ' . $str . "\n";
+       }
+       else
+       {
+         $str .= "\n";
+       }
+     }
+     if ($bullet_depth == 0)
+     {
+       $str .= "\n";
+     }
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, $str);
+     }
+     else
+     {
+       $frontmatter_text .= $str;
+     }
+   }
+   // - codified text blocks
+   elseif (preg_match('/<Text\s+type="code"\s+id="([^"]+)"\s*>(.+?)<\/Text>/', $line, $matches))
+   {
+     $code_id = $matches[1];
+     // - determine the appropriate code block prefix
+     $code_prefix = '';
+     if (!$in_code)
+     {
+       if ($is_code_linenumbered)
+       {
+         $code_prefix = '<code 1>';
+       }
+       else
+       {
+         $code_prefix = '<code>';
+       }
+       $in_code = true;
+     }
+     $code_txt = $code_prefix . '<!-- id:' . $matches[1] . ' -->' . translateText($matches[2], true);
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, $code_txt);
+     }
+     else
+     {
+       $frontmatter_text .= $code_txt;
+     }
+   }
+   elseif (preg_match('/<Text\s+id="([^"]+)"\s+type="code"\s*>(.+?)<\/Text>/', $line, $matches))
+   {
+     $code_id = $matches[1];
+     // - determine the appropriate code block prefix
+     $code_prefix = '';
+     if (!$in_code)
+     {
+       if ($is_code_linenumbered)
+       {
+         $code_prefix = '<code 1>';
+       }
+       else
+       {
+         $code_prefix = '<code>';
+       }
+       $in_code = true;
+     }
+     $code_txt = $code_prefix . '<!-- id:' . $matches[1] . ' -->' . translateText($matches[2], true);
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, $code_txt);
+     }
+     else
+     {
+       $frontmatter_text .= $code_txt;
+     }
+   }
+   elseif (preg_match('/<CodeLine>(.*?)$/', $line, $matches))
+   {
+     $code_txt = $matches[1];
+     // - determine the appropriate code block prefix
+     $code_prefix = '';
+     if (!$in_code)
+     {
+       if ($is_code_linenumbered)
+       {
+         $code_prefix = "<code 1>\n";
+       }
+       else
+       {
+         $code_prefix = "<code>\n";
+       }
+       $in_code = true;
+     }
+     // - arg. another special case for codelines that span more than one line
+     // (but I guess <CodeLineButSometimesMoreThanOneLine> is a bit cumbersome
+     // for an element name, eh?)
+     $another_line = '';
+     while (strpos($code_txt, '</CodeLine>') === false && ($another_line = getLine($xml_in)) !== false)
+     {
+       $code_txt .= ' ' . $another_line;
+     }
+     $code_txt = preg_replace('/<\/CodeLine>\s*/', '', $code_txt);
+     $code_txt = $code_prefix . translateText($code_txt, true);
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, $code_txt);
+     }
+     else
+     {
+       $frontmatter_text .= $code_txt;
+     }
+   }
+   // - there are also sometimes empty codelines - which indicate a newline in
+   //   the code listing
+   elseif (preg_match('/<CodeLine\s*\/>/', $line, $matches))
+   {
+     $code_txt = '';
+     if (!$in_code)
+     {
+       $code_txt = "\n";
+       if ($is_code_linenumbered)
+       {
+         $code_txt = "<code 1>\n" . $code_txt;
+       }
+       else
+       {
+         $code_txt = "<code>\n" . $code_txt;
+       }
+       $in_code = true;
+     }
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, $code_txt);
+     }
+     else
+     {
+       $frontmatter_text .= $code_txt;
+     }
+   }
+   // - reference to an external XML file
+   elseif (preg_match('/^\s*&[a-z0-9_]+;\s+$/is', $line))
+   {
+     if ($in_chapter)
+     {
+       fwrite($chapter_txt_out, $line);
+     }
+     else
+     {
+       $frontmatter_text .= $line;
+     }
+   }
+   elseif (strpos($line, '<FootnoteList>') !== false)
+   {
+     $in_footnotes = true;
+   }
+   elseif ($in_footnotes && strpos($line, '</FootnoteList>') !== false)
+   {
+     $in_footnotes = false;
+   }
+   // Indentation - the closest thing we have is quoting, so we'll use that
+   elseif (strpos($line, '<Indented>') !== false)
+   {
+     $in_indent = true;
+   }
+   elseif (strpos($line, '</Indented>') !== false)
+   {
+     $in_indent = false;
+   }
+   // - pattern of lines to ignore
+   elseif (preg_match('/^(<\?xml version="1.0" encoding="UTF-8"\?>|<\!DOCTYPE Manual \[|\]>|<Bullet>|<\/?Content>|<\/?Footnote|<Manual id=".+?" lang=".+?">|<\/Manual>)/', $line))
+   {
+   }
+   // - we ignore anything else in footnotes too, as they were handled in the
+   //   preprocessing pass
+   elseif ($in_footnotes)
+   {
+   }
+   // - danger Will Robinson!
+   else
+   {
+     echo '<div style="background-color:yellow;"><hr /><b>Warning!</b> Failed to parse line ' . $line_counter . ': |' . htmlspecialchars($line) . "|<hr /></div>\n";
+   }
+ }
+
+// 2. We should now have enough metadata to export the cover page
+$top_page_path = $dokuwiki_path . '/data/pages/' . $_REQUEST['l'] . '/manuals/' . $_REQUEST['m'] . '.txt';
+// - backup any existing file
+if (file_exists($top_page_path))
+ {
+   $top_page_backup_path = $top_page_path . '.bak';
+   if(!rename($top_page_path, $top_page_backup_path))
+   {
+     printError('Failed to rename existing top page for backup');
+   }
+ }
+// - and create a handle to the new file
+$txt_out = fopen($top_page_path, 'w');
+// - write the page (including the tables)
+fwrite($txt_out, '====== ' . noComments(ucfirst(getFirstMetadata('Heading'))) . ': ' . noComments(ucfirst(getFirstMetadata('Title'))) . ' (' . strtoupper($_REQUEST['l']) . ') ======' . "\n");
+fwrite($txt_out, "\n");
+
+// - *NEW* ability to request imports and exports from within the page
+fwrite($txt_out, "<ifauth @admin>\n\n");
+fwrite($txt_out, '**Administrator Commands:**' . "\n");
+// On second thoughts we probably never want to do this casually, as it boguses
+// all history/approval/edit information. Instead I'll leave this as a manual
+// process.
+fwrite($txt_out, '<!-- Import available at this link - but be warned all current wiki data for this manual will become bogus: http://~~baseurl~~/../../php/gs-manual-import.php?m=' . $_REQUEST['m'] . '&l=' . $_REQUEST['l'] . " -->\n");
+fwrite($txt_out, '  * Export manual: [[http://~~baseurl~~/../../php/gs-manual-export.php?m=' . $_REQUEST['m'] . '&l=' . $_REQUEST['l'] . '&v=draft&a=download|draft version]] [[http://~~baseurl~~/../../php/gs-manual-export.php?m=' . $_REQUEST['m'] . '&l=' . $_REQUEST['l'] . '&a=download|approved version]]' . "\n");
+fwrite($txt_out, "</ifauth>\n\n");
+
+// - regular metadata
+fwrite($txt_out, '<!-- Note: cover page information -->' . "\n");
+fwrite($txt_out, '^ Metadata  ^ Value  ^' . "\n");
+$fields = array('Heading','Title','Author','Affiliation','Text','Comment','Version','Date');
+foreach ($fields as $field)
+{
+  $values = getMetadata($field);
+  foreach ($values as $value)
+  {
+    fwrite($txt_out, '^ ' . $field . '  | ' . $value . '  |' . "\n");
+  }
+}
+fwrite($txt_out, "\n");
+// - contents (which also provides order information for exporting)
+fwrite($txt_out, '===== Contents =====' . "\n");
+fwrite($txt_out, "\n");
+fwrite($txt_out, '<!-- Note: The ordering of pages here is used when creating the HTML and PDF versions of the manual -->' . "\n");
+foreach ($page_order as $page_info)
+{
+  fwrite($txt_out, '  * [[.:' . $_REQUEST['m'] . ':' . $page_info . ']]' . "\n");
+}
+// - system metadata
+fwrite($txt_out, "<ifauth @admin>\n\n");
+fwrite($txt_out, '===== System Metadata =====' . "\n");
+fwrite($txt_out, '<!-- Note: configuration options for the manual -->' . "\n");
+fwrite($txt_out, '^ Metadata  ^ Value  ^' . "\n");
+$fields = array('ENTITY','SupplementaryText');
+foreach ($fields as $field)
+{
+  $values = getMetadata($field);
+  foreach ($values as $value)
+  {
+    fwrite($txt_out, '^ ' . $field . '  | ' . $value . '  |' . "\n");
+  }
+}
+fwrite($txt_out, "</ifauth>\n\n");
+fwrite($txt_out, "\n");
+// - done!
+fclose($txt_out);
+$page_count++;
+
+// 3. And the 'sections' page, grouping together all the loose sections as
+//    frontmatter
+$frontmatter_page_path = $dokuwiki_path . '/data/pages/' . $_REQUEST['l'] . '/manuals/' . $_REQUEST['m'] . '/' . $sections_page_name . '.txt';
+// - backup any existing file
+if (file_exists($frontmatter_page_path))
+ {
+   $frontmatter_page_backup_path = $frontmatter_page_path . '.bak';
+   if(!rename($frontmatter_page_path, $frontmatter_page_backup_path))
+   {
+     printError('Failed to rename existing frontmatter page for backup');
+   }
+ }
+// - populate the new frontmatter file
+file_put_contents($frontmatter_page_path, $frontmatter_text);
+
+echo "</p>\n<p><b>Complete!</b> Imported " . $page_count . " pages</p><hr/>\n";
+echo '<p>Click <a href="' . $dokuwiki_url . '/doku.php?id=' . $_REQUEST['l'] . ':manuals:' . $_REQUEST['m'] . '">here</a> to return to wiki page</p>' . "\n";
+exit(0);
+
+/**
+ */
+function addMetadata($field, $value)
+{
+  global $manual_metadata;
+  echo '[adding metadata: ' . $field . "] \n";
+  $values = array();
+  if (isset($manual_metadata[$field]))
+  {
+    $values = $manual_metadata[$field];
+  }
+  array_push($values, $value);
+  $manual_metadata[$field] = $values;
+}
+/** addMetadata() **/
+
+function getFirstMetadata($field)
+{
+  global $manual_metadata;
+  $value = '';
+  if (isset($manual_metadata[$field]))
+  {
+    $values = $manual_metadata[$field];
+    if (!empty($values))
+    {
+      $value = $values[0];
+    }
+  }
+  return $value;
+}
+/** getFirstMetadata() **/
+
+function getMetadata($field)
+{
+  global $manual_metadata;
+  $values = array();
+  if (isset($manual_metadata[$field]))
+  {
+    $values = $manual_metadata[$field];
+  }
+  return $values;
+}
+/** getMetadata() **/
+
+/**
+ *  Read in the next title element with nested text element and extract the
+ *  title.
+ */
+function getTitle($xml_in, $element)
+{
+  $title = '';
+  // - the first thing in a chapter will be it's title
+  $title_line = getLine($xml_in);
+  // - super special case: a table with an empty title
+  if (strpos($title_line, '<Title/>') !== false)
+  {
+    return '';
+  }
+  if (strpos($title_line, '<Title>') === false)
+  {
+    printError('Failed to find opening title for: ' . $element);
+  }
+  $title_line = getLine($xml_in);
+  // - grab the chapter title now so we can store it in the page ordering
+  if (preg_match('/<Text id="([^"]+)">(.*?)$/', $title_line, $matches))
+  {
+    $id = $matches[1];
+    $str = $matches[2];
+    // - special case for text blocks that span multiple lines (as discovered
+    //   in the russian "From Paper"
+    $another_line = '';
+    while (strpos($str, '</Text>') === false && ($another_line = getLine($xml_in)) !== false)
+    {
+      $str .= ' ' . $another_line;
+    }
+    // - now remove </Text>
+    $str = preg_replace('/<\/Text>\s*/', '', $str);
+    $title = '<!-- id:' . $id . ' -->' . $str;
+  }
+  // - special case for (stoopid) empty titles that use up a text id
+  elseif (preg_match('/<Text id="([^"]+)"\s*\/>/', $title_line, $matches))
+  {
+    $title = '<!-- id:' . $matches[1] . ' -->';
+  }
+  else
+  {
+    printError('Failed to find title text for: ' . $element);
+  }
+  // - watch for subtitle elements
+  $title_line = getLine($xml_in);
+  if (strpos($title_line, '<SubTitle>') !== false)
+  {
+    $title_line = getLine($xml_in);
+    if (preg_match('/<Text id="([^"]+)">(.+?)<\/Text>/', $title_line, $matches))
+    {
+      $title = '<!-- id:' . $matches[1] . ' -->' . $matches[2] . ' ' . $title;
+    }
+    $title_line = getLine($xml_in);
+    if (strpos($title_line, '</SubTitle>') === false)
+    {
+      printError('Failed to find closing title for: ' . $element);
+    }
+    $title_line = getLine($xml_in);
+  }
+  if (strpos($title_line, '</Title>') === false)
+  {
+    printError('Failed to find closing title for: ' . $element);
+  }
+  return $title;
+}
+/** getTitle() **/
+
+function alternateComments($text)
+{
+  $text = str_replace('<!--', '%!--', $text);
+  $text = str_replace('-->', '--%', $text);
+  return $text;
+}
+
+function noComments($text)
+{
+  $text = preg_replace('/<i>(.*?)<\/i>/','//\1//',$text);
+  return preg_replace('/<!--[^>]+-->/', '', $text);
+}
+
+function translateTableCodeline($text)
+{
+  ///cho "<b>Debug:</b> translateTableCodeLine('" . htmlspecialchars($text) . "')<br />\n";
+  // Escape the current italics tags to prevent the translate destroying them
+  $text = str_replace('<i>', '%!--i--%', $text);
+  $text = str_replace('</i>', '%/i%', $text);
+  // Translate the text, just decoding the entities
+  $text = translateText($text, true);
+  // Now turn the italic tags (escaped) into HTML comments so we remember them
+  // but they are hidden in the text
+  $text = str_replace('%!--i--%', '<!--i-->', $text);
+  $text = str_replace('%/i%', '<!--/i-->', $text);
+  ///cho " =&gt; '" . htmlspecialchars($text) . "<br />\n";
+  return $text;
+}
+
+function translateText($text, $entities_only=false)
+{
+  global $entity_replacements;
+  global $footnotes;
+  global $in_code;
+
+  // - immediate find and protect any legitimate HTML comments in the text
+  //   (so already using encoded entities), otherwise they'll be throughly
+  //   vladed during the following tranforms. This has to be matched with
+  //   changes to the HTMLComments plugin in Dokuwiki to allow the correct
+  //   thing to be displayed to the user.
+  if (!$in_code)
+  {
+    $text = str_replace('&lt;!--', '%!--', $text);
+    $text = str_replace('--&gt;', '--%', $text);
+  }
+
+  if (!$entities_only)
+  {
+    // - replace linking constructs with dokuwiki ones
+    // - external chapter section crossrefs are easily the worst of all...
+    while (preg_match('/<CrossRef\s[^>]*external[^>]*\/>/', $text) && preg_match('/<CrossRef\s[^>]*target="Chapter"[^>]*\/>/', $text) && preg_match('/<CrossRef\s+(.*?)\/>/', $text, $matches))
+    {
+      $pattern = $matches[0];
+      $attributes = $matches[1];
+      $manual_name = '';
+      if (preg_match('/external="([^"]+)"/', $attributes, $matches))
+      {
+        $manual_name = $matches[1];
+      }
+      $language = '';
+      if (preg_match('/lang="([^"]+)"/', $attributes, $matches))
+      {
+        $language = $matches[1];
+      }
+      $page_id = '';
+      if (preg_match('/ref="([^"]+)"/', $attributes, $matches))
+      {
+        $page_id = $matches[1];
+      }
+      if (empty($manual_name) || empty($language) || empty($page_id))
+      {
+        printError('Failed to parse external reference: ' . $pattern);
+      }
+      // - best we can do is a search within a restricted namespace
+      $reference = '[[?do=search&id=' . $page_id . ' @' . $language . ':manuals:' . $manual_name . '|' . $page_id . ']]';
+      $text = str_replace($pattern, $reference, $text);
+    }
+    // - chapter crossrefs are tricksie due to needing to know ordering numbers
+    while (preg_match('/<CrossRef target="Chapter" ref="([^"]+)"\/>/', $text, $matches))
+    {
+      $chapter_id = $matches[1];
+      $page_name = $chapter_id;
+      $text = preg_replace('/<CrossRef target="Chapter" ref="' . $chapter_id . '"\/>/', '[[.:' . $page_name . '|' . $chapter_id . ']]', $text);
+    }
+    // - internal figure and table references
+    $text = preg_replace('/<CrossRef target="Figure" ref="([^"]+)"\/>/','<imgref figure_\1>', $text);
+    $text = preg_replace('/<CrossRef target="Table" ref="([^"]+)"\/>/','<tblref table_\1>', $text);
+    // - simple internal reference
+    $text = preg_replace('/<CrossRef target="Section" ref="([^"]+)"\/>/', '[[#\1|\1]]', $text);
+    $text = preg_replace('/<CrossRef target="Subsection" ref="([^"]+)"\/>/', '[[##\1|\1]]', $text);
+    $text = preg_replace('/<CrossRef target="Part" ref="([^"]+)"\/>/', '[[###\1|\1]]', $text);
+    // - simple external url
+    $text = preg_replace('/<Link url="([^"]+)">(.+?)<\/Link>/', '[[\1|\2]]', $text);
+    // - footnote references are also tricksie as we've had to extract the
+    //   footnotes earlier (during chapter counting)
+    while (preg_match('/<FootnoteRef id="(\d+)"\/>/', $text, $matches))
+    {
+      $footnote_id = $matches[1];
+      if (!isset($footnotes[$footnote_id]))
+      {
+        printError('Unknown footnote referenced: ' . $footnote_id);
+      }
+      $footnote = $footnotes[$footnote_id];
+      $text = preg_replace('/<FootnoteRef id="' . $footnote_id . '"\/>/', '((' . $footnote . '))', $text);
+    }
+    // - detect and handle URLs surrounded by <i> tags very carefully (as doku
+    //   will less than helpfully turn them into an external link and screw up
+    //   everything that follows them on the page).
+    //   example: <i>www.microsoft.com</i>
+    //   example: <i>http://nzdl.org/cgi-bin/library</i>
+    //   example: <i>www.yourserver.com</i>
+    //   example: <i>http://www.yourserver.com</i>
+    //   example: <i>http://www.yourserver.com/greenstone</i>
+    $text = preg_replace('/<i>((?:http\:\/\/)?[a-z]+\.[a-z0-9\-]+\.[a-z0-9\.\-]+(?:\/.*?)?)<\/i>/i','// \1 //', $text);
+    // - superspecial case for two-part URLs ending in .org (like nzdl.org)
+    $text = preg_replace('/<i>((?:http\:\/\/)?[a-z0-9\-]+\.org(?:\/.*?)?)<\/i>/i','// \1 //', $text);
+    // - another superspecial case, this time for URLs on localhost
+    $text = preg_replace('/<i>((?:http\:\/\/)?localhost(?:\/.*?)?)<\/i>/i','// \1 //', $text);
+    // - p00p, underlines have the same issue around URLs.
+    $text = preg_replace('/<u>((?:http\:\/\/)?[a-z]+\.[a-z0-9\-]+\.[a-z0-9\.\-]+(?:\/.*?)?)<\/u>/i','__ \1 __', $text);
+    $text = preg_replace('/<u>((?:http\:\/\/)?[a-z0-9\-]+\.org(?:\/.*?)?)<\/u>/i','__ \1 __', $text);
+    $text = preg_replace('/<u>((?:http\:\/\/)?localhost(?:\/.*?)?)<\/u>/i','__ \1 __', $text);
+
+    // - replace HTML elements with the dokuwiki style equivilents
+    $text = preg_replace('/(file|ftp|http):\/\//','\1:%%//%%', $text);
+    // - restore the double slashes in dokuwiki links
+    while (preg_match('/\[\[[^\]]*%%[^\]]*\]\]/', $text))
+    {
+      $text = preg_replace('/(\[\[[^\]]*)%%([^\]]*\]\])/', '\1\2', $text);
+    }
+    $text = str_replace('<b>', '**', $text);
+    $text = str_replace('</b>', '**', $text);
+    //$text = str_replace(' <br/>', '\\\\ ', $text);
+    //$text = str_replace('Â <br/>', '\\\\ ', $text);
+    //$text = str_replace('<br/>', '\\\\ ', $text);
+    $text = str_replace('<i>', '//', $text);
+    $text = str_replace('</i>', '//', $text);
+    $text = str_replace('<u>', '__', $text);
+    $text = str_replace('</u>', '__', $text);
+  }
+  // Decode entities
+  // - user defined entities (in the manual metadata)
+  foreach ($entity_replacements as $entity=>$code)
+  {
+    $text = str_replace('&' . $entity . ';', html_entity_decode('&#'.$code.';',ENT_NOQUOTES,'UTF-8'), $text);
+  }
+  // - standard entities
+  $text = str_replace('&gt;','>', $text);
+  $text = str_replace('&lt;','<', $text);
+  $text = str_replace('&amp;','&', $text);
+  return $text;
+}
+/** translateText() **/
+
+function handleImage($filename, $width, $height)
+{
+  global $dokuwiki_path;
+  global $xml_source_path;
+  echo '[copying image: ' . $filename . "] \n";
+  // - copy file into place
+  $source_path = $xml_source_path . '/' . $_REQUEST['l'] . '/images/' . $filename;
+  $destination_path = $dokuwiki_path . '/data/media/' . $_REQUEST['l'] . '/manuals/images/' . strtolower($filename);
+  copy($source_path, $destination_path);
+  if (!file_exists($destination_path))
+  {
+    printError('Failed to copy image file: ' . $filename);
+  }
+  // - create the string
+  $image_txt = '{{..:images:' . strtolower($filename) . '?' . $width . 'x' . $height . '&direct}}';
+  return $image_txt;
+}
+
+function getLine($in)
+{
+  global $line_counter;
+  $line_counter++;
+  return fgets($in);
+}
+
+function noFormatting($text)
+{
+  $text = str_replace( '<b>', '', $text);
+  $text = str_replace('</b>', '', $text);
+  $text = str_replace( '<i>', '', $text);
+  $text = str_replace('</i>', '', $text);
+  $text = str_replace('<br/>', '', $text);
+  return $text;
+}
+
+?>
