Changeset 34921 for main/trunk/greenstone2/perllib/cpan/Image/ExifTool/BuildTagLookup.pm

Timestamp:

2021-02-26T19:39:51+13:00 (3 years ago)

Author:

anupama

Message:

Committing the improvements to EmbeddedMetaPlugin's processing of Keywords vs other metadata fields. Keywords were literally stored as arrays of words rather than phrases in PDFs (at least in Diego's sample PDF), whereas other meta fields like Subjects and Creators stored them as arrays of phrases. To get both to work, Kathy updated EXIF to a newer version, to retrieve the actual EXIF values stored in the PDF. And Kathy and Dr Bainbridge came up with a new option that I added called apply_join_before_split_to_metafields that's a regex which can list the metadata fields to apply the join_before_split to and whcih previously always got applied to all metadata fields. Now it's applied to any *Keywords metafields by default, as that's the metafield we have experience of that behaves differently to the others, as it stores by word instead of phrases. Tested on Diego's sample PDF. Diego has double-checked it to works on his sample PDF too, setting the split char to ; and turning on the join_before_split and leaving apply_join_before_split_to_metafields at its default of .*Keywords. File changes are strings.properties for the tooltip, the plugin introducing the option and working with it and Kathy's EXIF updates affecting cpan/File and cpan/Image.

File:

• main/trunk/greenstone2/perllib/cpan/Image/ExifTool/BuildTagLookup.pm (modified) (115 diffs)

main/trunk/greenstone2/perllib/cpan/Image/ExifTool/BuildTagLookup.pm

@@ -32 +32 @@
 use Image::ExifTool::Canon;
 use Image::ExifTool::Nikon;
-
-$VERSION = '2.28';
+use Image::ExifTool::Sony;
+use Image::ExifTool::Validate;
+use Image::ExifTool::MacOS;
+
+$VERSION = '3.42';
 @ISA = qw(Exporter);
 
-sub NumbersFirst;
-
-my $numbersFirst = 1;   # set to -1 to sort numbers last
+sub NumbersFirst($$);
+sub SortedTagTablekeys($);
+
+my $createDate = 'Feb 15, 2005';
+
+# global variables to control sorting order of table entries
+my $numbersFirst = 1;   # set to -1 to sort numbers last, or 2 to put negative numbers last
+my $caseInsensitive;    # flag to ignore case when sorting tag names
 
 # list of all tables in plug-in modules
@@ -51 +59 @@
 };
 
-my $homePage = 'http://owl.phy.queensu.ca/~phil/exiftool';
+my $homePage = 'https://exiftool.org';
+
+# tweak the ordering of tables in the documentation
+my %tweakOrder = (
+  # this    => comes after this
+  # -------    -----------------
+    JPEG    => '-',     # JPEG comes first
+    IPTC    => 'Exif',  # put IPTC after EXIF,
+    GPS     => 'XMP',   # etc...
+    Composite => 'Extra',
+    GeoTiff => 'GPS',
+    CanonVRD=> 'CanonCustom',
+    DJI     => 'Casio',
+    FLIR    => 'DJI',
+    FujiFilm => 'FLIR',
+    Kodak   => 'JVC',
+    Leaf    => 'Kodak',
+    Minolta => 'Leaf',
+    Motorola => 'Minolta',
+    Nikon   => 'Motorola',
+    NikonCustom => 'Nikon',
+    NikonCapture => 'NikonCustom',
+    Nintendo => 'NikonCapture',
+    Pentax  => 'Panasonic',
+    SonyIDC => 'Sony',
+    Unknown => 'SonyIDC',
+    DNG     => 'Unknown',
+    PrintIM => 'ICC_Profile',
+    Vorbis  => 'Ogg',
+    ID3     => 'PostScript',
+    MinoltaRaw => 'KyoceraRaw',
+    KyoceraRaw => 'CanonRaw',
+    SigmaRaw => 'PanasonicRaw',
+    Lytro   => 'SigmaRaw',
+    PhotoMechanic => 'FotoStation',
+    Microsoft     => 'PhotoMechanic',
+   'Microsoft::MP'=> 'Microsoft::MP1',
+    GIMP    => 'Microsoft',
+   'Nikon::CameraSettingsD300' => 'Nikon::ShotInfoD300b',
+   'Pentax::LensData' => 'Pentax::LensInfo2',
+   'Sony::SRF2' => 'Sony::SRF',
+    DarwinCore => 'AFCP',
+   'MWG::Regions' => 'MWG::Composite',
+   'MWG::Keywords' => 'MWG::Regions',
+   'MWG::Collections' => 'MWG::Keywords',
+   'GoPro::fdsc' => 'GoPro::KBAT',
+);
 
 # list of all recognized Format strings
 # (not a complete list, but this is all we use so far)
-# (also, formats like "var_X[Y]" are allowed for any valid X)
+# (also, formats like "var_X[num]" are allowed for any valid X)
 my %formatOK = (
     %Image::ExifTool::Exif::formatNumber,
     0 => 1,
     1 => 1,
+    2 => 1, # (writable for docs only)
     real    => 1,
     integer => 1,
@@ -74 +129 @@
     digits   => 1,
     int16uRev => 1,
+    int32uRev => 1,
     rational32u => 1,
     rational32s => 1,
+    pstring     => 1,
     var_string  => 1,
     var_int16u  => 1,
     var_pstr32  => 1,
+    var_ustr32  => 1,
+    var_ue7     => 1, # (BPG)
     # Matroska
     signed      => 1,
@@ -84 +143 @@
     utf8        => 1,
 );
-
-my $caseInsensitive;    # flag to ignore case when sorting tag names
 
 # Descriptions for the TagNames documentation
@@ -107 +164 @@
     ExifTool => q{
 The tables listed below give the names of all tags recognized by ExifTool.
+They contain a total of $$self{COUNT}{'total tags'} tags, with $$self{COUNT}{'unique tag names'} unique tag names.
 },
     ExifTool2 => q{
-B<Tag ID>, B<Index> or B<Sequence> is given in the first column of each
+B<Tag ID>, B<Index#> or B<Sequence> is given in the first column of each
 table.  A B<Tag ID> is the computer-readable equivalent of a tag name, and
-is the identifier that is actually stored in the file.  An B<Index> refers
-to the location of a value when found at a fixed position within a data
-block, and B<Sequence> gives the order of values for a serial data stream.
+is the identifier that is actually stored in the file.  B<Index#> refers to
+the offset of a value when found at a fixed position within a data block
+(B<#> is the multiplier for calculating a byte offset: B<1>, B<2>, B<4> or
+B<8>).  These offsets may have a decimal part which is used only to
+differentiate tags with values stored at the same position.  (Note that
+writable tags within binary data blocks are not individually deletable,
+and the usual alternative is to set them to a value of zero.)  B<Sequence>
+gives the order of values for a serial data stream.
 
 A B<Tag Name> is the handle by which the information is accessed in
@@ -121 +184 @@
 question mark (C<?>) after a tag name indicates that the information is
 either not understood, not verified, or not very useful -- these tags are
-not extracted by ExifTool unless the Unknown (-u) option is enabled.  Be
+not extracted by ExifTool unless the L<Unknown|../ExifTool.html#Unknown> (-u) option is enabled.  Be
 aware that some tag names are different than the descriptions printed out by
 default when extracting information with exiftool.  To see the tag names
@@ -127 +190 @@
 
 The B<Writable> column indicates whether the tag is writable by ExifTool.
-Anything but an C<N> in this column means the tag is writable.  A C<Y>
+Anything but a C<no> in this column means the tag is writable.  A C<yes>
 indicates writable information that is either unformatted or written using
-the existing format.  Other expressions give details about the information
-format, and vary depending on the general type of information.  The format
-name may be followed by a number in square brackets to indicate the number
-of values written, or the number of characters in a fixed-length string
-(including a null terminator which is added if required).
+the existing format.  Other expressions give details about the format of the
+stored value, and vary depending on the general type of information.  The
+format name may be followed by a number in square brackets to indicate the
+number of values written, or the number of characters in a fixed-length
+string (including a null terminator which is added if required).
 
 A plus sign (C<+>) after an entry in the B<Writable> column indicates a
-"list" tag which supports multiple values and allows individual values to be
-added and deleted.  A slash (C</>) indicates an "avoided" tag that is not
-created when writing if another same-named tag may be created instead.  To
-write these tags, the group should be specified.  A tilde (C<~>) indicates a
-tag this is writable only when the print conversion is disabled (by setting
-PrintConv to 0, using the -n option, or suffixing the tag name with a C<#>
-character).  An exclamation point (C<!>) indicates a tag that is considered
-unsafe to write under normal circumstances.  These "unsafe" tags are not set
-when calling SetNewValuesFromFile() or copied with the exiftool
--tagsFromFile option unless specified explicitly, and care should be taken
-when editing them manually since they may affect the way an image is
-rendered.  An asterisk (C<*>) indicates a "protected" tag which is not
+I<List> tag which supports multiple values and allows individual values to
+be added and deleted.  A slash (C</>) indicates a tag that ExifTool will
+I<Avoid> when writing.  These will be edited but not created if another
+same-named tag may be created instead.  To create these tags, the group
+should be specified.  A tilde (C<~>) indicates a tag this is writable only
+when the print conversion is disabled (by setting L<PrintConv|../ExifTool.html#PrintConv> to 0, using the
+-n option, or suffixing the tag name with a C<#> character). An exclamation
+point (C<!>) indicates a tag that is considered I<Unsafe> to write under
+normal circumstances.  These tags are not written unless specified
+explicitly (ie. not when wildcards or "all" are used), and care should be
+taken when editing them manually since they may affect the way an image is
+rendered.  An asterisk (C<*>) indicates a I<Protected> tag which is not
 writable directly, but is written automatically by ExifTool (often when a
-corresponding Composite or Extra tag is written).  A colon (C<:>) indicates
-a mandatory tag which may be added automatically when writing.
+corresponding L<Composite|Image::ExifTool::TagNames/Composite Tags> or
+L<Extra|Image::ExifTool::TagNames/Extra Tags> tag is written). A colon
+(C<:>) indicates a I<Mandatory> tag which may be added automatically when
+writing.  Normally MakerNotes tags may not be deleted individually, but a
+caret (C<^>) indicates a I<Deletable> MakerNotes tag.
 
 The HTML version of these tables also lists possible B<Values> for
 discrete-valued tags, as well as B<Notes> for some tags.  The B<Values> are
-listed as the computer-readable and human-readable values on the left and
-right hand side of an equals sign (C<=>) respectively.  The human-readable
+listed with the computer-readable values on the left of the equals sign
+(C<=>), and the human-readable values on the right.  The human-readable
 values are used by default when reading and writing, but the
 computer-readable values may be accessed by disabling the value conversion
-with the -n option on the command line, by setting the ValueConv option to 0
-in the API, or or on a per-tag basis by appending a number symbol (C<#>) to
-the tag name.
+with the -n option on the command line, by setting the L<PrintConv|../ExifTool.html#PrintConv> option to 0
+in the API, or or on a per-tag basis by adding a hash (C<#>) after the tag
+name.
 
 B<Note>: If you are familiar with common meta-information tag names, you may
@@ -168 +234 @@
 documentation or run C<exiftool -s> on a file containing the information in
 question.
+
+I<(This documentation is the result of years of research, testing and
+reverse engineering, and is the most complete metadata tag list available
+anywhere on the internet.  It is provided not only for ExifTool users, but
+more importantly as a public service to help augment the collective
+knowledge, and is often used as a primary source of information in the
+development of other metadata software.  Please help keep this documentation
+as accurate and complete as possible, and feed any new discoveries back to
+ExifTool.  A big thanks to everyone who has helped with this so far!)>
 },
     EXIF => q{
 EXIF stands for "Exchangeable Image File Format".  This type of information
 is formatted according to the TIFF specification, and may be found in JPG,
-TIFF, PNG, PGF, MIFF, HDP, PSP and XCF images, as well as many TIFF-based
-RAW images, and even some AVI and MOV videos.
+TIFF, PNG, JP2, PGF, MIFF, HDP, PSP and XCF images, as well as many
+TIFF-based RAW images, and even some AVI and MOV videos.
 
 The EXIF meta information is organized into different Image File Directories
@@ -180 +255 @@
 B<Group> listed below is used unless another group is specified.
 
+Mandatory tags (indicated by a colon after the B<Writable> type) may be
+added automatically with default values when creating a new IFD, and the IFD
+is removed automatically when deleting tags if only default-valued mandatory
+tags remain.
+
 The table below lists all EXIF tags.  Also listed are TIFF, DNG, HDP and
 other tags which are not part of the EXIF specification, but may co-exist
-with EXIF tags in some images.  Tags which are part of the EXIF 2.3
+with EXIF tags in some images.  Tags which are part of the EXIF 2.32
 specification have an underlined B<Tag Name> in the HTML version of this
 documentation.  See
-L<http://www.cipa.jp/english/hyoujunka/kikaku/pdf/DC-008-2010_E.pdf> for the
-official EXIF 2.3 specification.
+L<https://web.archive.org/web/20190624045241if_/http://www.cipa.jp:80/std/documents/e/DC-008-Translation-2019-E.pdf>
+for the official EXIF 2.32 specification.
 },
     GPS => q{
@@ -201 +281 @@
 indicated string lengths include a null terminator which is added
 automatically by ExifTool.  Remember that the descriptive values are used
-when writing (ie. 'Above Sea Level', not '0') unless the print conversion is
-disabled (with '-n' on the command line or the PrintConv option in the API,
+when writing (eg. 'Above Sea Level', not '0') unless the print conversion is
+disabled (with '-n' on the command line or the L<PrintConv|../ExifTool.html#PrintConv> option in the API,
 or by suffixing the tag name with a C<#> character).
 
 When adding GPS information to an image, it is important to set all of the
 following tags: GPSLatitude, GPSLatitudeRef, GPSLongitude, GPSLongitudeRef,
-GPSAltitude and GPSAltitudeRef.  ExifTool will write the required
-GPSVersionID tag automatically if new a GPS IFD is added to an image.
+and GPSAltitude and GPSAltitudeRef if the altitude is known.  ExifTool will
+write the required GPSVersionID tag automatically if new a GPS IFD is added
+to an image.
 },
     XMP => q{
@@ -214 +295 @@
 format which is being pushed by Adobe.  Information in this format can be
 embedded in many different image file types including JPG, JP2, TIFF, GIF,
-EPS, PDF, PSD, IND, PNG, DJVU, SVG, PGF, MIFF, XCF, CRW, DNG and a variety
-of proprietary TIFF-based RAW images, as well as MOV, AVI, ASF, WMV, FLV,
-SWF and MP4 videos, and WMA and audio formats supporting ID3v2 information.
+EPS, PDF, PSD, IND, INX, PNG, DJVU, SVG, PGF, MIFF, XCF, CRW, DNG and a
+variety of proprietary TIFF-based RAW images, as well as MOV, AVI, ASF, WMV,
+FLV, SWF and MP4 videos, and WMA and audio formats supporting ID3v2
+information.
 
 The XMP B<Tag ID>'s aren't listed because in most cases they are identical
@@ -229 +311 @@
 point number but stored as two C<integer> strings separated by a '/'
 character, C<date> is a date/time string entered in the format "YYYY:mm:dd
-HH:MM:SS[.ss][+/-HH:MM]", C<boolean> is either "True" or "False",
-C<lang-alt> indicates that the tag supports alternate languages (see below),
-and C<struct> is an XMP structure.  When reading, structures are extracted
-only if the Struct (-struct) option is used.  Otherwise the corresponding
-"flattened" tags, indicated by an underline (C<_>) after the B<Writable>
-type, are extracted.  When copying information, the Struct option is in
-effect by default.  When writing, the Struct option has no effect, and both
-structured and flattened tags may be written.  See
-L<http://owl.phy.queensu.ca/~phil/exiftool/struct.html> for more details.
+HH:MM:SS[.ss][+/-HH:MM]", C<boolean> is either "True" or "False" (but "true"
+and "false" may be written as a ValueConv value for compatibility with
+non-conforming applications), C<struct> indicates a structured tag, and
+C<lang-alt> is a tag that supports alternate languages.
+
+When reading, C<struct> tags are extracted only if the L<Struct|../ExifTool.html#Struct> (-struct)
+option is used.  Otherwise the corresponding I<Flattened> tags, indicated by
+an underline (C<_>) after the B<Writable> type, are extracted.  When
+copying, by default both structured and flattened tags are available, but
+the flattened tags are considered "unsafe" so they aren't copied unless
+specified explicitly.  The L<Struct|../ExifTool.html#Struct> option may be disabled by setting Struct
+to 0 via the API or with --struct on the command line to copy only flattened
+tags, or enabled by setting Struct to 1 via the API or with -struct on the
+command line to copy only as structures.  When writing, the L<Struct|../ExifTool.html#Struct> option
+has no effect, and both structured and flattened tags may be written.  See
+L<https://exiftool.org/struct.html> for more details.
 
 Individual languages for C<lang-alt> tags are accessed by suffixing the tag
-name with a '-', followed by an RFC 3066 language code (ie. "XMP:Title-fr",
+name with a '-', followed by an RFC 3066 language code (eg. "XMP:Title-fr",
 or "Rights-en-US").  (See L<http://www.ietf.org/rfc/rfc3066.txt> for the RFC
 3066 specification.)  A C<lang-alt> tag with no language code accesses the
 "x-default" language, but causes other languages for this tag to be deleted
 when writing.  The "x-default" language code may be specified when writing
-to preserve other existing languages (ie. "XMP-dc:Description-x-default"). 
+to preserve other existing languages (eg. "XMP-dc:Description-x-default"). 
 When reading, "x-default" is not specified.
 
 The XMP tags are organized according to schema B<Namespace> in the following
-tables.  Note that a few of the longer namespace prefixes given below have
-been shortened for convenience (since the family 1 group names are derived
-from these by adding a leading "XMP-").  In cases where a tag name exists in
-more than one namespace, less common namespaces are avoided when writing.
-However, any namespace may be written by specifying a family 1 group name
-for the tag, ie) XMP-exif:Contrast or XMP-crs:Contrast.  When deciding on
-which tags to add to an image, using standard schemas such as
-L<dc|/XMP dc Tags>, L<xmp|/XMP xmp Tags> or L<iptc|/XMP iptcCore Tags> is
+tables.  The ExifTool family 1 group names are derived from the namespace
+prefixes by adding a leading "XMP-" (eg. "XMP-dc").  A few of the longer
+prefixes have been shortened (as mentioned in the documentation below) to
+avoid excessively long ExifTool group names.  The tags of any namespace may
+be deleted as a group by specifying the family 1 group name (eg.
+"-XMP-dc:all=" on the command line).  This includes namespaces which are not
+pre-defined by ExifTool.
+
+In cases where a tag name exists in more than one namespace, less common
+namespaces are avoided when writing.  However, a specific namespace may be
+written by providing a family 1 group name for the tag (eg. XMP-crs:Contrast
+or XMP-exif:Contrast).  When deciding on which tags to add to an image,
+using standard schemas such as L<dc|/XMP dc Tags>, L<xmp|/XMP xmp Tags>,
+L<iptcCore|/XMP iptcCore Tags> and L<iptcExt|/XMP iptcExt Tags> is
 recommended if possible.
 
@@ -262 +357 @@
 names are very similar to tag names, except they are used to identify fields
 inside structures instead of stand-alone tags.  See
-L<the Field Name section of the Structured Information documentation|http://owl.phy.queensu.ca/~phil/exiftool/struct.html#Fields> for more
+L<the Field Name section of the Structured Information documentation|https://exiftool.org/struct.html#Fields> for more
 details.
 
 ExifTool will extract XMP information even if it is not listed in these
-tables.  For example, the C<pdfx> namespace doesn't have a predefined set of
-tag names because it is used to store application-defined PDF information,
-but this information is extracted by ExifTool.
-
-See L<http://www.adobe.com/devnet/xmp/> for the official XMP specification.
+tables, but other tags are not writable unless added as user-defined tags in
+the L<ExifTool config file|../config.html>.  For example, the C<pdfx> namespace doesn't have a
+predefined set of tag names because it is used to store application-defined
+PDF information, so although this information will be extracted, it is only
+writable if the corresponding user-defined tags have been created.
+
+The tables below list tags from the official XMP specification (with an
+underlined B<Namespace> in the HTML version of this documentation), as well
+as extensions from various other sources.  See
+L<http://www.adobe.com/devnet/xmp/> for the official XMP specification.
 },
     IPTC => q{
@@ -276 +376 @@
 Council (IPTC) and the Newspaper Association of America (NAA) Information
 Interchange Model (IIM).  This is an older meta information format, slowly
-being phased out in favor of XMP.  (In fact, the newer IPTCCore
-specification actually uses XMP format!)  IPTC information may be embedded
-in JPG, TIFF, PNG, MIFF, PS, PDF, PSD, XCF and DNG images.
+being phased out in favor of XMP -- the newer IPTCCore specification uses
+XMP format.  IPTC information may be found in JPG, TIFF, PNG, MIFF, PS, PDF,
+PSD, XCF and DNG images.
 
 IPTC information is separated into different records, each of which has its
 own set of tags.  See
 L<http://www.iptc.org/std/IIM/4.1/specification/IIMV4.1.pdf> for the
-official specification.
+official IPTC IIM specification.
 
 This specification dictates a length for ASCII (C<string> or C<digits>) and
@@ -289 +389 @@
 the B<Writable> format name.  For tags where a range of lengths is allowed,
 the minimum and maximum lengths are separated by a comma within the
-brackets.  IPTC strings are not null terminated.  When writing, ExifTool
-issues a minor warning and truncates the value if it is longer than allowed
-by the IPTC specification. Minor errors may be ignored with the
-IgnoreMinorErrors (-m) option, allowing longer values to be written, but
-beware that values like this may cause problems for some other IPTC readers.
+brackets.  When writing, ExifTool issues a minor warning and truncates the
+value if it is longer than allowed by the IPTC specification.  Minor errors
+may be ignored with the L<IgnoreMinorErrors|../ExifTool.html#IgnoreMinorErrors> (-m) option, allowing longer
+values to be written, but beware that values like this may cause problems
+for some other IPTC readers. ExifTool will happily read IPTC values of any
+length.
 
 Separate IPTC date and time tags may be written with a combined date/time
@@ -306 +407 @@
 offset at the specified date/time is used, which may be different due to
 changes in daylight savings time).
+
+Note that it is not uncommon for IPTC to be found in non-standard locations
+in JPEG and TIFF-based images.  When reading, the family 1 group name has a
+number added for non-standard IPTC ("IPTC2", "IPTC3", etc), but when writing
+only "IPTC" may be specified as the group.  To keep the IPTC consistent,
+ExifTool updates tags in all existing IPTC locations, but will create a new
+IPTC group only in the standard location.
+},
+    QuickTime => q{
+The QuickTime format is used for many different types of audio, video and
+image files (most notably, MOV/MP4 videos and HEIC/CR3 images).  ExifTool
+extracts standard meta information and a variety of audio, video and image
+parameters, as well as proprietary information written by many camera
+models.  Tags with a question mark after their name are not extracted unless
+the L<Unknown|../ExifTool.html#Unknown> option is set.
+
+When writing, ExifTool creates both QuickTime and XMP tags by default, but
+the group may be specified to write one or the other separately.  If no
+location is specified, newly created QuickTime tags are added in the
+L<ItemList|Image::ExifTool::TagNames/QuickTime ItemList Tags> location if
+possible, otherwise in
+L<UserData|Image::ExifTool::TagNames/QuickTime UserData Tags>, and
+finally in L<Keys|Image::ExifTool::TagNames/QuickTime Keys Tags>,
+but this order may be changed by setting the PREFERRED level of the
+appropriate table in the config file (see
+L<example.config|../config.html#PREF> in the full distribution for an
+example).  Note that some tags with the same name but different ID's may
+exist in the same location, but the family 7 group names may be used to
+differentiate these.  ExifTool currently writes only top-level metadata in
+QuickTime-based files; it extracts other track-specific and timed metadata,
+but can not yet edit tags in these locations (with the exception of
+track-level date/time tags).
+
+Alternate language tags may be accessed for
+L<ItemList|Image::ExifTool::TagNames/QuickTime ItemList Tags> and
+L<Keys|Image::ExifTool::TagNames/QuickTime Keys Tags> tags by adding
+a 3-character ISO 639-2 language code and an optional ISO 3166-1 alpha 2
+country code to the tag name (eg. "ItemList:Artist-deu" or
+"ItemList::Artist-deu-DE").  Most
+L<UserData|Image::ExifTool::TagNames/QuickTime UserData Tags> tags support a
+language code, but without a country code.  If no language code is specified
+when writing, the default language is written and alternate languages for
+the tag are deleted.  Use the "und" language code to write the default
+language without deleting alternate languages.  Note that "eng" is treated
+as a default language when reading, but not when writing.
+
+According to the specification, integer-format QuickTime date/time tags
+should be stored as UTC.  Unfortunately, digital cameras often store local
+time values instead (presumably because they don't know the time zone).  For
+this reason, by default ExifTool does not assume a time zone for these
+values.  However, if the L<QuickTimeUTC|../ExifTool.html#QuickTimeUTC> API option is set, then ExifTool will
+assume these values are properly stored as UTC, and will convert them to
+local time when extracting.
+
+When writing string-based date/time tags, the system time zone is added if
+the PrintConv option is enabled and no time zone is specified.  This is
+because Apple software may display crazy values if the time zone is missing
+for some tags.
+
+See
+L<https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/>
+for the official specification.
 },
     Photoshop => q{
@@ -314 +477 @@
 Many Photoshop tags are marked as Unknown (indicated by a question mark
 after the tag name) because the information they provide is not very useful
-under normal circumstances I<[and because Adobe denied my application for
-their file format documentation -- apparently open source software is too
-big a concept for them]>.  These unknown tags are not extracted unless the
-Unknown (-u) option is used.
+under normal circumstances.  These unknown tags are not extracted unless the
+L<Unknown|../ExifTool.html#Unknown> (-u) option is used.  See
+L<http://www.adobe.com/devnet-apps/photoshop/fileformatashtml/> for the
+official specification
+
+Photoshop path tags (Tag ID's 0x7d0 to 0xbb5) are not defined by default,
+but a config file included in the full ExifTool distribution
+(config_files/photoshop_paths.config) contains the tag definitions to allow
+access to this information.
 },
     PrintIM => q{
 The format of the PrintIM information is known, however no PrintIM tags have
-been decoded.  Use the Unknown (-u) option to extract PrintIM information.
+been decoded.  Use the L<Unknown|../ExifTool.html#Unknown> (-u) option to extract PrintIM information.
 },
     GeoTiff => q{
 ExifTool extracts the following tags from GeoTIFF images.  See
 L<http://www.remotesensing.org/geotiff/spec/geotiffhome.html> for the
-complete GeoTIFF specification.
+complete GeoTIFF specification.  Also included in the table below are
+ChartTIFF tags (see L<http://www.charttiff.com/whitepapers.shtml>). GeoTIFF
+tags are not writable individually, but they may be copied en mass via the
+block tags GeoTiffDirectory, GeoTiffDoubleParams and GeoTiffAsciiParams.
 },
     JFIF => q{
 The following information is extracted from the JPEG JFIF header.  See
-L<http://www.jpeg.org/public/jfif.pdf> for the JFIF 1.02 specification.
+L<https://www.w3.org/Graphics/JPEG/jfif3.pdf> for the JFIF 1.02
+specification.
 },
     Kodak => q{
@@ -350 +522 @@
     Olympus => q{
 Tags 0x0000 through 0x0103 are used by some older Olympus cameras, and are
-the same as Konica/Minolta tags.  The Olympus tags are also used for Epson
-and Agfa cameras.
+the same as Konica/Minolta tags.  These tags are also used for some models
+from other brands such as Acer, BenQ, Epson, Hitachi, HP, Maginon, Minolta,
+Pentax, Ricoh, Samsung, Sanyo, SeaLife, Sony, Supra and Vivitar.
 },
     Panasonic => q{
@@ -358 +531 @@
     Pentax => q{
 These tags are used in Pentax/Asahi cameras.
-},
-    Sigma => q{
-These tags are used in Sigma/Foveon cameras.
-},
-    Sony => q{
-The maker notes in images from most recent Sony camera models contain a
-wealth of information, but for some models very little has been decoded. 
-Use the ExifTool Unknown (-u) or Verbose (-v) options to see information
-about the unknown tags.  Also see the Minolta tags which are used by some
-Sony models.
 },
     CanonRaw => q{
@@ -374 +537 @@
 length of the information is preserved (and the new information is truncated
 or padded as required) unless B<Writable> is C<resize>. Currently, only
-JpgFromRaw and ThumbnailImage are allowed to change size.
+JpgFromRaw and ThumbnailImage are allowed to change size.  See
+L<https://exiftool.org/canon_raw.html> for a description of the Canon CRW
+format.
 
 CRW images also support the addition of a CanonVRD trailer, which in turn
@@ -380 +545 @@
 ExifTool is used to write XMP to a CRW image.
 },
+    NikonCustom => q{
+Unfortunately, the NikonCustom settings are stored in a binary data block
+which changes from model to model.  This means that significant effort must
+be spent in decoding these for each model, usually requiring hundreds of
+test images from a dedicated Nikon owner.  For this reason, the NikonCustom
+settings have not been decoded for all models.  The tables below list the
+custom settings for the currently supported models.
+},
     Unknown => q{
-The following tags are decoded in unsupported maker notes.  Use the Unknown
+The following tags are decoded in unsupported maker notes.  Use the L<Unknown|../ExifTool.html#Unknown>
 (-u) option to display other unknown tags.
 },
@@ -393 +566 @@
 ExifTool supports reading and writing PDF documents up to version 1.7
 extension level 3, including support for RC4, AES-128 and AES-256
-encryption.  A Password option is provided to allow processing of
+encryption.  A L<Password|../ExifTool.html#Password> option is provided to allow processing of
 password-protected PDF files.
 
-When writing PDF files, ExifTool uses an incremental update.  This has the
-advantages of being fast and reversible.  The original PDF can be easily
-recovered by deleting the C<PDF-update> pseudo-group (with
-C<-PDF-update:all=> on the command line).  But there are two main
+ExifTool may be used to write native PDF and XMP metadata to PDF files. It
+uses an incremental update technique that has the advantages of being both
+fast and reversible.  If ExifTool was used to modify a PDF file, the
+original may be recovered by deleting the C<PDF-update> pseudo-group (with
+C<-PDF-update:all=> on the command line).  However, there are two main
 disadvantages to this technique:
 
@@ -407 +581 @@
 2) All metadata edits are reversible.  While this would normally be
 considered an advantage, it is a potential security problem because old
-information is never actually deleted from the file.
+information is never actually deleted from the file.  (However, after
+running ExifTool the old information may be removed permanently using the
+"qpdf" utility with this command: "qpdf --linearize in.pdf out.pdf".)
 },
     DNG => q{
@@ -425 +601 @@
 },
     Extra => q{
-The extra tags represent extra information extracted or generated by
-ExifTool that is not directly associated with another tag group.  The three
-writable "pseudo" tags (FileName, Directory and FileModifyDate) may be
-written without the need to rewrite the file since their values are not
-contained within the file data.  These "pseudo" tags belong to the family 1
-"System" group.
+The extra tags provide extra features or extra information extracted or
+generated by ExifTool that is not directly associated with another tag
+group.  The B<Group> column lists the family 1 group name when reading. 
+Tags with a "-" in this column are write-only.
+
+Tags in the family 1 "System" group are referred to as "pseudo" tags because
+they don't represent real metadata in the file.  Instead, this information
+is stored in the directory structure of the filesystem.  The B<Writable>
+System "pseudo" tags in this table may be written without modifying the file
+itself.  The TestName tag is used for dry-run testing before writing
+FileName.
 },
     Composite => q{
 The values of the composite tags are B<Derived From> the values of other
 tags.  These are convenience tags which are calculated after all other
-information is extracted.
+information is extracted.  Only a few of these tags are writable directly,
+the others are changed by writing the corresponding B<Derived From> tags.
+User-defined Composite tags, also useful for custom-formatting of tag
+values, may created via the L<ExifTool configuration file|../config.html>.
 },
     Shortcuts => q{
@@ -447 +631 @@
 for more details.
 },
+    MWG => q{
+The Metadata Working Group (MWG) recommends techniques to allow certain
+overlapping EXIF, IPTC and XMP tags to be reconciled when reading, and
+synchronized when writing.  The MWG Composite tags below are designed to aid
+in the implementation of these recommendations.  As well, the MWG defines
+new XMP tags which are listed in the subsequent tables below.  See
+L<https://web.archive.org/web/20181006115950/http://www.metadataworkinggroup.org/specs/>
+for the official MWG specification.
+},
+    MacOS => q{
+On MacOS systems, the there are additional MDItem and XAttr Finder tags that
+may be extracted.  These tags are not extracted by default -- they must be
+specifically requested or enabled via an API option.  (Except when reading
+MacOS "._" files directly, see below.)
+
+The tables below list some of the tags that may be extracted, but ExifTool
+will extract all available information even for tags not listed.
+
+Tags in these tables are referred to as "pseudo" tags because their
+information is not stored in the file itself.  As such, B<Writable> tags in
+these tables may be changed without having to rewrite the file.
+},
     PodTrailer => q{
 ~head1 NOTES
@@ -455 +661 @@
 ~head1 AUTHOR
 
-Copyright 2003-2011, Phil Harvey (phil at owl.phy.queensu.ca)
+Copyright 2003-2021, Phil Harvey (philharvey66 at gmail.com)
 
 This library is free software; you can redistribute it and/or modify it
@@ -470 +676 @@
 # notes for Shortcuts tags
 my %shortcutNotes = (
+    AllDates => q{
+        contrary to the shortcut name, this represents only the common EXIF
+        date/time tags.  To access all date/time tags, use Time:All instead
+    },
     MakerNotes => q{
         useful when copying tags between files to either copy the maker notes as a
         block or prevent it from being copied
     },
+    ColorSpaceTags => q{
+        standard tags which carry color space information.  Useful for preserving
+        color space when deleting all other metadata
+    },
     CommonIFD0 => q{
         common metadata tags found in IFD0 of TIFF-format images.  Used to simpify
-        deletion of all metadata from these images.  See FAQ number 7 for details
+        deletion of all metadata from these images.  See
+        L<FAQ number 7|../faq.html#Q7> for details
     },
     Unsafe => q{
-        "unsafe" tags in JPEG images which are normally not copied.  Defined here
-        as a shortcut to use when rebuilding JPEG EXIF from scratch
+        I<Unsafe> tags in JPEG images which are normally not copied.  Defined here
+        as a shortcut to use when rebuilding JPEG EXIF from scratch. See
+        L<FAQ number 20|../faq.html#Q20> for more information
+    },
+    LargeTags => q{
+        large binary data tags which may be excluded to reduce memory usage if
+        memory limitations are a problem
+    },
+   'ls-l' => q{
+        mimics columns shown by Unix "ls -l" command.  Includes some tags which are
+        extracted only if the API L<SystemTags|../ExifTool.html#SystemTags> option
+        is enabled
     },
 );
 
 
-
-# EXIF table tag ID's which are part of the EXIF 2.3 specification
-# (used only to add underlines in HTML version of EXIF Tag Table)
-my %exifSpec = (
-    0x100 => 1,  0x212 => 1,   0x9204 => 1,  0xa217 => 1,
-    0x101 => 1,  0x213 => 1,   0x9205 => 1,  0xa300 => 1,
-    0x102 => 1,  0x214 => 1,   0x9206 => 1,  0xa301 => 1,
-    0x103 => 1,  0x8298 => 1,  0x9207 => 1,  0xa302 => 1,
-    0x106 => 1,  0x829a => 1,  0x9208 => 1,  0xa401 => 1,
-    0x10e => 1,  0x829d => 1,  0x9209 => 1,  0xa402 => 1,
-    0x10f => 1,  0x8769 => 1,  0x920a => 1,  0xa403 => 1,
-    0x110 => 1,  0x8822 => 1,  0x9214 => 1,  0xa404 => 1,
-    0x111 => 1,  0x8824 => 1,  0x927c => 1,  0xa405 => 1,
-    0x112 => 1,  0x8825 => 1,  0x9286 => 1,  0xa406 => 1,
-    0x115 => 1,  0x8827 => 1,  0x9290 => 1,  0xa407 => 1,
-    0x116 => 1,  0x8828 => 1,  0x9291 => 1,  0xa408 => 1,
-    0x117 => 1,  0x8830 => 1,  0x9292 => 1,  0xa409 => 1,
-    0x11a => 1,  0x8831 => 1,  0xa000 => 1,  0xa40a => 1,
-    0x11b => 1,  0x8832 => 1,  0xa001 => 1,  0xa40b => 1,
-    0x11c => 1,  0x8833 => 1,  0xa002 => 1,  0xa40c => 1,
-    0x128 => 1,  0x8834 => 1,  0xa003 => 1,  0xa420 => 1,
-    0x12d => 1,  0x8835 => 1,  0xa004 => 1,  0xa430 => 1,
-    0x131 => 1,  0x9000 => 1,  0xa005 => 1,  0xa431 => 1,
-    0x132 => 1,  0x9003 => 1,  0xa20b => 1,  0xa432 => 1,
-    0x13b => 1,  0x9004 => 1,  0xa20c => 1,  0xa433 => 1,
-    0x13e => 1,  0x9101 => 1,  0xa20e => 1,  0xa434 => 1,
-    0x13f => 1,  0x9102 => 1,  0xa20f => 1,  0xa435 => 1,
-    0x201 => 1,  0x9201 => 1,  0xa210 => 1,
-    0x202 => 1,  0x9202 => 1,  0xa214 => 1,
-    0x211 => 1,  0x9203 => 1,  0xa215 => 1,
-);
 # same thing for RIFF INFO tags found in the EXIF spec
 my %riffSpec = (
@@ -523 +717 @@
     ICMT => 1,  IDPI => 1,  IMED => 1,  ISFT => 1,
     ICOP => 1,  IENG => 1,  INAM => 1,  ISHP => 1,
+);
+# same thing for XMP namespaces
+my %xmpSpec = (
+    aux       => 1,   'x'        => 1,
+    crs       => 1,    xmp       => 1,
+    dc        => 1,    xmpBJ     => 1,
+    exif      => 1,    xmpDM     => 1,
+    pdf       => 1,    xmpMM     => 1,
+    pdfx      => 1,    xmpNote   => 1,
+    photoshop => 1,    xmpRights => 1,
+    tiff      => 1,    xmpTPg    => 1,
 );
 
@@ -534 +739 @@
     my $class = ref($that) || $that || 'Image::ExifTool::BuildTagLookup';
     my $self = bless {}, $class;
-    my (%subdirs, %isShortcut, %allStructs);
+    my (%subdirs, %isShortcut);
     my %count = (
         'unique tag names' => 0,
@@ -543 +748 @@
 #
     my (%tagNameInfo, %id, %longID, %longName, %shortName, %tableNum,
-        %tagLookup, %tagExists, %tableWritable, %sepTable, %structs,
-        %compositeModules, %isPlugin, %flattened, %structLookup);
+        %tagLookup, %tagExists, %noLookup, %tableWritable, %sepTable, %case,
+        %structs, %compositeModules, %isPlugin, %flattened, %structLookup,
+        @writePseudo);
     $self->{TAG_NAME_INFO} = \%tagNameInfo;
     $self->{ID_LOOKUP} = \%id;
@@ -560 +766 @@
     $self->{COMPOSITE_MODULES} = \%compositeModules;
     $self->{COUNT} = \%count;
+    $self->{WRITE_PSEUDO} = \@writePseudo;
 
     Image::ExifTool::LoadAllTables();
@@ -566 +773 @@
     push @tableNames, 'Image::ExifTool::Shortcuts::Main';
     # add plug-in modules last
-    $Image::ExifTool::documentOnly = 1; # (don't really load them)
     foreach (@pluginTables) {
         push @tableNames, $_;
@@ -573 +779 @@
 
     my $tableNum = 0;
-    my $exifTool = new Image::ExifTool;
+    my $et = new Image::ExifTool;
     my ($tableName, $tag);
     # create lookup for short table names
@@ -583 +789 @@
         $short =~ s/^(.+)Tags$/\u$1/ unless $short eq 'Nikon AVITags';
         $short =~ s/^Exif\b/EXIF/;
+        # change underlines to dashes in XMP-mwg group names
+        # (we used underlines just because Perl variables can't contain dashes)
+        $short =~ s/^XMP mwg_/XMP mwg-/;
         $shortName{$tableName} = $short;    # remember short name
         $tableNum{$tableName} = $tableNum++;
@@ -622 +831 @@
         $longName{$tableName} = 0;
         # save all tag names
-        my ($tagID, $binaryTable, $noID, $isIPTC, $isXMP);
+        my ($tagID, $binaryTable, $noID, $hexID, $isIPTC, $isXMP);
         $isIPTC = 1 if $writeProc and $writeProc eq \&Image::ExifTool::IPTC::WriteIPTC;
         # generate flattened tag names for structure fields if this is an XMP table
         if ($$table{GROUPS} and $$table{GROUPS}{0} eq 'XMP') {
+            Image::ExifTool::XMP::AddFlattenedTags($table);
             $isXMP = 1;
-            foreach $tagID (TagTableKeys($table)) {
-                my $tagInfo = $$table{$tagID};
-                next unless ref $tagInfo eq 'HASH' and $$tagInfo{Struct};
-                Image::ExifTool::XMP::AddFlattenedTags($table, $tagID);
-            }
         }
         $noID = 1 if $isXMP or $short =~ /^(Shortcuts|ASF.*)$/ or $$vars{NO_ID};
+        $hexID = $$vars{HEX_ID};
         my $processBinaryData = ($$table{PROCESS_PROC} and (
             $$table{PROCESS_PROC} eq \&Image::ExifTool::ProcessBinaryData or
-            $$table{PROCESS_PROC} eq \&Image::ExifTool::Nikon::ProcessNikonEncrypted));
+            $$table{PROCESS_PROC} eq \&Image::ExifTool::Nikon::ProcessNikonEncrypted or
+            $$table{PROCESS_PROC} eq \&Image::ExifTool::Sony::ProcessEnciphered));
         if ($$vars{ID_LABEL} or $processBinaryData) {
+            my $s = $$table{FORMAT} ? Image::ExifTool::FormatSize($$table{FORMAT}) || 1 : 1;
             $binaryTable = 1;
-            $id{$tableName} = $$vars{ID_LABEL} || 'Index';
+            $id{$tableName} = $$vars{ID_LABEL} || "Index$s";
         } elsif ($isIPTC and $$table{PROCESS_PROC}) { #only the main IPTC table has a PROCESS_PROC
             $id{$tableName} = 'Record';
@@ -646 +854 @@
         }
         $caseInsensitive = $isXMP;
+        $numbersFirst = 2;
         $numbersFirst = -1 if $$table{VARS} and $$table{VARS}{ALPHA_FIRST};
-        my @keys = sort NumbersFirst TagTableKeys($table);
+        my @keys = SortedTagTableKeys($table);
         $numbersFirst = 1;
         my $defFormat = $table->{FORMAT};
@@ -654 +863 @@
 
 TagID:  foreach $tagID (@keys) {
-            my ($tagInfo, @tagNames, $subdir, $format, @values);
+            my ($tagInfo, @tagNames, $subdir, @values);
             my (@infoArray, @require, @writeGroup, @writable);
             if ($shortcut) {
@@ -672 +881 @@
                 @infoArray = GetTagInfoList($table,$tagID);
             }
-            $format = $defFormat;
             foreach $tagInfo (@infoArray) {
                 my $name = $$tagInfo{Name};
-                # validate Name
-                warn "Warning: Invalid tag name $short '$name'\n" if $name !~ /^[-\w]+$/;
+                if ($$tagInfo{WritePseudo}) {
+                    push @writePseudo, $name;
+                    warn "Writable pseudo tag $name is not protected!\n" unless $$tagInfo{Protected};
+                }
+                unless ($$tagInfo{SubDirectory} or $$tagInfo{Struct}) {
+                    my $lc = lc $name;
+                    warn "Different case for $tableName $name $case{$lc}\n" if $case{$lc} and $case{$lc} ne $name;
+                    $case{$lc} = $name;
+                }
+                my $format = $$tagInfo{Format};
+                # check TagID's to make sure they don't start with 'ID-'
+                my @grps = $et->GetGroup($tagInfo);
+                foreach (@grps) {
+                    warn "Group name starts with 'ID-' for $short $name\n" if /^ID-/i;
+                }
+                # validate Name (must not start with a digit or else XML output will not be valid;
+                # must not start with a dash or exiftool command line may get confused)
+                if ($name !~ /^[_A-Za-z][-\w]+$/ and
+                    # single-character subdirectory names are allowed
+                    (not $$tagInfo{SubDirectory} or $name !~ /^[_A-Za-z]$/))
+                {
+                    warn "Warning: Invalid tag name $short '${name}'\n";
+                }
+                # validate list type
+                if ($$tagInfo{List} and $$tagInfo{List} !~ /^(1|Alt|Bag|Seq|array|string)$/) {
+                    warn "Warning: Unknown List type ($$tagInfo{List}) for $name in $tableName\n";
+                }
                 # accumulate information for consistency check of BinaryData tables
                 if ($processBinaryData and $$table{WRITABLE}) {
+                    # can't currently write tag if Condition accesses $valPt
+                    if ($$tagInfo{Condition} and not $$tagInfo{SubDirectory} and
+                        $$tagInfo{Condition} =~ /\$valPt/ and
+                        ($$tagInfo{Writable} or not defined $$tagInfo{Writable}))
+                    {
+                        warn "Warning: Tag not writable due to Condition - $short $name\n";
+                    }
                     $isOffset{$tagID} = $name if $$tagInfo{IsOffset};
                     $hasSubdir{$tagID} = $name if $$tagInfo{SubDirectory};
                     # require DATAMEMBER for writable var-format tags, Hook and DataMember tags
-                    if ($$tagInfo{Format} and $$tagInfo{Format} =~ /^var_/) {
+                    if ($format and $format =~ /^var_/) {
                         $datamember{$tagID} = $name;
                         unless (defined $$tagInfo{Writable} and not $$tagInfo{Writable}) {
                             warn "Warning: Var-format tag is writable - $short $name\n"
+                        }
+                        # also need DATAMEMBER for tags used in length of var-sized value
+                        while ($format =~ /\$val\{(.*?)\}/g) {
+                            my $id = $1;
+                            $id = hex($id) if $id =~ /^0x/; # convert from hex if necessary
+                            $datamember{$id} = $$table{$id}{Name} if ref $$table{$id} eq 'HASH';
+                            unless ($datamember{$id}) {
+                                warn "Warning: Unknown ID ($id) used in Format - $short $name\n";
+                            }
                         }
                     } elsif ($$tagInfo{Hook} or ($$tagInfo{RawConv} and
@@ -692 +941 @@
                         $datamember{$tagID} = $name;
                     }
+                    if ($format and $format =~ /\$val\{/ and
+                        ($$tagInfo{Writable} or not defined $$tagInfo{Writable}))
+                    {
+                        warn "Warning: \$var{} used in Format of writable tag - $short $name\n"
+                    }
                 }
                 if ($$tagInfo{Hidden}) {
-                    warn "Warning: Hidden tag in list - $short $name\n" if @infoArray > 1;
-                    next TagID;
+                    if ($tagInfo == $infoArray[0]) {
+                        next TagID; # hide all tags with this ID if first tag in list is hidden
+                    } else {
+                        next;       # only hide this tag
+                    }
                 }
                 my $writable;
@@ -702 +959 @@
                     # validate Writable
                     unless ($formatOK{$writable} or  ($writable =~ /(.*)\[/ and $formatOK{$1})) {
-                        warn "Warning: Unknown Writable ($writable) for $short $name\n",
+                        warn "Warning: Unknown Writable ($writable) - $short $name\n",
                     }
                 } elsif (not $$tagInfo{SubDirectory}) {
                     $writable = $$table{WRITABLE};
                 }
+                # in general, we can't write unless we have a WRITE_PROC
+                if ($writable and not ($$table{WRITE_PROC} or $tableName =~ /Shortcuts/ or $writable eq '2')) {
+                    undef $writable;
+                }
                 # validate some characteristics of obvious date/time tags
+                my @g = $et->GetGroup($tagInfo);
+                if ($$tagInfo{List} and $g[2] eq 'Time' and $writable and not $$tagInfo{Protected} and
+                    not $$tagInfo{PrintConvInv})
+                {
+                    # (this is a problem because shifting Time:All would create a new list entry)
+                    warn "Writable List-type Time tag $g[1]:$name has no PrintConvInv and is not Protected!\n";
+                }
                 if ($$tagInfo{PrintConv} and $$tagInfo{PrintConv} eq '$self->ConvertDateTime($val)') {
-                    my @g = $exifTool->GetGroup($tagInfo);
                     warn "$short $name should be in 'Time' group!\n" unless $g[2] eq 'Time';
-                    if ($writable and not $$tagInfo{Shift} and $g[0] ne 'Composite' and
-                        $short ne 'PostScript')
+                    if ($writable and not defined $$tagInfo{Shift} and $short ne 'PostScript') {
+                        warn "$short $name is not shiftable!\n";
+                    }
+                    if ($writable and (not $$tagInfo{PrintConvInv} or
+                        $$tagInfo{PrintConvInv} !~ /InverseDateTime/))
                     {
-                        warn "$short $name is not shiftable!\n";
+                        warn "$short $name missing InverseDateTime PrintConvInv\n";
                     }
                 } elsif ($name =~ /DateTime(?!Stamp)/ and (not $$tagInfo{Groups}{2} or
                     $$tagInfo{Groups}{2} ne 'Time') and $short ne 'DICOM') {
                     warn "$short $name should be in 'Time' group!\n";
+                }
+                if ($name eq 'DateTimeOriginal' and (not $$tagInfo{Description} or
+                    $$tagInfo{Description} ne 'Date/Time Original'))
+                {
+                    warn "Need Description for $short DateTimeOriginal\n";
                 }
                 # validate Description (can't contain special characters)
@@ -728 +1003 @@
                     warn "$name description contains special characters!\n";
                 }
-                # validate SubIFD flag
-                my $subdir = $$tagInfo{SubDirectory};
+                # generate structure lookup
                 my $struct = $$tagInfo{Struct};
                 my $strTable;
@@ -749 +1023 @@
                     $strTable = $structLookup{$struct};
                     unless ($strTable) {
-                        warn "Missing XMP $struct structure!\n";
+                        $struct = "XMP $struct" unless $struct =~ / /;
+                        warn "Missing $struct structure!\n";
                         undef $struct;
                     }
                 }
-                my $isSub = ($subdir and $$subdir{Start} and $$subdir{Start} eq '$val');
+                # validate SubIFD flag
+                my $subdir = $$tagInfo{SubDirectory};
+                my $isSub = ($subdir and $$subdir{Start} and $$subdir{Start} =~ /\$val\b/);
                 if ($$tagInfo{SubIFD}) {
                     warn "Warning: Wrong SubDirectory Start for SubIFD tag - $short $name\n" unless $isSub;
@@ -762 +1039 @@
                     my $note = $$tagInfo{Notes};
                     # remove leading/trailing blank lines
-                    $note =~ s/(^\s+|\s+$)//g;
+                    $note =~ s/^\s+//; $note =~ s/\s+$//;
                     # remove leading/trailing spaces on each line
                     $note =~ s/(^[ \t]+|[ \t]+$)//mg;
                     push @values, "($note)";
-                } elsif ($isXMP and lc $tagID ne lc $name) {
-                    # add note about different XMP Tag ID
-                    if ($$tagInfo{RootTagInfo}) {
-                        push @values, "($tagID)";
+                }
+                if ($isXMP and (lc $tagID ne lc $name or $$tagInfo{NotFlat})) {
+                    my $note;
+                    if ($$tagInfo{NotFlat}) {
+                        $note = 'NOT a flattened tag!';
                     } else {
-                        push @values,"(called $tagID by the spec)";
+                        # add note about different XMP Tag ID
+                        $note = $$tagInfo{RootTagInfo} ? $tagID : "called $tagID by the spec";
+                    }
+                    if ($$tagInfo{Notes}) {
+                        $values[-1] =~ s/^\(/($note; /;
+                    } else {
+                        push @values, "($note)";
                     }
                 }
                 my $writeGroup;
-                $writeGroup = $$tagInfo{WriteGroup};
+                if ($short eq 'Extra') {
+                    $writeGroup = $$tagInfo{WriteOnly} ? '-' : $g[1];
+                } else {
+                    $writeGroup = $$tagInfo{WriteGroup};
+                }
                 unless ($writeGroup) {
                     $writeGroup = $$table{WRITE_GROUP} if $writable;
                     $writeGroup = '-' unless $writeGroup;
                 }
-                if (defined $$tagInfo{Format}) {
-                    $format = $$tagInfo{Format};
+                if (defined $format) {
                     # validate Format
                     unless ($formatOK{$format} or $short eq 'PICT' or
@@ -788 +1075 @@
                         warn "Warning: Unknown Format ($format) for $short $name\n";
                     }
+                } else {
+                    $format = $defFormat;
                 }
                 if ($subdir) {
@@ -810 +1099 @@
                 if ($$tagInfo{Mask}) {
                     my $val = $$tagInfo{Mask};
-                    push @values, sprintf('[Mask 0x%.2x]',$val);
-                    $$tagInfo{PrintHex} = 1 unless defined $$tagInfo{PrintHex};
+                    my $bsh = $$tagInfo{BitShift};
+                    if ($bsh) {
+                        push @values, sprintf('[val >> %d & 0x%x]',$bsh,$val>>$bsh);
+                    } else {
+                        push @values, sprintf('[val & 0x%x]',$val);
+                    }
                     # verify that all values are within the mask
                     if (ref $printConv eq 'HASH') {
-                        # convert mask if necessary
-                        if ($$tagInfo{ValueConv}) {
-                            my $v = eval $$tagInfo{ValueConv};
-                            $val = $v if defined $v;
-                        }
+                        $val >>= $$tagInfo{BitShift};
                         foreach (keys %$printConv) {
                             next if $_ !~ /^\d+$/ or ($_ & $val) == $_;
@@ -827 +1116 @@
                 }
                 if (ref($printConv) =~ /^(HASH|ARRAY)$/) {
-                    my (@printConvList, @indexList, $index);
+                    my (@printConvList, @indexList, $index, $valueConvHash);
                     if (ref $printConv eq 'ARRAY') {
                         for ($index=0; $index<@$printConv; ++$index) {
-                            next if ref $$printConv[$index] ne 'HASH';
-                            next unless %{$$printConv[$index]};
-                            push @printConvList, $$printConv[$index];
-                            push @indexList, $index;
+                            if (ref $$printConv[$index] eq 'HASH') {
+                                next unless %{$$printConv[$index]};
+                                push @printConvList, $$printConv[$index];
+                                push @indexList, $index;
+                            } elsif ($$printConv[$index] and $$printConv[$index] eq 'REPEAT' and $index) {
+                                push @printConvList, $$printConv[$index-1];
+                                push @indexList, 'N';
+                            } else {
+                                next;
+                            }
                             # collapse values with identical PrintConv's
                             if (@printConvList >= 2 and $printConvList[-1] eq $printConvList[-2]) {
@@ -847 +1142 @@
                         $printConv = shift @printConvList;
                         $index = shift @indexList;
+                    } else {
+                        $valueConvHash = $$tagInfo{ValueConv} if ref $$tagInfo{ValueConv} eq 'HASH';
                     }
                     while (defined $printConv) {
@@ -858 +1155 @@
                                 my ($i, @i, $rngStart);
                                 for ($i=0; $i<@$idx; ++$i) {
-                                    if ($i < @$idx - 1 and $$idx[$i+1] == $$idx[$i] + 1) {
-                                        $rngStart = $i unless defined $rngStart;
+                                    if ($i < @$idx - 1 and ($$idx[$i+1] eq 'N' or $$idx[$i+1] == $$idx[$i] + 1)) {
+                                        $rngStart = $$idx[$i] unless defined $rngStart;
                                         next;
                                     }
-                                    push @i, defined($rngStart) ? "$rngStart-$i" : $i;
+                                    push @i, (defined($rngStart) ? "$rngStart-" : '') . $$idx[$i];
                                 }
                                 ($idx = join ', ', @i) =~ s/(.*),/$1 and/;
@@ -890 +1187 @@
                         } else {
                             $caseInsensitive = 0;
-                            my @pk = sort NumbersFirst keys %$printConv;
+                            my @pk;
+                            if ($$tagInfo{PrintSort}) {
+                                @pk = sort { NumbersFirst($$printConv{$a},$$printConv{$b}) } keys %$printConv;
+                            } else {
+                                @pk = sort { NumbersFirst($a,$b) } keys %$printConv;
+                            }
                             my $n = scalar @values;
-                            my ($bits, $cols, $i);
+                            my ($bits, $i, $v);
                             foreach (@pk) {
-                                next if $_ eq '';
+                                next if $_ eq '' and $$printConv{$_} eq '';
                                 $_ eq 'BITMASK' and $bits = $$printConv{$_}, next;
                                 $_ eq 'OTHER' and next;
                                 my $index;
-                                if (($$tagInfo{PrintHex} or $$printConv{BITMASK}) and /^\d+$/) {
-                                    $index = sprintf('0x%x',$_);
+                                if (($$tagInfo{PrintHex} or $$printConv{BITMASK}) and /^-?\d+$/) {
+                                    my $dig = $$tagInfo{PrintHex} || 1;
+                                    if ($_ >= 0) {
+                                        $index = sprintf('0x%.*x', $dig, $_);
+                                    } elsif ($format and $format =~ /int(16|32)/) {
+                                        # mask off unused bits of signed integer hex value
+                                        my $mask = { 16 => 0xffff, 32 => 0xffffffff }->{$1};
+                                        $index = sprintf('0x%.*x', $dig, $_ & $mask);
+                                    } else {
+                                        $index = $_;
+                                    }
                                 } elsif (/^[+-]?(?=\d|\.\d)\d*(\.\d*)?$/ and not $$tagInfo{PrintString}) {
                                     $index = $_;
@@ -908 +1219 @@
                                         $index = qq{"$index"};
                                     } else {
-                                        $index = qq{'$index'};
+                                        $index = qq{'${index}'};
                                     }
                                 }
                                 push @values, "$index = " . $$printConv{$_};
+                                if ($valueConvHash) {
+                                    foreach $v (keys %$valueConvHash) {
+                                        next unless $$valueConvHash{$v} eq $_;
+                                        $values[-1] = "$v => " . $values[-1];
+                                        last;
+                                    }
+                                }
                                 # validate all PrintConv values
                                 if ($$printConv{$_} =~ /[\0-\x1f\x7f-\xff]/) {
@@ -918 +1236 @@
                             }
                             if ($bits) {
-                                my @pk = sort NumbersFirst keys %$bits;
+                                my @pk = sort { NumbersFirst($a,$b) } keys %$bits;
                                 foreach (@pk) {
                                     push @values, "Bit $_ = " . $$bits{$_};
@@ -924 +1242 @@
                             }
                             # organize values into columns if specified
-                            if (defined($cols = $$tagInfo{PrintConvColumns})) {
+                            my $cols = $$tagInfo{PrintConvColumns};
+                            if (not $cols and scalar(@values) - $n >= 6) {
+                                # do columns if more than 6 short entries
+                                my $maxLen = 0;
+                                for ($i=$n; $i<@values; ++$i) {
+                                    next unless $maxLen < length $values[$i];
+                                    $maxLen = length $values[$i];
+                                }
+                                my $num = scalar(@values) - $n;
+                                $cols = int(50 / ($maxLen + 2)); # (50 chars max width)
+                                # have 3 rows minimum
+                                --$cols while $cols and $num / $cols < 3;
+                            }
+                            if ($cols) {
                                 my @new = splice @values, $n;
                                 my $v = '[!HTML]<table class=cols><tr>';
                                 my $rows = int((scalar(@new) + $cols - 1) / $cols);
-                                for ($n=0; $n<@new; $n+=$rows) {
+                                for ($n=0; ;) {
                                     $v .= "\n  <td>";
                                     for ($i=0; $i<$rows and $n+$i<@new; ++$i) {
@@ -934 +1265 @@
                                         $v .= EscapeHTML($new[$n+$i]);
                                     }
-                                    $v .= '</td><td>&nbsp;&nbsp;</td>';
+                                    $v .= '</td>';
+                                    last if ($n += $rows) >= @new;
+                                    $v .= '<td>&nbsp;&nbsp;</td>'; # add spaces between columns
                                 }
                                 push @values, $v . "</tr></table>\n";
@@ -950 +1283 @@
                         warn $@;
                     } else {
-                        my @pk = sort NumbersFirst keys %$bits;
+                        my @pk = sort { NumbersFirst($a,$b) } keys %$bits;
                         foreach (@pk) {
                             push @values, "Bit $_ = " . $$bits{$_};
@@ -957 +1290 @@
                 }
                 if ($subdir and not $$tagInfo{SeparateTable}) {
-                    # subdirectories are only writable if specified explicitly
-                    my $tw = $$tagInfo{Writable};
-                    $writable = '-' . ($tw ? $writable : '');
-                    $writable .= '!' if $tw and ($$tagInfo{Protected} || 0) & 0x01;
+                    if ($writable) {
+                        # subdirectories are only writable if specified explicitly
+                        my $tw = $$tagInfo{Writable};
+                        $writable = 'yes' if $tw and $writable eq '1' or $writable eq '2';
+                        $writable = '-' . ($tw ? $writable : '');
+                        $writable .= '!' if $tw and ($$tagInfo{Protected} || 0) & 0x01;
+                        $writable .= '+' if $$tagInfo{List};
+                        if (defined $$tagInfo{Permanent}) {
+                            $writable .= '^' unless $$tagInfo{Permanent};
+                        } elsif (defined $$table{PERMANENT}) {
+                            $writable .= '^' unless $$table{PERMANENT};
+                        }
+                    } else {
+                        $writable = '-';
+                    }
                 } else {
                     # not writable if we can't do the inverse conversions
@@ -978 +1322 @@
                     }
                     if (not $writable) {
-                        $writable = 'N';
+                        $writable = 'no';
                     } else {
-                        $writable eq '1' and $writable = $format ? $format : 'Y';
+                        $writable = $format ? $format : 'yes' if $writable eq '1' or $writable eq '2';
                         my $count = $$tagInfo{Count} || 1;
                         # adjust count to Writable size if different than Format
@@ -1008 +1352 @@
                     $writable .= '+' if $$tagInfo{List};
                     $writable .= ':' if $$tagInfo{Mandatory};
+                    if (defined $$tagInfo{Permanent}) {
+                        $writable .= '^' unless $$tagInfo{Permanent};
+                    } elsif (defined $$table{PERMANENT}) {
+                        $writable .= '^' unless $$table{PERMANENT};
+                    }
                     # separate tables link like subdirectories (flagged with leading '-')
                     $writable = "-$writable" if $subdir;
@@ -1028 +1377 @@
                 }
 #
-# add this tag to the tag lookup unless PROCESS_PROC is 0 or shortcut or plug-in tag
+# add this tag to the tag lookup unless NO_LOOKUP is set or shortcut or plug-in tag
 #
                 next if $shortcut or $isPlugin;
-                next if defined $$table{PROCESS_PROC} and not $$table{PROCESS_PROC};
-                # count our tags
+                # count tags
                 if ($$tagInfo{SubDirectory}) {
+                    next if $$vars{NO_LOOKUP};
                     $subdirs{$lcName} or $subdirs{$lcName} = 0;
                     ++$subdirs{$lcName};
                 } else {
                     ++$count{'total tags'};
-                    unless ($tagExists{$lcName} and (not $subdirs{$lcName} or $subdirs{$lcName} == $tagExists{$lcName})) {
-                        ++$count{'unique tag names'};
-                    }
+                    unless ($tagExists{$lcName} and
+                        (not $subdirs{$lcName} or $subdirs{$lcName} == $tagExists{$lcName}))
+                    {
+                        ++$count{'unique tag names'} unless $noLookup{$lcName};
+                    }
+                    # don't add to tag lookup if specified
+                    $$vars{NO_LOOKUP} and $noLookup{$lcName} = 1, next;
                 }
                 $tagExists{$lcName} or $tagExists{$lcName} = 0;
@@ -1071 +1424 @@
                 $tableWritable{$tableName} = 1;
                 $tagLookup{$lcName}->{$tableNum} = $tagIDs;
-                if ($short eq 'Composite' and $$tagInfo{Module}) {
-                    $compositeModules{$lcName} = $$tagInfo{Module};
-                }
+                # keep track of extra modules needed for Composite tags
+                $compositeModules{$lcName} = $$tagInfo{Module} if $$tagInfo{Module};
             }
 #
@@ -1079 +1431 @@
 #
             my $tagIDstr;
-            if ($tagID =~ /^\d+(\.\d+)?$/) {
-                if ($1 or $binaryTable or $isIPTC or ($short =~ /^CanonCustom/ and $tagID < 256)) {
-                    if ($tagID < 0x10000) {
-                        $tagIDstr = $tagID;
-                    } else {
-                        $tagIDstr = sprintf('0x%.8x',$tagID);
-                    }
+            if ($tagID =~ /^(-)?\d+(\.\d+)?$/) {
+                if ($1) {
+                    $tagIDstr = $tagID;
+                } elsif (defined $hexID) {
+                    $tagIDstr = $hexID ? sprintf('0x%.4x',$tagID) : $tagID;
+                } elsif (not $2 and not $binaryTable and not $isIPTC and
+                         not ($short =~ /^CanonCustom/ and $tagID < 256))
+                {
+                    $tagIDstr = sprintf('0x%.4x',$tagID);
+                } elsif ($tagID < 0x10000) {
+                    $tagIDstr = $tagID;
                 } else {
-                    $tagIDstr = sprintf('0x%.4x',$tagID);
+                    $tagIDstr = sprintf('0x%.8x',$tagID);
                 }
             } elsif ($short eq 'DICOM') {
                 ($tagIDstr = $tagID) =~ s/_/,/;
+            } elsif ($tagID =~ /^0x([0-9a-f]+)\.(\d+)$/) {  # DR4 tags like '0x20500.0'
+                $tagIDstr = $tagID;
             } else {
                 # convert non-printable characters to hex escape sequences
@@ -1098 +1456 @@
                     $tagIDstr = qq{"$tagID"};
                 } else {
-                    $tagIDstr = "'$tagID'";
+                    $tagIDstr = "'${tagID}'";
                 }
             }
@@ -1122 +1480 @@
                     foreach $tagID (@{$$table{$var}}) {
                         $$hash{$tagID} and delete($$hash{$tagID}), next;
-                        warn "Warning: Extra $var for $short tag $tagID\n";
+                        warn sprintf("Warning: Extra %s for %s tag %d (0x%.4x)\n",
+                                     $var, $short, $tagID, $tagID);
                     }
                 }
                 foreach $tagID (sort keys %$hash) {
-                    warn "Warning: Missing $var for $short $$hash{$tagID}\n";
+                    warn sprintf("Warning: Missing %s for %s %s, tag %d (0x%.4x)\n",
+                                 $var, $short, $$hash{$tagID}, $tagID, $tagID);
                 }
             }
@@ -1135 +1495 @@
     foreach $strName (keys %structs) {
         my $struct = $structLookup{$strName};
-        my $info = $tagNameInfo{"XMP $strName Struct"} = [ ];
+        my $fullName = ($strName =~ / / ? '' : 'XMP ') . "$strName Struct";
+        my $info = $tagNameInfo{$fullName} = [ ];
         my $tag;
         foreach $tag (sort keys %$struct) {
             my $tagInfo = $$struct{$tag};
-            next unless ref $tagInfo eq 'HASH';
+            next unless ref $tagInfo eq 'HASH' and $tag ne 'NAMESPACE' and $tag ne 'GROUPS';
+            warn "WARNING: $strName Struct containes $tag\n" if $Image::ExifTool::specialTags{$tag};
             my $writable = $$tagInfo{Writable};
             my @vals;
@@ -1154 +1516 @@
             }
             $writable .= '+' if $$tagInfo{List};
+            push @vals, "($$tagInfo{Notes})" if $$tagInfo{Notes};
+            # handle PrintConv lookups in Structure elements
+            my $printConv = $$tagInfo{PrintConv};
+            if (ref $printConv eq 'HASH') {
+                foreach (sort keys %$printConv) {
+                    next if /^(OTHER|BITMASK)$/;
+                    push @vals, "$_ = $$printConv{$_}";
+                }
+            }
             push @$info, [
                 $tag,
@@ -1169 +1540 @@
 # Rewrite this file to build the lookup tables
 # Inputs: 0) BuildTagLookup object reference
-#         1) output tag lookup module name (ie. 'lib/Image/ExifTool/TagLookup.pm')
+#         1) output tag lookup module name (eg. 'lib/Image/ExifTool/TagLookup.pm')
 # Returns: true on success
 sub WriteTagLookup($$)
@@ -1210 +1581 @@
     foreach $tableName (@tableNames) {
         if ($$tableWritable{$tableName}) {
-            print OUTFILE "\t'$tableName',\n";
+            print OUTFILE "\t'${tableName}',\n";
             $wrNum{$count} = $num++;
         }
@@ -1223 +1594 @@
         next unless $$tagLookup{$tag};
         my $n = scalar keys %{$$tagLookup{$tag}};
-        warn "Warning: $n writable '$tag' tags!\n" if $n > 1;
+        warn "Warning: $n writable '${tag}' tags!\n" if $n > 1;
     }
     print OUTFILE ");\n\n# lookup for all writable tags\nmy \%tagLookup = (\n";
     foreach $tag (sort keys %$tagLookup) {
-        print OUTFILE "\t'$tag' => { ";
+        print OUTFILE "\t'${tag}' => { ";
         my @tableNums = sort { $a <=> $b } keys %{$$tagLookup{$tag}};
         my (@entries, $tableNum);
@@ -1249 +1620 @@
                 # reference to root structure ID must come first in lookup
                 # (so we can generate the flattened tags just before we need them)
-                unshift @tagIDs, "\\'$rootID'" if $rootID;
+                unshift @tagIDs, "\\'${rootID}'" if $rootID;
                 $entry = '[' . join(',', @tagIDs) . ']';
             } elsif ($tagID =~ /^\d+$/) {
                 $entry = sprintf('0x%x',$tagID);
             } else {
-                $entry = "'$tagID'";
+                my $quot = "'";
+                # escape non-printable characters in tag ID if necessary
+                $quot = '"' if $tagID =~ s/[\x00-\x1f,\x7f-\xff]/sprintf('\\x%.2x',ord($&))/ge;
+                $entry = "$quot${tagID}$quot";
             }
             my $wrNum = $wrNum{$tableNum};
@@ -1269 +1643 @@
     foreach $tag (sort keys %$tagExists) {
         next if $$tagLookup{$tag};
-        print OUTFILE "\t'$tag' => 1,\n";
+        print OUTFILE "\t'${tag}' => 1,\n";
     }
 #
@@ -1278 +1652 @@
     print OUTFILE "my \%compositeModules = (\n";
     foreach (sort keys %$compositeModules) {
-        print OUTFILE "\t'$_' => '$$compositeModules{$_}',\n";
+        print OUTFILE "\t'${_}' => '$$compositeModules{$_}',\n";
     }
     print OUTFILE ");\n\n";
@@ -1325 +1699 @@
 
 #------------------------------------------------------------------------------
-# sort numbers first numerically, then strings alphabetically (case insensitive)
-sub NumbersFirst
+# Sort numbers first numerically, then strings alphabetically (case insensitive)
+# - two global variables are used to change the sort algorithm:
+#   $numbersFirst: -1 = put numbers after other strings
+#                   1 = put numbers before other strings
+#                   2 = put numbers first, but negative numbers last
+#   $caseInsensitive: flag set for case-insensitive sorting
+sub NumbersFirst($$)
 {
+    my ($a, $b) = @_;
     my $rtnVal;
-    my $bNum = ($b =~ /^-?[0-9]+(\.\d*)?$/);
-    if ($a =~ /^-?[0-9]+(\.\d*)?$/) {
-        $rtnVal = ($bNum ? $a <=> $b : -$numbersFirst);
-    } elsif ($bNum) {
+    my ($bNum, $bDec);
+    ($bNum, $bDec) = ($1, $3) if $b =~ /^(-?[0-9]+)(\.(\d*))?$/;
+    if ($a =~ /^(-?[0-9]+)(\.(\d*))?$/) {
+        if (defined $bNum) {
+            $bNum += 1e9 if $numbersFirst == 2 and $bNum < 0;
+            my $aInt = $1;
+            $aInt += 1e9 if $numbersFirst == 2 and $aInt < 0;
+            # compare integer part as a number
+            $rtnVal = $aInt <=> $bNum;
+            unless ($rtnVal) {
+                my $aDec = $3 || 0;
+                $bDec or $bDec = 0;
+                # compare decimal part as an integer too
+                # (so that "1.10" comes after "1.9")
+                $rtnVal = $aDec <=> $bDec;
+            }
+        } else {
+            $rtnVal = -$numbersFirst;
+        }
+    } elsif (defined $bNum) {
         $rtnVal = $numbersFirst;
     } else {
         my ($a2, $b2) = ($a, $b);
         # expand numbers to 3 digits (with restrictions to avoid messing up ascii-hex tags)
-        $a2 =~ s/(\d+)/sprintf("%.3d",$1)/eg if $a2 =~ /^(APP)?[.0-9 ]*$/ and length($a2)<16;
-        $b2 =~ s/(\d+)/sprintf("%.3d",$1)/eg if $b2 =~ /^(APP)?[.0-9 ]*$/ and length($b2)<16;
+        $a2 =~ s/(\d+)/sprintf("%.3d",$1)/eg if $a2 =~ /^(APP|DMC-\w+ )?[.0-9 ]*$/ and length($a2)<16;
+        $b2 =~ s/(\d+)/sprintf("%.3d",$1)/eg if $b2 =~ /^(APP|DMC-\w+ )?[.0-9 ]*$/ and length($b2)<16;
         $caseInsensitive and $rtnVal = (lc($a2) cmp lc($b2));
         $rtnVal or $rtnVal = ($a2 cmp $b2);
@@ -1346 +1742 @@
 
 #------------------------------------------------------------------------------
-# Convert pod documentation to pod
+# Convert our pod-like documentation to pod
 # (funny, I know, but the pod headings must be hidden to prevent confusing
 #  the pod parser)
@@ -1357 +1753 @@
     $doc =~ s/\n~/\n=/g;
     $doc =~ s/L<[^>]+?\|(http[^>]+)>/L<$1>/g; # POD doesn't support text for http links
+    $doc =~ s/L<([^>]+?)\|[^>]+\.html(#\w+)?>/$1/g;  # remove relative HTML links
     return $doc;
 }
 
 #------------------------------------------------------------------------------
-# Convert pod documentation to html
+# Convert our pod-like documentation to html
 # Inputs: 0) string
 sub Doc2Html($)
@@ -1370 +1767 @@
     $doc =~ s/C&lt;(.*?)&gt;/<code>$1<\/code>/sg;
     $doc =~ s/I&lt;(.*?)&gt;/<i>$1<\/i>/sg;
-    $doc =~ s{L&lt;([^&]+?)\|\Q$homePage\E/TagNames/(.*?)&gt;}{<a href="$2">$1<\/a>}sg;
+    # L<some text|https://exiftool.org/struct.html#Fields> --> <a href="../struct.html#Fields">some text</a>
     $doc =~ s{L&lt;([^&]+?)\|\Q$homePage\E/(.*?)&gt;}{<a href="../$2">$1<\/a>}sg;
-    $doc =~ s{L&lt;\Q$homePage\E/TagNames/(.*?)&gt;}{<a href="$1">$1<\/a>}sg;
+    # L<https://exiftool.org/struct.html> --> <a href="https://exiftool.org/struct.html">https://exiftool.org/struct.html</a>
     $doc =~ s{L&lt;\Q$homePage\E/(.*?)&gt;}{<a href="../$1">$1<\/a>}sg;
+    # L<XMP DICOM Tags|Image::ExifTool::TagNames/XMP DICOM Tags> --> <a href="XMP.html#DICOM">XMP DICOM Tags</a>
+    # (specify "Image::ExifTool::TagNames" to link to another html file)
+    $doc =~ s{L&lt;([^&]+?)\|Image::ExifTool::TagNames/(\w+) ([^/&|]+) Tags&gt;}{<a href="$2.html#$3">$1</a>}sg;
+    # L<DICOM Tags|Image::ExifTool::TagNames/DICOM Tags> --> <a href="DICOM.html">DICOM Tags</a>
+    $doc =~ s{L&lt;([^&]+?)\|Image::ExifTool::TagNames/(\w+) Tags&gt;}{<a href="$2.html">$1</a>}sg;
+    # L<dc|/XMP dc Tags> --> <a href="#dc">dc</a>
+    # (a relative POD link turns into a relative HTML link)
     $doc =~ s{L&lt;([^&]+?)\|/\w+ ([^/&|]+) Tags&gt;}{<a href="#$2">$1</a>}sg;
+    # L<sample config file|../config.html> --> <a href="../config.html">sample config file</a>
     $doc =~ s/L&lt;([^&]+?)\|(.+?)&gt;/<a href="$2">$1<\/a>/sg;
-    $doc =~ s/L&lt;(.*?)&gt;/<a href="$1">$1<\/a>/sg;
+    # L<http://some.web.site/> --> <a href="http://some.web.site">http://some.web.site</a>
+    $doc =~ s/L&lt;(http.*?)&gt;/<a href="$1">$1<\/a>/sg;
     return $doc;
 }
 
 #------------------------------------------------------------------------------
+# Tweak order of tables
+# Inputs: 0) table list ref, 1) reference to tweak hash
+sub TweakOrder($$)
+{
+    local $_;
+    my ($sortedTables, $tweakOrder) = @_;
+    my @tweak = sort keys %$tweakOrder;
+    my (%addedMain, @sorted);
+    # flag files which have a "Main" table
+    foreach (@$sortedTables) {
+        $addedMain{$1} = 0 if /^Image::ExifTool::(\w+)::(\w+)/ and $2 eq 'Main';
+    }
+    # make sure that the main table always comes first in each file
+    foreach (@$sortedTables) {
+        if (/^Image::ExifTool::(\w+)::(\w+)/) {
+            if ($addedMain{$1}) {
+                next if $2 eq 'Main';   # don't add again
+            } elsif (defined $addedMain{$1}) {
+                push @sorted, "Image::ExifTool::${1}::Main" if $2 ne 'Main';
+                $addedMain{$1} = 1;
+            }
+        }
+        push @sorted, $_;
+    }
+    @$sortedTables = @sorted;
+    # apply manual tweaks
+    while (@tweak) {
+        my $table = shift @tweak;
+        my $first = $$tweakOrder{$table};
+        if ($$tweakOrder{$first}) {
+            push @tweak, $table;    # must defer this till later
+            next;
+        }
+        delete $$tweakOrder{$table}; # because the table won't move again
+        my @moving = grep /^Image::ExifTool::$table\b/, @$sortedTables;
+        my @notMoving = grep !/^Image::ExifTool::$table\b/, @$sortedTables;
+        my @after;
+        while (@notMoving) {
+            last if $notMoving[-1] =~ /^Image::ExifTool::$first\b/;
+            unshift @after, pop @notMoving;
+        }
+        @$sortedTables = (@notMoving, @moving, @after);
+    }
+}
+
+#------------------------------------------------------------------------------
+# Get a list of sorted tag ID's from a table
+# Inputs: 0) tag table ref
+# Returns: list of sorted keys
+sub SortedTagTableKeys($)
+{
+    my $table = shift;
+    my $vars = $$table{VARS} || { };
+    my @keys = TagTableKeys($table);
+    if ($$vars{NO_ID}) {
+        # sort by tag name if ID not shown
+        my ($key, %name);
+        foreach $key (@keys) {
+            my ($tagInfo) = GetTagInfoList($table, $key);
+            $name{$key} = $$tagInfo{Name};
+        }
+        return sort { $name{$a} cmp $name{$b} or $a cmp $b } @keys;
+    } else {
+        my $sortProc = $$vars{SORT_PROC} || \&NumbersFirst;
+        return sort { &$sortProc($a,$b) } @keys;
+    }
+}
+
+#------------------------------------------------------------------------------
 # Get the order that we want to print the tables in the documentation
+# Inputs: 0-N) Extra tables to add at end
 # Returns: tables in the order we want
-sub GetTableOrder()
+sub GetTableOrder(@)
 {
     my %gotTable;
@@ -1406 +1882 @@
         $caseInsensitive = ($$table{GROUPS} and $$table{GROUPS}{0} eq 'XMP');
         $numbersFirst = -1 if $$table{VARS} and $$table{VARS}{ALPHA_FIRST};
-        my @keys = sort NumbersFirst TagTableKeys($table);
+        my @keys = SortedTagTableKeys($table);
         $numbersFirst = 1;
         foreach (@keys) {
@@ -1450 +1926 @@
         }
     }
+    # add the extra tables
+    push @sortedTables, @_;
     # tweak the table order
-    my %tweakOrder = (
-        JPEG    => '-',     # JPEG comes first
-        IPTC    => 'Exif',  # put IPTC after EXIF,
-        GPS     => 'XMP',   # etc...
-        GeoTiff => 'GPS',
-        CanonVRD=> 'CanonCustom',
-        Kodak   => 'JVC',
-       'Kodak::IFD' => 'Kodak::Unknown',
-       'Kodak::TextualInfo' => 'Kodak::IFD',
-       'Kodak::Processing' => 'Kodak::TextualInfo',
-        Leaf    => 'Kodak',
-        Minolta => 'Leaf',
-        SonyIDC => 'Sony',
-        Unknown => 'SonyIDC',
-        DNG     => 'Unknown',
-        PrintIM => 'ICC_Profile',
-        ID3     => 'PostScript',
-        MinoltaRaw => 'KyoceraRaw',
-        KyoceraRaw => 'CanonRaw',
-        SigmaRaw => 'PanasonicRaw',
-        Olympus => 'NikonCapture',
-        PhotoMechanic => 'FotoStation',
-        Microsoft     => 'PhotoMechanic',
-       'Microsoft::MP'=> 'Microsoft::MP1',
-        GIMP    => 'Microsoft',
-       'Nikon::CameraSettingsD300' => 'Nikon::ShotInfoD300b',
-       'Pentax::LensData' => 'Pentax::LensInfo2',
-       'Sony::SRF2' => 'Sony::SRF',
-       'Samsung::INFO' => 'Samsung::Type2', # (necessary because Samsung doesn't have a main table)
-       'Samsung::MP4' => 'Samsung::INFO', # (necessary because Samsung doesn't have a main table)
-    );
-    my @tweak = sort keys %tweakOrder;
-    while (@tweak) {
-        my $table = shift @tweak;
-        my $first = $tweakOrder{$table};
-        if ($tweakOrder{$first}) {
-            push @tweak, $table;    # must defer this till later
-            next;
-        }
-        delete $tweakOrder{$table}; # because the table won't move again
-        my @moving = grep /^Image::ExifTool::$table\b/, @sortedTables;
-        my @notMoving = grep !/^Image::ExifTool::$table\b/, @sortedTables;
-        my @after;
-        while (@notMoving) {
-            last if $notMoving[-1] =~ /^Image::ExifTool::$first\b/;
-            unshift @after, pop @notMoving;
-        }
-        @sortedTables = (@notMoving, @moving, @after);
-    }
-    return @sortedTables
+    TweakOrder(\@sortedTables, \%tweakOrder);
+
+    return @sortedTables;
 }
 
@@ -1519 +1950 @@
         $htmlFile = "$htmldir/TagNames/$class.html";
         $head = $category;
-        if ($head =~ /^XMP .+ Struct$/) {
+        if ($head =~ /^\S+ .+ Struct$/) {
             pop @names;
         } else {
@@ -1535 +1966 @@
     } else {
         open(HTMLFILE, ">${htmlFile}_tmp") or return 0;
-        print HTMLFILE "$docType<html>\n<head>\n<title>$title</title>\n";
+        print HTMLFILE "$docType<html>\n";
+        print HTMLFILE "<!-- (this file generated automatically by Image::ExifTool::BuildTagLookup) -->\n";
+        print HTMLFILE "<head>\n<title>$title</title>\n";
         print HTMLFILE "<link rel=stylesheet type='text/css' href='style.css' title='Style'>\n";
         print HTMLFILE "</head>\n<body>\n";
@@ -1545 +1978 @@
         }
     }
-    $head = "<a name='$url'>$head</a>" if $url;
+    $head = "<a name='${url}'>$head</a>" if $url;
     print HTMLFILE "<h2$top>$head</h2>\n" or return 0;
     print HTMLFILE '<p>',Doc2Html($docs{$category}),"</p>\n" if $docs{$category};
@@ -1580 +2013 @@
         print HTMLFILE "<hr>\n";
         print HTMLFILE "(This document generated automatically by Image::ExifTool::BuildTagLookup)\n";
+        print HTMLFILE "<br><i>Created $createDate</i>\n" if $htmlFile eq 'html/TagNames/index.html';
         print HTMLFILE "<br><i>Last revised $fileDate</i>\n";
         print HTMLFILE "<p class=lf><a href=";
@@ -1628 +2062 @@
 
 #------------------------------------------------------------------------------
+# Get bitmask for POD documentation
+# Inputs: mask string from HTML docs
+# Returns: mask string for POD, or ''
+sub PodMask($)
+{
+    my $mask = shift;
+    return '' unless $mask =~ /^\[val( >> (\d+))? \& (0x[\da-f]+)\]/;
+    return sprintf(' & 0x%.2x', hex($3) << ($2 || 0));
+}
+
+#------------------------------------------------------------------------------
 # Write the TagName HTML and POD documentation
 # Inputs: 0) BuildTagLookup object reference
-#         1) output pod file (ie. 'lib/Image/ExifTool/TagNames.pod')
-#         2) output html directory (ie. 'html')
+#         1) output pod file (eg. 'lib/Image/ExifTool/TagNames.pod')
+#         2) output html directory (eg. 'html')
 # Returns: true on success
 # Notes: My apologies for the patchwork code, but this is only used to generate the docs.
@@ -1650 +2095 @@
     # open the file and write the header
     open(PODFILE, ">$podFile") or return 0;
+    $docs{ExifTool} = eval qq{"$docs{ExifTool}"};
     print PODFILE Doc2Pod($docs{PodHeader}, $docs{ExifTool}, $docs{ExifTool2});
-    mkdir "$htmldir/TagNames";
+    mkdir "$htmldir/TagNames", 0777;
     OpenHtmlFile($htmldir) or return 0;
     print HTMLFILE "<blockquote>\n";
@@ -1658 +2104 @@
     print HTMLFILE "<th colspan=$columns><span class=l>Tag Table Index</span></th></tr>\n";
     print HTMLFILE "<tr class=b><td width='$percent%'>\n";
-    # write the index
-    my @tableNames = GetTableOrder();
-    # add shortcuts last
-    push @tableNames, 'Image::ExifTool::Shortcuts::Main';
-    push @tableNames, @pluginTables;
+
+    # get the full table list, adding shortcuts and plug-in tables to the end
+    my @tableNames = GetTableOrder('Image::ExifTool::Shortcuts::Main', @pluginTables);
+
     # get list of headings and add any missing ones
     my $heading = 'xxx';
@@ -1698 +2143 @@
         $short = $tableName unless $short;
         $url = "$short.html";
-        print HTMLFILE "<a href='$url'>$short</a>";
+        print HTMLFILE "<a href='${url}'>$short</a>";
         ++$count;
     }
     print HTMLFILE "\n</td></tr></table></td></tr></table></blockquote>\n";
     print HTMLFILE '<p>',Doc2Html($docs{ExifTool2}),"</p>\n";
+
     # write all the tag tables
     while (@tableNames or @sepTables or @structs) {
@@ -1713 +2159 @@
             if ($notes) {
                 # remove unnecessary whitespace
-                $notes =~ s/(^\s+|\s+$)//g;
+                $notes =~ s/^\s+//; $notes =~ s/\s+$//;
                 $notes =~ s/(^[ \t]+|[ \t]+$)//mg;
             }
             my $head = $tableName;
-            $head =~ s/.* //;
+            $head =~ s/^.* //s;
             close HTMLFILE;
             if (OpenHtmlFile($htmldir, $tableName, 1)) {
@@ -1727 +2173 @@
                 my $wid = 0;
                 my @keys;
-                foreach (sort NumbersFirst keys %$printConv) {
+                foreach (sort { NumbersFirst($a,$b) } keys %$printConv) {
                     next if /^(Notes|PrintHex|PrintString|OTHER)$/;
                     $align = '' if $align and /[^\d]/;
@@ -1733 +2179 @@
                     $wid = $w if $wid < $w;
                     push @keys, $_;
+                    if ($$printConv{$_} =~ /[\0-\x1f\x7f-\xff]/) {
+                        warn "Warning: Special characters in $tableName PrintConv ($$printConv{$_})\n";
+                    }
                 }
                 $wid = length($tableName)+7 if $wid < length($tableName)+7;
                 # print in multiple columns if there is room
-                my $cols = int(80 / ($wid + 4));
+                my $cols = int(110 / ($wid + 4));
                 $cols = 1 if $cols < 1 or $cols > @keys or @keys < 4;
                 my $rows = int((scalar(@keys) + $cols - 1) / $cols);
@@ -1754 +2203 @@
                             $prt = '= ' . EscapeHTML($$printConv{$key});
                             if ($$printConv{PrintHex}) {
+                                $index =~ s/(\.\d+)$//; # remove decimal
                                 $index = sprintf('0x%x',$index);
+                                $index .= $1 if $1; # add back decimal
                             } elsif ($$printConv{PrintString} or
                                 $index !~ /^[+-]?(?=\d|\.\d)\d*(\.\d*)?$/)
@@ -1786 +2237 @@
             $isStruct = $$structLookup{$tableName};
             $isStruct or warn("Missing structure $tableName\n"), next;
-            $short = $tableName = "XMP $tableName Struct";
+            $tableName = "XMP $tableName" unless $tableName =~ / /;
+            $short = $tableName = "$tableName Struct";
             my $maxLen = 0;
             $maxLen < length and $maxLen = length foreach keys %$isStruct;
@@ -1800 +2252 @@
             }
         }
-        my $isExif = $tableName eq 'Image::ExifTool::Exif::Main' ? 1 : undef;
-        my $isRiff = $tableName eq 'Image::ExifTool::RIFF::Info' ? 1 : undef;
+        my $isExif = ($tableName eq 'Image::ExifTool::Exif::Main');
+        my $isRiff = ($tableName eq 'Image::ExifTool::RIFF::Info');
+        my $isXmpMain = ($tableName eq 'Image::ExifTool::XMP::Main');
         my $info = $$tagNameInfo{$tableName};
         my $id = $$idLabel{$tableName};
@@ -1807 +2260 @@
         # widths of the different columns in the POD documentation
         my ($wID,$wTag,$wReq,$wGrp) = (8,36,24,10);
-        my ($composite, $derived, $notes, $prefix);
+        my ($composite, $derived, $notes, $longTags, $wasLong, $prefix);
         if ($short eq 'Shortcuts') {
             $derived = '<th>Refers To</th>';
@@ -1817 +2270 @@
             my $table = GetTagTable($tableName);
             $notes = $$table{NOTES};
+            if ($$table{NAMESPACE}) {
+                my $ns = $Image::ExifTool::XMP::stdXlatNS{$$table{NAMESPACE}} || $$table{NAMESPACE};
+                my $msg = "These tags belong to the ExifTool XMP-$ns family 1 group.";
+                if ($notes) {
+                    $notes =~ s/\s+$//;
+                    $notes .= "\n\n" . $msg;
+                } else {
+                    $notes = $msg;
+                }
+            }
+            $longTags = $$table{VARS}{LONG_TAGS} if $$table{VARS};
             if ($$table{GROUPS}{0} eq 'Composite') {
                 $composite = 1;
@@ -1828 +2292 @@
         if ($notes) {
             # remove unnecessary whitespace
-            $notes =~ s/(^\s+|\s+$)//g;
+            $notes =~ s/^\s+//; $notes =~ s/\s+$//;
             $notes =~ s/(^[ \t]+|[ \t]+$)//mg;
             if ($notes =~ /leading '(.*?_)' which/) {
@@ -1850 +2314 @@
             my $longTag = $self->{LONG_NAME}->{$tableName};
             if ($wTag < $longTag) {
-                warn "Notice: Long tags in $tableName table\n";
                 if ($wID - $longTag + $wTag >= 6) { # don't let ID column get too narrow
                     $wID -= $longTag - $wTag;
                     $wTag = $longTag;
                 }
+                $wasLong = 1 if $wID <= $self->{LONG_ID}->{$tableName};
             }
         } elsif ($composite) {
@@ -1863 +2327 @@
             $hid = '';
         }
-        if ($short eq 'EXIF') {
+        if ($short eq 'EXIF' or $short eq 'Extra') {
             $derived = '<th>Group</th>';
             $showGrp = 1;
+            $wGrp += 8 if $short eq 'Extra';    # tweak Group column width for "Extra" tags
             $wTag -= $wGrp + 1;
         }
@@ -1887 +2352 @@
         $line .= ' Writable';
         print PODFILE $line;
-        $line =~ s/^(\s*\w.{6}\w) /$1\t/;   # change space to tab after long ID label (ie. "Sequence")
+        $line =~ s/^(\s*\w.{6}\w) /$1\t/;   # change space to tab after long ID label (eg. "Sequence")
         $line =~ s/\S/-/g;
         $line =~ s/- -/---/g;
@@ -1910 +2375 @@
             if (not $id) {
                 $idStr = '  ';
-            } elsif ($tagIDstr =~ /^\d+(\.\d+)?$/) {
+            } elsif ($tagIDstr =~ /^-?\d+(\.\d+)?$/) {
                 $w = $wID - 3;
                 $idStr = sprintf "  %${w}g    ", $tagIDstr;
+                my $tooLong = length($idStr) - 6 - $w;
+                if ($tooLong) {
+                    $tooLong = 3 if $tooLong > 3;
+                    $idStr = substr($idStr, 0, -$tooLong);
+                }
                 $align = " class=r";
             } else {
@@ -1926 +2396 @@
                         # put tag name on next line if ID is too long
                         $idStr = "  $tagIDstr\n   " . (' ' x $w);
-                        warn "Notice: Split $$tagNames[0] line\n";
+                        if ($longTags) {
+                            --$longTags;
+                        } else {
+                            warn "Notice: Split $$tagNames[0] line\n";
+                        }
                     }
                 }
@@ -1938 +2412 @@
             my $wrStr = shift @vals;
             my $subdir;
-            my @masks = grep /^\[Mask 0x[\da-f]+\]/, @$values;
+            my @masks = grep /^\[val( >> \d+)? \& 0x[\da-f]+\]/, @$values;
             my $tag = shift @tags;
             # if this is a subdirectory or structure, print subdir name (from values) instead of writable
@@ -1945 +2419 @@
                 if (@masks) {
                     # combine any mask into the format string
-                    $wrStr .= " & $1" if $masks[0] =~ /(0x[\da-f]+)/;
+                    $wrStr .= PodMask($masks[0]);
                     shift @masks;
-                    @vals = grep !/^\[Mask 0x[\da-f]+\]/, @$values;
+                    @vals = grep !/^\[val( >> \d+)? \& 0x[\da-f]+\]/, @$values;
                 } else {
                     @vals = @$values;
@@ -1958 +2432 @@
                     $vals[$i] = $$writable[$i] unless defined $vals[$i];
                     if (@masks) {
-                        $vals[$i] .= " & $1" if $masks[0] =~ /(0x[\da-f]+)/;
+                        $vals[$i] .= PodMask($masks[0]);
                         shift @masks;
                     }
@@ -1964 +2438 @@
                 if ($$sepTable{$vals[0]}) {
                     $wrStr =~ s/^[-=]//;
-                    $wrStr = 'N' unless $wrStr;
+                    $wrStr = 'no' unless $wrStr;
                 } elsif ($$structs{$vals[0]}) {
                     my $flags = $wrStr =~ /([+_]+)$/ ? $1 : '';
@@ -1972 +2446 @@
                 }
                 shift @vals;
-            } elsif ($wrStr and $wrStr ne 'N' and @masks) {
+            } elsif ($wrStr and $wrStr ne 'no' and @masks) {
                 # fill in missing entries if masks are different
                 my $mask = shift @masks;
@@ -1981 +2455 @@
                 }
                 # add Mask to Writable column in POD doc
-                $wrStr .= " & $1" if $mask =~ /(0x[\da-f]+)/;
-            }
-            printf PODFILE "%s%-${wTag2}s", $idStr, $tag;
-            warn "Warning: Pushed $tag\n" if $id and length($tag) > $wTag2;
-            printf PODFILE " %-${wGrp}s", shift(@wGrp) || '-' if $showGrp;
+                $wrStr .= PodMask($mask);
+            }
+            my $pod = sprintf "%s%-${wTag2}s", $idStr, $tag;
+            my $tGrp = $wGrp;
+            if ($id and length($tag) > $wTag2) {
+                my $madeRoom;
+                if ($showGrp) {
+                    my $wGrp0 = length($wGrp[0] || '-');
+                    if (not $composite and $wGrp > $wGrp0) {
+                        $tGrp = $wGrp - (length($tag) - $wTag2);
+                        if ($tGrp < length $wGrp0) {
+                            $tGrp = length $wGrp0;
+                        } else {
+                            $madeRoom = 1;
+                        }
+                    }
+                }
+                warn "Warning: Pushed $tag\n" unless $madeRoom;
+            }
+            $pod .= sprintf " %-${tGrp}s", shift(@wGrp) || '-' if $showGrp;
             if ($composite) {
                 @reqs = @$require;
                 $w = $wReq; # Keep writable column in line
                 length($tag) > $wTag2 and $w -= length($tag) - $wTag2;
-                printf PODFILE " %-${w}s", shift(@reqs) || '';
-            }
-            printf PODFILE " $wrStr\n";
+                $pod .= sprintf " %-${w}s", shift(@reqs) || '';
+            }
+            $pod .= " $wrStr";
+            # limit line length to 82 characters (even if it means messing up column alignment)
+            if (length $pod > 82 and $pod !~ /\n/) {
+                my $remove = length($pod) - 82;
+                $pod =~ s/(\w)\s{$remove}/$1/;
+            }
+            print PODFILE $pod, "\n";
             my $numTags = scalar @$tagNames;
             my $n = 0;
@@ -2019 +2514 @@
                         $line .= " $val";
                         if (@masks) {
-                            $line .= " & $1" if $masks[0] =~ /(0x[\da-f]+)/;
+                            $line .= PodMask($masks[0]);
                             shift @masks;
                         }
@@ -2031 +2526 @@
                 push @htmlTags, EscapeHTML($_);
             }
-            if (($isExif and $exifSpec{hex $tagIDstr}) or
-                ($isRiff and $tagIDstr=~/(\w+)/ and $riffSpec{$1}))
+            if (($isExif and $Image::ExifTool::Validate::exifSpec{hex $tagIDstr}) or
+                ($isRiff and $tagIDstr=~/(\w+)/ and $riffSpec{$1}) or
+                ($isXmpMain and $tagIDstr=~/([-\w]+)/ and $xmpSpec{$1}))
             {
                 # underline "unknown" makernote tags only
@@ -2045 +2541 @@
             }
             # add tooltip for hex conversion of Tag ID
-            if ($tagIDstr =~ /^0x[0-9a-f]+$/i) {
-                $tip = sprintf(" title='$tagIDstr = %u'",hex $tagIDstr);
+            if ($tagIDstr =~ /^(0x[0-9a-f]+)(\.\d+)?$/i) {
+                $tip = sprintf(" title='$tagIDstr = %u%s'", hex($1), $2||'');
             } elsif ($tagIDstr =~ /^(\d+)(\.\d*)?$/) {
                 $tip = sprintf(" title='%u = 0x%x'", $1, $1);
@@ -2059 +2555 @@
                 '_' => 'Flattened',
                 '+' => 'List',
-                '/' => 'Avoided',
+                '/' => 'Avoid',
                 '~' => 'Writable only with -n',
                 '!' => 'Unsafe',
                 '*' => 'Protected',
                 ':' => 'Mandatory',
+                '^' => 'Deletable',
             );
             my ($wstr, %hasAttr, @hasAttr);
             foreach $wstr (@$writable) {
-                next unless $wstr =~ m{([+/~!*:_]+)$};
+                next unless $wstr =~ m{([+/~!*:_^]+)$};
                 my @a = split //, $1;
                 foreach (@a) {
@@ -2098 +2595 @@
                     foreach (@$values) {
                         if (/^\(/) {
+                            $_ = Doc2Html($_);
                             # set the note font
                             $smallNote = 1 if $numTags < 2;
@@ -2104 +2602 @@
                         }
                         # make text in square brackets small
-                        /^\[/ and push(@values, "<span class=s>$_</span>"), next;
+                        if (/^\[/) {
+                            if (s/^\[!HTML\]//) {
+                                push @values, $_;
+                            } else {
+                                $_ = Doc2Html($_);
+                                push @values, "<span class=s>$_</span>";
+                            }
+                            next;
+                        }
                         /=/ and push(@values, $_), next;
                         my @names = split;
@@ -2112 +2618 @@
                             $suffix = ' Values';
                         }
-                        # currently all structures are in XMP documentation
-                        if ($$structs{$_} and $short =~ /^XMP/) {
-                            unshift @names, 'XMP';
+                        # handle structures specially
+                        if ($$structs{$_}) {
+                            # assume XMP module for this struct unless otherwise specified
+                            unshift @names, 'XMP' unless / /;
                             push @structs, $_;  # list this later
+                            # hack to put Area Struct in with XMP tags,
+                            # even though it is only used by the MWG module
+                            push @structs, 'Area' if $_ eq 'Dimensions';
                             $suffix = ' Struct';
                         }
                         $url = (shift @names) . '.html';
                         @names and $url .= '#' . join '_', @names;
-                        push @values, "--&gt; <a href='$url'>$_$suffix</a>";
+                        push @values, "--&gt; <a href='${url}'>$_$suffix</a>";
                     }
                     # put small note last
@@ -2151 +2661 @@
             print HTMLFILE "</td></tr>\n";
         }
+        warn "$longTags unaccounted-for long tags in $tableName\n" if $longTags;
+        if ($wasLong and not defined $longTags) {
+            warn "Notice: Long tags in $tableName table\n";
+        }
         unless ($infoCount) {
-            printf PODFILE "  [no tags known]\n";
+            print PODFILE "  [no tags known]\n";
             my $cols = 3;
             ++$cols if $hid;
@@ -2182 +2696 @@
 Image::ExifTool::TagNames.pod, as well as HTML tag name documentation.  It
 is used before each new ExifTool release to update the lookup tables and
-documentation, but it is not used otherwise.
+documentation, but it is not used otherwise.  It also performs some
+validation and consistency checks on the tag tables.
 
 =head1 SYNOPSIS
@@ -2190 +2705 @@
   $builder = new Image::ExifTool::BuildTagLookup;
 
+  # update Image::ExifTool::TagLookup
   $ok = $builder->WriteTagLookup('lib/Image/ExifTool/TagLookup.pm');
 
+  # update the tag name documentation
   $ok = $builder->WriteTagNames('lib/Image/ExifTool/TagNames.pod','html');
+
+  # print some statistics
+  my $count = $$builder{COUNT};
+  foreach (sort keys %$count) {
+      printf "%5d %s\n", $$count{$_}, $_;
+  }
 
 =head1 MEMBER VARIABLES
@@ -2210 +2733 @@
 WriteTagNames().
 
+=item WRITE_PSEUDO
+
+Returned list of writable pseudo tags.
+
 =back
 
 =head1 AUTHOR
 
-Copyright 2003-2011, Phil Harvey (phil at owl.phy.queensu.ca)
+Copyright 2003-2021, Phil Harvey (philharvey66 at gmail.com)
 
 This library is free software; you can redistribute it and/or modify it