Changeset 34921 for main/trunk/greenstone2/perllib/cpan/Image/ExifTool.pm

Timestamp:

2021-02-26T19:39:51+13:00 (3 years ago)

Author:

anupama

Message:

Committing the improvements to EmbeddedMetaPlugin's processing of Keywords vs other metadata fields. Keywords were literally stored as arrays of words rather than phrases in PDFs (at least in Diego's sample PDF), whereas other meta fields like Subjects and Creators stored them as arrays of phrases. To get both to work, Kathy updated EXIF to a newer version, to retrieve the actual EXIF values stored in the PDF. And Kathy and Dr Bainbridge came up with a new option that I added called apply_join_before_split_to_metafields that's a regex which can list the metadata fields to apply the join_before_split to and whcih previously always got applied to all metadata fields. Now it's applied to any *Keywords metafields by default, as that's the metafield we have experience of that behaves differently to the others, as it stores by word instead of phrases. Tested on Diego's sample PDF. Diego has double-checked it to works on his sample PDF too, setting the split char to ; and turning on the join_before_split and leaving apply_join_before_split_to_metafields at its default of .*Keywords. File changes are strings.properties for the tooltip, the plugin introducing the option and working with it and Kathy's EXIF updates affecting cpan/File and cpan/Image.

File:

• main/trunk/greenstone2/perllib/cpan/Image/ExifTool.pm (modified) (298 diffs)

main/trunk/greenstone2/perllib/cpan/Image/ExifTool.pm

@@ -4 +4 @@
 # Description:  Read and write meta information
 #
-# URL:          http://owl.phy.queensu.ca/~phil/exiftool/
+# URL:          https://exiftool.org/
 #
 # Revisions:    Nov. 12/2003 - P. Harvey Created
 #               (See html/history.html for revision history)
 #
-# Legal:        Copyright (c) 2003-2010, Phil Harvey (phil at owl.phy.queensu.ca)
+# Legal:        Copyright (c) 2003-2021, Phil Harvey (philharvey66 at gmail.com)
 #               This library is free software; you can redistribute it and/or
 #               modify it under the same terms as Perl itself.
@@ -20 +20 @@
 require Exporter;
 use File::RandomAccess;
-
-use vars qw($VERSION $RELEASE @ISA %EXPORT_TAGS $AUTOLOAD @fileTypes %allTables
-            @tableOrder $exifAPP1hdr $xmpAPP1hdr $xmpExtAPP1hdr $psAPP13hdr
-            $psAPP13old @loadAllTables %UserDefined $evalWarning %noWriteFile
-            %magicNumber @langs $defaultLang %langName %charsetName %mimeType
-            $swapBytes $swapWords $currentByteOrder %unpackStd);
-
-$VERSION = '8.57';
+use overload;
+
+use vars qw($VERSION $RELEASE @ISA @EXPORT_OK %EXPORT_TAGS $AUTOLOAD @fileTypes
+            %allTables @tableOrder $exifAPP1hdr $xmpAPP1hdr $xmpExtAPP1hdr
+            $psAPP13hdr $psAPP13old @loadAllTables %UserDefined $evalWarning
+            %noWriteFile %magicNumber @langs $defaultLang %langName %charsetName
+            %mimeType $swapBytes $swapWords $currentByteOrder %unpackStd
+            %jpegMarker %specialTags %fileTypeLookup $testLen $exePath);
+
+$VERSION = '12.19';
 $RELEASE = '';
 @ISA = qw(Exporter);
@@ -35 +37 @@
         ImageInfo GetTagName GetShortcuts GetAllTags GetWritableTags
         GetAllGroups GetDeleteGroups GetFileType CanWrite CanCreate
+        AddUserDefinedTags
     )],
     # exports not part of the public API, but used by ExifTool modules:
@@ -40 +43 @@
         ReadValue GetByteOrder SetByteOrder ToggleByteOrder Get8u Get8s Get16u
         Get16s Get32u Get32s Get64u GetFloat GetDouble GetFixed32s Write
-        WriteValue Tell Set8u Set8s Set16u Set32u
+        WriteValue Tell Set8u Set8s Set16u Set32u Set64u
     )],
-    Utils => [qw(GetTagTable TagTableKeys GetTagInfoList)],
+    Utils => [qw(GetTagTable TagTableKeys GetTagInfoList AddTagToTable HexDump)],
     Vars  => [qw(%allTables @tableOrder @fileTypes)],
 );
+
 # set all of our EXPORT_TAGS in EXPORT_OK
 Exporter::export_ok_tags(keys %EXPORT_TAGS);
@@ -51 +55 @@
 { my $t = "\xff"; die "Incompatible encoding!\n" if ord($t) != 0xff; }
 
-# The following functions defined in Image::ExifTool::Writer are declared
+# The following functions defined in Image::ExifTool::Writer.pl are declared
 # here so their prototypes will be available.  These Writer routines will be
 # autoloaded when any of them is called.
 sub SetNewValue($;$$%);
 sub SetNewValuesFromFile($$;@);
-sub GetNewValues($;$$);
+sub GetNewValue($$;$);
+sub GetNewValues($$;$);
 sub CountNewValues($);
 sub SaveNewValues($);
 sub RestoreNewValues($);
 sub WriteInfo($$;$$);
-sub SetFileModifyDate($$;$);
-sub SetFileName($$;$);
+sub SetFileModifyDate($$;$$$);
+sub SetFileName($$;$$$);
+sub SetSystemTags($$);
 sub GetAllTags(;$);
 sub GetWritableTags(;$);
-sub GetAllGroups($);
+sub GetAllGroups($;$);
 sub GetNewGroups($);
 sub GetDeleteGroups();
+sub AddUserDefinedTags($%);
 # non-public routines below
-sub InsertTagValues($$$;$);
+sub InsertTagValues($$$;$$$);
 sub IsWritable($);
+sub IsSameFile($$$);
+sub IsRawType($);
 sub GetNewFileName($$);
-sub NextTagKey($$);
 sub LoadAllTables();
 sub GetNewTagInfoList($;$);
@@ -79 +87 @@
 sub Get64s($$);
 sub Get64u($$);
+sub GetFixed64s($$);
 sub GetExtended($$);
+sub Set64u(@);
 sub DecodeBits($$;$);
 sub EncodeBits($$;$$);
+sub Filter($$$);
 sub HexDump($;$%);
 sub DumpTrailer($$);
 sub DumpUnknownTrailer($$);
 sub VerboseInfo($$$%);
-sub VerboseDir($$;$$);
 sub VerboseValue($$$;$);
 sub VPrint($$@);
@@ -101 +111 @@
 sub PackUTF8(@);
 sub UnpackUTF8($);
-sub SetPreferredByteOrder($);
+sub SetPreferredByteOrder($;$);
 sub CopyBlock($$$);
-sub CopyFileAttrs($$);
+sub CopyFileAttrs($$$);
+sub TimeNow(;$$);
+sub NewGUID();
+sub MakeTiffHeader($$$$;$$);
 
 # other subroutine definitions
+sub SplitFileName($);
+sub EncodeFileName($$;$);
+sub Open($*$;$);
+sub Exists($$);
+sub IsDirectory($$);
+sub Rename($$$);
+sub Unlink($@);
+sub SetFileTime($$;$$$$);
 sub DoEscape($$);
 sub ConvertFileSize($);
 sub ParseArguments($;@); #(defined in attempt to avoid mod_perl problem)
+sub ReadValue($$$;$$$);
 
 # list of main tag tables to load in LoadAllTables() (sub-tables are recursed
@@ -114 +136 @@
 # unless tweaked in BuildTagLookup::GetTableOrder().
 @loadAllTables = qw(
-    PhotoMechanic Exif GeoTiff CanonRaw KyoceraRaw MinoltaRaw PanasonicRaw
-    SigmaRaw JPEG GIMP Jpeg2000 GIF BMP BMP::OS2 PICT PNG MNG DjVu PGF MIFF PSP
-    PDF PostScript Photoshop::Header FujiFilm::RAF Sony::SRF2 Sony::SR2SubIFD
-    Sony::PMP ITC ID3 Vorbis FLAC APE APE::NewHeader APE::OldHeader MPC
-    MPEG::Audio MPEG::Video MPEG::Xing M2TS QuickTime QuickTime::ImageFile
-    Matroska MXF DV Flash Flash::FLV Real::Media Real::Audio Real::Metafile RIFF
-    AIFF ASF DICOM MIE HTML XMP::SVG EXE EXE::PEVersion EXE::PEString EXE::MachO
-    EXE::PEF EXE::ELF LNK Font RSRC Rawzor ZIP ZIP::GZIP ZIP::RAR RTF OOXML
-    iWork
+    PhotoMechanic Exif GeoTiff CanonRaw KyoceraRaw Lytro MinoltaRaw PanasonicRaw
+    SigmaRaw JPEG GIMP Jpeg2000 GIF BMP BMP::OS2 BMP::Extra BPG BPG::Extensions
+    PICT PNG MNG FLIF DjVu DPX OpenEXR ZISRAW MIFF PCX PGF PSP PhotoCD Radiance
+    PDF PostScript Photoshop::Header Photoshop::Layers Photoshop::ImageData
+    FujiFilm::RAF FujiFilm::IFD Samsung::Trailer Sony::SRF2 Sony::SR2SubIFD
+    Sony::PMP ITC ID3 ID3::Lyrics3 FLAC Ogg Vorbis APE APE::NewHeader
+    APE::OldHeader Audible MPC MPEG::Audio MPEG::Video MPEG::Xing M2TS QuickTime
+    QuickTime::ImageFile QuickTime::Stream QuickTime::Tags360Fly Matroska MOI
+    MXF DV Flash Flash::FLV Real::Media Real::Audio Real::Metafile Red RIFF AIFF
+    ASF WTV DICOM FITS MIE JSON HTML XMP::SVG Palm Palm::MOBI Palm::EXTH Torrent
+    EXE EXE::PEVersion EXE::PEString EXE::MachO EXE::PEF EXE::ELF EXE::AR
+    EXE::CHM LNK Font VCard Text VCard::VCalendar RSRC Rawzor ZIP ZIP::GZIP
+    ZIP::RAR RTF OOXML iWork ISO FLIR::AFF FLIR::FPF MacOS MacOS::MDItem
+    FlashPix::DocTable
 );
 
 # alphabetical list of current Lang modules
-@langs = qw(cs de en en_ca en_gb es fr it ja ko nl pl ru sv tr zh_cn zh_tw);
+@langs = qw(cs de en en_ca en_gb es fi fr it ja ko nl pl ru sv tr zh_cn zh_tw);
 
 $defaultLang = 'en';    # default language
@@ -138 +165 @@
     en_gb => 'British English',
     es => 'Spanish (Español)',
+    fi => 'Finnish (Suomi)',
     fr => 'French (Français)',
     it => 'Italian (Italiano)',
@@ -154 +182 @@
 # Notes: 1) There is no need to test for like types separately here
 # 2) Put types with weak file signatures at end of list to avoid false matches
-@fileTypes = qw(JPEG CRW TIFF GIF MRW RAF X3F JP2 PNG MIE MIFF PS PDF PSD XMP
-                BMP PPM RIFF AIFF ASF MOV MPEG Real SWF PSP FLV OGG FLAC APE MPC
-                MKV MXF DV PMP IND PGF ICC ITC HTML VRD RTF XCF QTIF FPX PICT
-                ZIP GZIP RAR BZ2 TAR RWZ EXE LNK RAW Font RSRC M2TS MP3 DICM);
+# 3) PLIST must be in this list for the binary PLIST format, although it may
+#    cause a file to be checked twice for XML
+@fileTypes = qw(JPEG EXV CRW DR4 TIFF GIF MRW RAF X3F JP2 PNG MIE MIFF PS PDF
+                PSD XMP BMP BPG PPM RIFF AIFF ASF MOV MPEG Real SWF PSP FLV OGG
+                FLAC APE MPC MKV MXF DV PMP IND PGF ICC ITC FLIR FLIF FPF LFP
+                HTML VRD RTF FITS XCF DSS QTIF FPX PICT ZIP GZIP PLIST RAR BZ2
+                CZI TAR  EXE EXR HDR CHM LNK WMF AVC DEX DPX RAW Font RSRC M2TS
+                MacOS PHP PCX DCX DWF DWG WTV Torrent VCard LRI R3D AA PDB MOI
+                ISO ALIAS JSON MP3 DICOM PCD TXT);
 
 # file types that we can write (edit)
-my @writeTypes = qw(JPEG TIFF GIF CRW MRW ORF RAF RAW PNG MIE PSD XMP PPM
-                    EPS X3F PS PDF ICC VRD JP2 EXIF AI AIT IND);
+my @writeTypes = qw(JPEG TIFF GIF CRW MRW ORF RAF RAW PNG MIE PSD XMP PPM EPS
+                    X3F PS PDF ICC VRD DR4 JP2 EXIF AI AIT IND MOV EXV FLIF);
+my %writeTypes; # lookup for writable file types (hash filled if required)
 
 # file extensions that we can't write for various base types
 %noWriteFile = (
     TIFF => [ qw(3FR DCR K25 KDC SRF) ],
-    XMP  => [ 'SVG' ],
+    XMP  => [ qw(SVG INX) ],
+    JP2  => [ qw(J2C JPC) ],
+    MOV  => [ qw(INSV) ],
 );
 
 # file types that we can create from scratch
 # - must update CanCreate() documentation if this list is changed!
-my %createTypes = (XMP=>1, ICC=>1, MIE=>1, VRD=>1, EXIF=>1);
-
-# file type lookup for all recognized file extensions
-my %fileTypeLookup = (
+my %createTypes = map { $_ => 1 } qw(XMP ICC MIE VRD DR4 EXIF EXV);
+
+# file type lookup for all recognized file extensions (upper case)
+# (if extension may be more than one type, the type is a list where
+#  the writable type should come first if it exists)
+%fileTypeLookup = (
+   '360' => ['MOV',  'GoPro 360 video'],
    '3FR' => ['TIFF', 'Hasselblad RAW format'],
    '3G2' => ['MOV',  '3rd Gen. Partnership Project 2 audio/video'],
@@ -180 +219 @@
    '3GP2'=>  '3G2',
    '3GPP'=>  '3GP',
-    ACR  => ['DICM', 'American College of Radiology ACR-NEMA'],
+    A    => ['EXE',  'Static library'],
+    AA   => ['AA',   'Audible Audiobook'],
+    AAE  => ['PLIST','Apple edit information'],
+    AAX  => ['MOV',  'Audible Enhanced Audiobook'],
+    ACR  => ['DICOM','American College of Radiology ACR-NEMA'],
     ACFM => ['Font', 'Adobe Composite Font Metrics'],
     AFM  => ['Font', 'Adobe Font Metrics'],
@@ -189 +232 @@
     AIFF => ['AIFF', 'Audio Interchange File Format'],
     AIT  =>  'AI',
+    ALIAS=> ['ALIAS','MacOS file alias'],
     APE  => ['APE',  "Monkey's Audio format"],
+    APNG => ['PNG',  'Animated Portable Network Graphics'],
     ARW  => ['TIFF', 'Sony Alpha RAW format'],
+    ARQ  => ['TIFF', 'Sony Alpha Pixel-Shift RAW format'],
     ASF  => ['ASF',  'Microsoft Advanced Systems Format'],
+    AVC  => ['AVC',  'Advanced Video Connection'], # (extensions are actually _AU,_AD,_IM,_ID)
     AVI  => ['RIFF', 'Audio Video Interleaved'],
+    AVIF => ['MOV',  'AV1 Image File Format'],
+    AZW  =>  'MOBI', # (see http://wiki.mobileread.com/wiki/AZW)
+    AZW3 =>  'MOBI',
     BMP  => ['BMP',  'Windows Bitmap'],
+    BPG  => ['BPG',  'Better Portable Graphics'],
     BTF  => ['BTF',  'Big Tagged Image File Format'], #(unofficial)
     BZ2  => ['BZ2',  'BZIP2 archive'],
+    CHM  => ['CHM',  'Microsoft Compiled HTML format'],
     CIFF => ['CRW',  'Camera Image File Format'],
     COS  => ['COS',  'Capture One Settings'],
     CR2  => ['TIFF', 'Canon RAW 2 format'],
+    CR3  => ['MOV',  'Canon RAW 3 format'],
+    CRM  => ['MOV',  'Canon RAW Movie'],
     CRW  => ['CRW',  'Canon RAW format'],
     CS1  => ['PSD',  'Sinar CaptureShop 1-Shot RAW'],
+    CSV  => ['TXT',  'Comma-Separated Values'],
+    CZI  => ['CZI',  'Zeiss Integrated Software RAW'],
     DC3  =>  'DICM',
     DCM  =>  'DICM',
     DCP  => ['TIFF', 'DNG Camera Profile'],
     DCR  => ['TIFF', 'Kodak Digital Camera RAW'],
+    DCX  => ['DCX',  'Multi-page PC Paintbrush'],
+    DEX  => ['DEX',  'Dalvik Executable format'],
     DFONT=> ['Font', 'Macintosh Data fork Font'],
     DIB  => ['BMP',  'Device Independent Bitmap'],
     DIC  =>  'DICM',
-    DICM => ['DICM', 'Digital Imaging and Communications in Medicine'],
+    DICM => ['DICOM','Digital Imaging and Communications in Medicine'],
+    DIR  => ['DIR',  'Directory'],
     DIVX => ['ASF',  'DivX media format'],
     DJV  =>  'DJVU',
@@ -223 +282 @@
     DOTM => [['ZIP','FPX'], 'Office Open XML Document Template Macro-enabled'],
     DOTX => [['ZIP','FPX'], 'Office Open XML Document Template'],
+    DPX  => ['DPX',  'Digital Picture Exchange' ],
+    DR4  => ['DR4',  'Canon VRD version 4 Recipe'],
+    DS2  => ['DSS',  'Digital Speech Standard 2'],
+    DSS  => ['DSS',  'Digital Speech Standard'],
     DV   => ['DV',   'Digital Video'],
     DVB  => ['MOV',  'Digital Video Broadcasting'],
+   'DVR-MS'=>['ASF', 'Microsoft Digital Video recording'],
+    DWF  => ['DWF',  'Autodesk drawing (Design Web Format)'],
+    DWG  => ['DWG',  'AutoCAD Drawing'],
     DYLIB=> ['EXE',  'Mach-O Dynamic Link Library'],
     EIP  => ['ZIP',  'Capture One Enhanced Image Package'],
@@ -231 +297 @@
     EPS3 =>  'EPS',
     EPSF =>  'EPS',
+    EPUB => ['ZIP',  'Electronic Publication'],
     ERF  => ['TIFF', 'Epson Raw Format'],
     EXE  => ['EXE',  'Windows executable file'],
+    EXR  => ['EXR', 'Open EXR'],
     EXIF => ['EXIF', 'Exchangable Image File Metadata'],
+    EXV  => ['EXV',  'Exiv2 metadata'],
     F4A  => ['MOV',  'Adobe Flash Player 9+ Audio'],
     F4B  => ['MOV',  'Adobe Flash Player 9+ audio Book'],
     F4P  => ['MOV',  'Adobe Flash Player 9+ Protected'],
     F4V  => ['MOV',  'Adobe Flash Player 9+ Video'],
+    FFF  => [['TIFF','FLIR'], 'Hasselblad Flexible File Format'],
+    FIT  =>  'FITS',
+    FITS => ['FITS', 'Flexible Image Transport System'],
     FLAC => ['FLAC', 'Free Lossless Audio Codec'],
     FLA  => ['FPX',  'Macromedia/Adobe Flash project'],
+    FLIF => ['FLIF', 'Free Lossless Image Format'],
+    FLIR => ['FLIR', 'FLIR File Format'], # (not an actual extension)
     FLV  => ['FLV',  'Flash Video'],
+    FPF  => ['FPF',  'FLIR Public image Format'],
     FPX  => ['FPX',  'FlashPix'],
     GIF  => ['GIF',  'Compuserve Graphics Interchange Format'],
+    GPR  => ['TIFF', 'GoPro RAW'],
     GZ   =>  'GZIP',
     GZIP => ['GZIP', 'GNU ZIP compressed archive'],
     HDP  => ['TIFF', 'Windows HD Photo'],
+    HDR  => ['HDR',  'Radiance RGBE High Dynamic Range'],
+    HEIC => ['MOV',  'High Efficiency Image Format still image'],
+    HEIF => ['MOV',  'High Efficiency Image Format'],
+    HIF  =>  'HEIF',
     HTM  =>  'HTML',
     HTML => ['HTML', 'HyperText Markup Language'],
+    ICAL =>  'ICS',
     ICC  => ['ICC',  'International Color Consortium'],
     ICM  =>  'ICC',
+    ICS  => ['VCard','iCalendar Schedule'],
+    IDML => ['ZIP',  'Adobe InDesign Markup Language'],
     IIQ  => ['TIFF', 'Phase One Intelligent Image Quality RAW'],
     IND  => ['IND',  'Adobe InDesign'],
     INDD => ['IND',  'Adobe InDesign Document'],
     INDT => ['IND',  'Adobe InDesign Template'],
+    INSV => ['MOV',  'Insta360 Video'],
+    INSP => ['JPEG', 'Insta360 Picture'],
+    INX  => ['XMP',  'Adobe InDesign Interchange'],
+    ISO  => ['ISO',  'ISO 9660 disk image'],
     ITC  => ['ITC',  'iTunes Cover Flow'],
+    J2C  => ['JP2',  'JPEG 2000 codestream'],
+    J2K  =>  'J2C',
     JNG  => ['PNG',  'JPG Network Graphics'],
     JP2  => ['JP2',  'JPEG 2000 file'],
     # JP4? - looks like a JPEG but the image data is different
-    JPEG =>  'JPG',
-    JPG  => ['JPEG', 'Joint Photographic Experts Group'],
+    JPC  =>  'J2C',
+    JPE  =>  'JPEG',
+    JPEG => ['JPEG', 'Joint Photographic Experts Group'],
+    JPF  =>  'JP2',
+    JPG =>   'JPEG',
     JPM  => ['JP2',  'JPEG 2000 compound image'],
     JPX  => ['JP2',  'JPEG 2000 with extensions'],
+    JSON => ['JSON', 'JavaScript Object Notation'],
+    JXR  => ['TIFF', 'JPEG XR'],
     K25  => ['TIFF', 'Kodak DC25 RAW'],
     KDC  => ['TIFF', 'Kodak Digital Camera RAW'],
     KEY  => ['ZIP',  'Apple Keynote presentation'],
     KTH  => ['ZIP',  'Apple Keynote Theme'],
+    LA   => ['RIFF', 'Lossless Audio'],
+    LFP  => ['LFP',  'Lytro Light Field Picture'],
+    LFR  =>  'LFP', # (Light Field RAW)
     LNK  => ['LNK',  'Windows shortcut'],
+    LRI  => ['LRI',  'Light RAW'],
+    LRV  => ['MOV',  'Low-Resolution Video'],
     M2T  =>  'M2TS',
     M2TS => ['M2TS', 'MPEG-2 Transport Stream'],
@@ -274 +373 @@
     M4P  => ['MOV',  'MPEG-4 Protected'],
     M4V  => ['MOV',  'MPEG-4 Video'],
+    MAX  => ['FPX',  '3D Studio MAX'],
     MEF  => ['TIFF', 'Mamiya (RAW) Electronic Format'],
     MIE  => ['MIE',  'Meta Information Encapsulation format'],
@@ -282 +382 @@
     MKV  => ['MKV',  'Matroska Video'],
     MNG  => ['PNG',  'Multiple-image Network Graphics'],
-  # MODD => ['PLIST','Sony Picture Motion Metadata'],
+    MOBI => ['PDB',  'Mobipocket electronic book'],
+    MODD => ['PLIST','Sony Picture Motion metadata'],
+    MOI  => ['MOI',  'MOD Information file'],
     MOS  => ['TIFF', 'Creo Leaf Mosaic'],
     MOV  => ['MOV',  'Apple QuickTime movie'],
@@ -293 +395 @@
     MQV  => ['MOV',  'Sony Mobile Quicktime Video'],
     MRW  => ['MRW',  'Minolta RAW format'],
-    MTS  => ['M2TS', 'MPEG-2 Transport Stream'],
+    MTS  =>  'M2TS',
     MXF  => ['MXF',  'Material Exchange Format'],
   # NDPI => ['TIFF', 'Hamamatsu NanoZoomer Digital Pathology Image'],
@@ -301 +403 @@
     NRW  => ['TIFF', 'Nikon RAW (2)'],
     NUMBERS => ['ZIP','Apple Numbers spreadsheet'],
+    O    => ['EXE',  'Relocatable Object'],
+    ODB  => ['ZIP',  'Open Document Database'],
+    ODC  => ['ZIP',  'Open Document Chart'],
+    ODF  => ['ZIP',  'Open Document Formula'],
+    ODG  => ['ZIP',  'Open Document Graphics'],
+    ODI  => ['ZIP',  'Open Document Image'],
     ODP  => ['ZIP',  'Open Document Presentation'],
     ODS  => ['ZIP',  'Open Document Spreadsheet'],
     ODT  => ['ZIP',  'Open Document Text file'],
+    OFR  => ['RIFF', 'OptimFROG audio'],
     OGG  => ['OGG',  'Ogg Vorbis audio file'],
+    OGV  => ['OGG',  'Ogg Video file'],
+    ONP  => ['JSON', 'ON1 Presets'],
+    OPUS => ['OGG',  'Ogg Opus audio file'],
     ORF  => ['ORF',  'Olympus RAW format'],
     OTF  => ['Font', 'Open Type Font'],
+    PAC  => ['RIFF', 'Lossless Predictive Audio Compression'],
     PAGES => ['ZIP', 'Apple Pages document'],
     PBM  => ['PPM',  'Portable BitMap'],
+    PCD  => ['PCD',  'Kodak Photo CD Image Pac'],
     PCT  =>  'PICT',
+    PCX  => ['PCX',  'PC Paintbrush'],
+    PDB  => ['PDB',  'Palm Database'],
     PDF  => ['PDF',  'Adobe Portable Document Format'],
     PEF  => ['TIFF', 'Pentax (RAW) Electronic Format'],
@@ -317 +433 @@
     PGF  => ['PGF',  'Progressive Graphics File'],
     PGM  => ['PPM',  'Portable Gray Map'],
+    PHP  => ['PHP',  'PHP Hypertext Preprocessor'],
+    PHP3 =>  'PHP',
+    PHP4 =>  'PHP',
+    PHP5 =>  'PHP',
+    PHPS =>  'PHP',
+    PHTML=>  'PHP',
     PICT => ['PICT', 'Apple PICTure'],
-  # PLIST=> ['PLIST','Apple Property List'],
+    PLIST=> ['PLIST','Apple Property List'],
     PMP  => ['PMP',  'Sony DSC-F1 Cyber-Shot PMP'], # should stand for Proprietery Metadata Package ;)
     PNG  => ['PNG',  'Portable Network Graphics'],
@@ -324 +446 @@
     POTM => [['ZIP','FPX'], 'Office Open XML Presentation Template Macro-enabled'],
     POTX => [['ZIP','FPX'], 'Office Open XML Presentation Template'],
+    PPAM => [['ZIP','FPX'], 'Office Open XML Presentation Addin Macro-enabled'],
+    PPAX => [['ZIP','FPX'], 'Office Open XML Presentation Addin'],
     PPM  => ['PPM',  'Portable Pixel Map'],
     PPS  => ['FPX',  'Microsoft PowerPoint Slideshow'],
@@ -331 +455 @@
     PPTM => [['ZIP','FPX'], 'Office Open XML Presentation Macro-enabled'],
     PPTX => [['ZIP','FPX'], 'Office Open XML Presentation'],
+    PRC  => ['PDB',  'Palm Database'],
     PS   => ['PS',   'PostScript'],
     PS2  =>  'PS',
     PS3  =>  'PS',
     PSB  => ['PSD',  'Photoshop Large Document'],
-    PSD  => ['PSD',  'Photoshop Drawing'],
+    PSD  => ['PSD',  'Photoshop Document'],
+    PSDT => ['PSD',  'Photoshop Document Template'],
     PSP  => ['PSP',  'Paint Shop Pro'],
     PSPFRAME => 'PSP',
@@ -342 +468 @@
     PSPTUBE  => 'PSP',
     QIF  =>  'QTIF',
-    QT   => ['MOV',  'QuickTime movie'],
+    QT   =>  'MOV',
     QTI  =>  'QTIF',
     QTIF => ['QTIF', 'QuickTime Image File'],
+    R3D  => ['R3D',  'Redcode RAW Video'],
     RA   => ['Real', 'Real Audio'],
     RAF  => ['RAF',  'FujiFilm RAW Format'],
@@ -361 +488 @@
     RWL  => ['TIFF', 'Leica RAW'],
     RWZ  => ['RWZ',  'Rawzor compressed image'],
+    SEQ  => ['FLIR', 'FLIR image Sequence'],
+    SKETCH => ['ZIP', 'Sketch design file'],
     SO   => ['EXE',  'Shared Object file'],
     SR2  => ['TIFF', 'Sony RAW Format 2'],
@@ -368 +497 @@
     SWF  => ['SWF',  'Shockwave Flash'],
     TAR  => ['TAR',  'TAR archive'],
-    THM  => ['JPEG', 'Canon Thumbnail'],
+    THM  => ['JPEG', 'Thumbnail'],
     THMX => [['ZIP','FPX'], 'Office Open XML Theme'],
     TIF  =>  'TIFF',
     TIFF => ['TIFF', 'Tagged Image File Format'],
+    TORRENT => ['Torrent', 'BitTorrent description file'],
     TS   =>  'M2TS',
     TTC  => ['Font', 'True Type Font Collection'],
     TTF  => ['Font', 'True Type Font'],
     TUB  => 'PSP',
+    TXT  => ['TXT',  'Text file'],
+    VCARD=> ['VCard','Virtual Card'],
+    VCF  => 'VCARD',
     VOB  => ['MPEG', 'Video Object'],
     VRD  => ['VRD',  'Canon VRD Recipe Data'],
@@ -384 +517 @@
     WEBP => ['RIFF', 'Google Web Picture'],
     WMA  => ['ASF',  'Windows Media Audio'],
+    WMF  => ['WMF',  'Windows Metafile Format'],
     WMV  => ['ASF',  'Windows Media Video'],
+    WV   => ['RIFF', 'WavePack lossless audio'],
     X3F  => ['X3F',  'Sigma RAW format'],
+    MACOS=> ['MacOS','MacOS ._ sidecar file'],
     XCF  => ['XCF',  'GIMP native image format'],
     XHTML=> ['HTML', 'Extensible HyperText Markup Language'],
@@ -398 +534 @@
     XLTX => [['ZIP','FPX'], 'Office Open XML Spreadsheet Template'],
     XMP  => ['XMP',  'Extensible Metadata Platform'],
+    WOFF => ['Font', 'Web Open Font Format'],
+    WOFF2=> ['Font', 'Web Open Font Format2'],
+    WTV  => ['WTV',  'Windows recorded TV show'],
     ZIP  => ['ZIP',  'ZIP archive'],
+);
+
+# typical extension for each file type (if different than FileType)
+# - case is not significant
+my %fileTypeExt = (
+    'Canon 1D RAW' => 'tif',
+    DICOM   => 'dcm',
+    FLIR    => 'fff',
+    GZIP    => 'gz',
+    JPEG    => 'jpg',
+    M2TS    => 'mts',
+    MPEG    => 'mpg',
+    TIFF    => 'tif',
+    VCard   => 'vcf',
 );
 
@@ -404 +557 @@
 my %fileDescription = (
     DICOM => 'Digital Imaging and Communications in Medicine',
-    PLIST => 'Property List',
     XML   => 'Extensible Markup Language',
-    'DJVU (multi-page)' => 'DjVu multi-page image',
     'Win32 EXE' => 'Windows 32-bit Executable',
     'Win32 DLL' => 'Windows 32-bit Dynamic Link Library',
+    'Win64 EXE' => 'Windows 64-bit Executable',
+    'Win64 DLL' => 'Windows 64-bit Dynamic Link Library',
 );
 
 # MIME types for applicable file types above
-# (missing entries default to 'application/unknown', but note that
-#  other mime types may be specified by some modules, ie. QuickTime.pm)
+# (missing entries default to 'application/unknown', but note that other MIME
+#  types may be specified by some modules, eg. QuickTime.pm and RIFF.pm)
 %mimeType = (
    '3FR' => 'image/x-hasselblad-3fr',
+    AA   => 'audio/audible',
+    AAE  => 'application/vnd.apple.photos',
     AI   => 'application/vnd.adobe.illustrator',
     AIFF => 'audio/x-aiff',
+    ALIAS=> 'application/x-macos',
     APE  => 'audio/x-monkeys-audio',
+    APNG => 'image/apng',
     ASF  => 'video/x-ms-asf',
     ARW  => 'image/x-sony-arw',
-    AVI  => 'video/x-msvideo',
     BMP  => 'image/bmp',
+    BPG  => 'image/bpg',
     BTF  => 'image/x-tiff-big', #(NC) (ref http://www.asmail.be/msg0055371937.html)
     BZ2  => 'application/bzip2',
    'Canon 1D RAW' => 'image/x-raw', # (uses .TIF file extension)
+    CHM  => 'application/x-chm',
+    COS  => 'application/octet-stream', #PH (NC)
     CR2  => 'image/x-canon-cr2',
+    CR3  => 'image/x-canon-cr3',
+    CRM  => 'video/x-canon-crm',
     CRW  => 'image/x-canon-crw',
+    CSV  => 'text/csv',
+    CZI  => 'image/x-zeiss-czi', #PH (NC)
+    DCP  => 'application/octet-stream', #PH (NC)
     DCR  => 'image/x-kodak-dcr',
+    DCX  => 'image/dcx',
+    DEX  => 'application/octet-stream',
     DFONT=> 'application/x-dfont',
-    DICM => 'application/dicom',
+    DICOM=> 'application/dicom',
     DIVX => 'video/divx',
     DJVU => 'image/vnd.djvu',
@@ -440 +606 @@
     DOTM => 'application/vnd.ms-word.template.macroEnabledTemplate',
     DOTX => 'application/vnd.openxmlformats-officedocument.wordprocessingml.template',
+    DPX  => 'image/x-dpx',
+    DR4  => 'application/octet-stream', #PH (NC)
+    DS2  => 'audio/x-ds2',
+    DSS  => 'audio/x-dss',
     DV   => 'video/x-dv',
+   'DVR-MS' => 'video/x-ms-dvr',
+    DWF  => 'model/vnd.dwf',
+    DWG  => 'image/vnd.dwg',
     EIP  => 'application/x-captureone', #(NC)
     EPS  => 'application/postscript',
     ERF  => 'image/x-epson-erf',
     EXE  => 'application/octet-stream',
+    EXR  => 'image/x-exr',
+    EXV  => 'image/x-exv',
+    FFF  => 'image/x-hasselblad-fff',
+    FITS => 'image/fits',
     FLA  => 'application/vnd.adobe.fla',
     FLAC => 'audio/flac',
+    FLIF => 'image/flif',
+    FLIR => 'image/x-flir-fff', #PH (NC)
     FLV  => 'video/x-flv',
     Font => 'application/x-font-type1', # covers PFA, PFB and PFM (not sure about PFM)
+    FPF  => 'image/x-flir-fpf', #PH (NC)
     FPX  => 'image/vnd.fpx',
     GIF  => 'image/gif',
+    GPR  => 'image/x-gopro-gpr',
     GZIP => 'application/x-gzip',
     HDP  => 'image/vnd.ms-photo',
+    HDR  => 'image/vnd.radiance',
     HTML => 'text/html',
     ICC  => 'application/vnd.iccprofile',
+    ICS  => 'text/calendar',
+    IDML => 'application/vnd.adobe.indesign-idml-package',
     IIQ  => 'image/x-raw',
     IND  => 'application/x-indesign',
+    INX  => 'application/x-indesign-interchange', #PH (NC)
+    ISO  => 'application/x-iso9660-image',
     ITC  => 'application/itunes',
+    J2C  => 'image/x-j2c', #PH (NC)
     JNG  => 'image/jng',
     JP2  => 'image/jp2',
@@ -463 +650 @@
     JPM  => 'image/jpm',
     JPX  => 'image/jpx',
+    JSON => 'application/json',
+    JXR  => 'image/jxr',
     K25  => 'image/x-kodak-k25',
     KDC  => 'image/x-kodak-kdc',
+    KEY  => 'application/x-iwork-keynote-sffkey',
+    LFP  => 'image/x-lytro-lfp', #PH (NC)
     LNK  => 'application/octet-stream',
+    LRI  => 'image/x-light-lri',
     M2T  => 'video/mpeg',
     M2TS => 'video/m2ts',
+    MAX  => 'application/x-3ds',
     MEF  => 'image/x-mamiya-mef',
     MIE  => 'application/x-mie',
@@ -475 +668 @@
     MKV  => 'video/x-matroska',
     MNG  => 'video/mng',
+    MOBI => 'application/x-mobipocket-ebook',
+    MOI  => 'application/octet-stream', #PH (NC)
     MOS  => 'image/x-raw',
     MOV  => 'video/quicktime',
@@ -485 +680 @@
     NEF  => 'image/x-nikon-nef',
     NRW  => 'image/x-nikon-nrw',
+    NUMBERS => 'application/x-iwork-numbers-sffnumbers',
+    ODB  => 'application/vnd.oasis.opendocument.database',
+    ODC  => 'application/vnd.oasis.opendocument.chart',
+    ODF  => 'application/vnd.oasis.opendocument.formula',
+    ODG  => 'application/vnd.oasis.opendocument.graphics',
+    ODI  => 'application/vnd.oasis.opendocument.image',
     ODP  => 'application/vnd.oasis.opendocument.presentation',
     ODS  => 'application/vnd.oasis.opendocument.spreadsheet',
     ODT  => 'application/vnd.oasis.opendocument.text',
-    OGG  => 'audio/x-ogg',
+    OGG  => 'audio/ogg',
+    OGV  => 'video/ogg',
+    ONP  => 'application/on1',
     ORF  => 'image/x-olympus-orf',
     OTF  => 'application/x-font-otf',
+    PAGES=> 'application/x-iwork-pages-sffpages',
     PBM  => 'image/x-portable-bitmap',
+    PCD  => 'image/x-photo-cd',
+    PCX  => 'image/pcx',
+    PDB  => 'application/vnd.palm',
     PDF  => 'application/pdf',
     PEF  => 'image/x-pentax-pef',
+    PFA  => 'application/x-font-type1', # (needed if handled by PostScript module)
     PGF  => 'image/pgf',
     PGM  => 'image/x-portable-graymap',
+    PHP  => 'application/x-httpd-php',
     PICT => 'image/pict',
-    PLIST=> 'application/xml',
+    PLIST=> 'application/xml', # (binary PLIST format is 'application/x-plist', recognized at run time)
+    PMP  => 'image/x-sony-pmp', #PH (NC)
     PNG  => 'image/png',
     POT  => 'application/vnd.ms-powerpoint',
     POTM => 'application/vnd.ms-powerpoint.template.macroEnabled',
     POTX => 'application/vnd.openxmlformats-officedocument.presentationml.template',
+    PPAM => 'application/vnd.ms-powerpoint.addin.macroEnabled',
+    PPAX => 'application/vnd.openxmlformats-officedocument.presentationml.addin', # (NC, PH invented)
     PPM  => 'image/x-portable-pixmap',
     PPS  => 'application/vnd.ms-powerpoint',
@@ -513 +725 @@
     PSP  => 'image/x-paintshoppro', #(NC)
     QTIF => 'image/x-quicktime',
+    R3D  => 'video/x-red-r3d', #PH (invented)
     RA   => 'audio/x-pn-realaudio',
     RAF  => 'image/x-fujifilm-raf',
@@ -527 +740 @@
     RWL  => 'image/x-leica-rwl',
     RWZ  => 'image/x-rawzor', #(duplicated in Rawzor.pm)
+    SEQ  => 'image/x-flir-seq', #PH (NC)
+    SKETCH => 'application/sketch',
     SR2  => 'image/x-sony-sr2',
     SRF  => 'image/x-sony-srf',
@@ -535 +750 @@
     THMX => 'application/vnd.ms-officetheme',
     TIFF => 'image/tiff',
+    Torrent => 'application/x-bittorrent',
     TTC  => 'application/x-font-ttf',
     TTF  => 'application/x-font-ttf',
+    TXT  => 'text/plain',
+    VCard=> 'text/vcard',
+    VRD  => 'application/octet-stream', #PH (NC)
     VSD  => 'application/x-visio',
-    WAV  => 'audio/x-wav',
     WDP  => 'image/vnd.ms-photo',
     WEBM => 'video/webm',
-    WEBP => 'image/webp',
     WMA  => 'audio/x-ms-wma',
+    WMF  => 'application/x-wmf',
     WMV  => 'video/x-ms-wmv',
+    WTV  => 'video/x-ms-wtv',
     X3F  => 'image/x-sigma-x3f',
     XCF  => 'image/x-xcf',
@@ -565 +784 @@
 # - module name '0' indicates a recognized but unsupported file
 my %moduleName = (
+    AA   => 'Audible',
+    ALIAS=> 0,
+    AVC  => 0,
     BTF  => 'BigTIFF',
     BZ2  => 0,
     CRW  => 'CanonRaw',
-    DICM => 'DICOM',
+    CHM  => 'EXE',
     COS  => 'CaptureOne',
+    CZI  => 'ZISRAW',
+    DEX  => 0,
     DOCX => 'OOXML',
+    DCX  => 0,
+    DR4  => 'CanonVRD',
+    DSS  => 'Olympus',
+    DWF  => 0,
+    DWG  => 0,
     EPS  => 'PostScript',
     EXIF => '',
+    EXR  => 'OpenEXR',
+    EXV  => '',
     ICC  => 'ICC_Profile',
     IND  => 'InDesign',
     FLV  => 'Flash',
+    FPF  => 'FLIR',
     FPX  => 'FlashPix',
     GZIP => 'ZIP',
+    HDR  => 'Radiance',
     JP2  => 'Jpeg2000',
     JPEG => '',
-  # MODD => 'XML',
+    LFP  => 'Lytro',
+    LRI  => 0,
     MOV  => 'QuickTime',
     MKV  => 'Matroska',
     MP3  => 'ID3',
     MRW  => 'MinoltaRaw',
-    OGG  => 'Vorbis',
+    OGG  => 'Ogg',
     ORF  => 'Olympus',
-  # PLIST=> 'XML',
+    PDB  => 'Palm',
+    PCD  => 'PhotoCD',
+    PHP  => 0,
     PMP  => 'Sony',
     PS   => 'PostScript',
     PSD  => 'Photoshop',
     QTIF => 'QuickTime',
+    R3D  => 'Red',
     RAF  => 'FujiFilm',
     RAR  => 'ZIP',
@@ -599 +836 @@
     TAR  => 0,
     TIFF => '',
+    TXT  => 'Text',
     VRD  => 'CanonVRD',
+    WMF  => 0,
     X3F  => 'SigmaRaw',
     XCF  => 'GIMP',
 );
 
+$testLen = 1024;    # number of bytes to read when testing for magic number
+
 # quick "magic number" file test used to avoid loading module unnecessarily:
-# - regular expression evaluated on first 1024 bytes of file
+# - regular expression evaluated on first $testLen bytes of file
 # - must match beginning at first byte in file
 # - this test must not be more stringent than module logic
 %magicNumber = (
+    AA   => '.{4}\x57\x90\x75\x36',
     AIFF => '(FORM....AIF[FC]|AT&TFORM)',
+    ALIAS=> "book\0\0\0\0mark\0\0\0\0",
     APE  => '(MAC |APETAGEX|ID3)',
     ASF  => '\x30\x26\xb2\x75\x8e\x66\xcf\x11\xa6\xd9\x00\xaa\x00\x62\xce\x6c',
+    AVC  => '\+A\+V\+C\+',
+    Torrent => 'd\d+:\w+',
     BMP  => 'BM',
+    BPG  => "BPG\xfb",
     BTF  => '(II\x2b\0|MM\0\x2b)',
     BZ2  => 'BZh[1-9]\x31\x41\x59\x26\x53\x59',
+    CHM  => 'ITSF.{20}\x10\xfd\x01\x7c\xaa\x7b\xd0\x11\x9e\x0c\0\xa0\xc9\x22\xe6\xec',
     CRW  => '(II|MM).{4}HEAP(CCDR|JPGM)',
-    DICM => '(.{128}DICM|\0[\x02\x04\x06\x08]\0[\0-\x20]|[\x02\x04\x06\x08]\0[\0-\x20]\0)',
+    CZI  => 'ZISRAWFILE\0{6}',
+    DCX  => '\xb1\x68\xde\x3a',
+    DEX  => "dex\n035\0",
+    DICOM=> '(.{128}DICM|\0[\x02\x04\x06\x08]\0[\0-\x20]|[\x02\x04\x06\x08]\0[\0-\x20]\0)',
     DOCX => 'PK\x03\x04',
+    DPX  => '(SDPX|XPDS)',
+    DR4  => 'IIII\x04\0\x04\0',
+    DSS  => '(\x02dss|\x03ds2)',
     DV   => '\x1f\x07\0[\x3f\xbf]', # (not tested if extension recognized)
+    DWF  => '\(DWF V\d',
+    DWG  => 'AC10\d{2}\0',
     EPS  => '(%!PS|%!Ad|\xc5\xd0\xd3\xc6)',
     EXE  => '(MZ|\xca\xfe\xba\xbe|\xfe\xed\xfa[\xce\xcf]|[\xce\xcf]\xfa\xed\xfe|Joy!peff|\x7fELF|#!\s*/\S*bin/|!<arch>\x0a)',
     EXIF => '(II\x2a\0|MM\0\x2a)',
+    EXR  => '\x76\x2f\x31\x01',
+    EXV  => '\xff\x01Exiv2',
+    FITS => 'SIMPLE  = {20}T',
     FLAC => '(fLaC|ID3)',
+    FLIF => 'FLIF[0-\x6f][0-2]',
+    FLIR => '[AF]FF\0',
     FLV  => 'FLV\x01',
     Font => '((\0\x01\0\0|OTTO|true|typ1)[\0\x01]|ttcf\0[\x01\x02]\0\0|\0[\x01\x02]|' .
-            '(.{6})?%!(PS-(AdobeFont-|Bitstream )|FontType1-)|Start(Comp|Master)?FontMetrics)',
+            '(.{6})?%!(PS-(AdobeFont-|Bitstream )|FontType1-)|Start(Comp|Master)?FontMetrics|wOF[F2])',
+    FPF  => 'FPF Public Image Format\0',
     FPX  => '\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1',
     GIF  => 'GIF8[79]a',
     GZIP => '\x1f\x8b\x08',
-    HTML => '(?i)<(!DOCTYPE\s+HTML|HTML|\?xml)', # (case insensitive)
+    HDR  => '#\?(RADIANCE|RGBE)\x0a',
+    HTML => '(\xef\xbb\xbf)?\s*(?i)<(!DOCTYPE\s+HTML|HTML|\?xml)', # (case insensitive)
     ICC  => '.{12}(scnr|mntr|prtr|link|spac|abst|nmcl|nkpf)(XYZ |Lab |Luv |YCbr|Yxy |RGB |GRAY|HSV |HLS |CMYK|CMY |[2-9A-F]CLR){2}',
     IND  => '\x06\x06\xed\xf5\xd8\x1d\x46\xe5\xbd\x31\xef\xe7\xfe\x74\xb7\x1d',
+  # ISO  =>  signature is at byte 32768
     ITC  => '.{4}itch',
-    JP2  => '\0\0\0\x0cjP(  |\x1a\x1a)\x0d\x0a\x87\x0a',
+    JP2  => '(\0\0\0\x0cjP(  |\x1a\x1a)\x0d\x0a\x87\x0a|\xff\x4f\xff\x51\0)',
     JPEG => '\xff\xd8\xff',
+    JSON => '(\xef\xbb\xbf)?\s*(\[\s*)?\{\s*"[^"]*"\s*:',
+    LFP  => '\x89LFP\x0d\x0a\x1a\x0a',
     LNK  => '.{4}\x01\x14\x02\0{5}\xc0\0{6}\x46',
+    LRI  => 'LELR \0',
     M2TS => '(....)?\x47',
     MIE  => '~[\x10\x18]\x04.0MIE',
     MIFF => 'id=ImageMagick',
     MKV  => '\x1a\x45\xdf\xa3',
-    MOV  => '.{4}(free|skip|wide|ftyp|pnot|PICT|pict|moov|mdat|junk|uuid)',
+    MOV  => '.{4}(free|skip|wide|ftyp|pnot|PICT|pict|moov|mdat|junk|uuid)', # (duplicated in WriteQuickTime.pl !!)
   # MP3  =>  difficult to rule out
     MPC  => '(MP\+|ID3)',
+    MOI  => 'V6',
     MPEG => '\0\0\x01[\xb0-\xbf]',
     MRW  => '\0MR[MI]',
@@ -648 +915 @@
     OGG  => '(OggS|ID3)',
     ORF  => '(II|MM)',
-    PDF  => '%PDF-\d+\.\d+',
+    PDB  => '.{60}(\.pdfADBE|TEXtREAd|BVokBDIC|DB99DBOS|PNRdPPrs|DataPPrs|vIMGView|PmDBPmDB|InfoINDB|ToGoToGo|SDocSilX|JbDbJBas|JfDbJFil|DATALSdb|Mdb1Mdb1|BOOKMOBI|DataPlkr|DataSprd|SM01SMem|TEXtTlDc|InfoTlIf|DataTlMl|DataTlPt|dataTDBP|TdatTide|ToRaTRPW|zTXTGPlm|BDOCWrdS)',
+  # PCD  =>  signature is at byte 2048
+    PCX  => '\x0a[\0-\x05]\x01[\x01\x02\x04\x08].{64}[\0-\x02]',
+    PDF  => '\s*%PDF-\d+\.\d+',
     PGF  => 'PGF',
+    PHP  => '<\?php\s',
     PICT => '(.{10}|.{522})(\x11\x01|\x00\x11)',
+    PLIST=> '(bplist0|\s*<|\xfe\xff\x00)',
     PMP  => '.{8}\0{3}\x7c.{112}\xff\xd8\xff\xdb',
     PNG  => '(\x89P|\x8aM|\x8bJ)NG\r\n\x1a\n',
@@ -658 +930 @@
     PSP  => 'Paint Shop Pro Image File\x0a\x1a\0{5}',
     QTIF => '.{4}(idsc|idat|iicc)',
+    R3D  => '\0\0..RED(1|2)',
     RAF  => 'FUJIFILM',
     RAR  => 'Rar!\x1a\x07\0',
     RAW  => '(.{25}ARECOYK|II|MM)',
     Real => '(\.RMF|\.ra\xfd|pnm://|rtsp://|http://)',
-    RIFF => 'RIFF',
+    RIFF => '(RIFF|LA0[234]|OFR |LPAC|wvpk|RF64)', # RIFF plus other variants
     RSRC => '(....)?\0\0\x01\0',
     RTF  => '[\n\r]*\\{[\n\r]*\\\\rtf',
-    # (don't be too restrictive for RW2/RWL -- how does magic number change for big-endian?)
-    RW2  => '(II|MM)', #(\x55\0\x18\0\0\0\x88\xe7\x74\xd8\xf8\x25\x1d\x4d\x94\x7a\x6e\x77\x82\x2b\x5d\x6a)
-    RWL  => '(II|MM)', #(ditto)
     RWZ  => 'rawzor',
     SWF  => '[FC]WS[^\0]',
     TAR  => '.{257}ustar(  )?\0', # (this doesn't catch old-style tar files)
+    TXT  => '(\xff\xfe|(\0\0)?\xfe\xff|(\xef\xbb\xbf)?[\x07-\x0d\x20-\x7e\x80-\xfe]*$)',
     TIFF => '(II|MM)', # don't test magic number (some raw formats are different)
+    VCard=> '(?i)BEGIN:(VCARD|VCALENDAR)\r\n',
     VRD  => 'CANON OPTIONAL DATA\0',
+    WMF  => '(\xd7\xcd\xc6\x9a\0\0|\x01\0\x09\0\0\x03)',
+    WTV  => '\xb7\xd8\x00\x20\x37\x49\xda\x11\xa6\x4e\x00\x07\xe9\x5e\xad\x8d',
     X3F  => 'FOVb',
+    MacOS=> '\0\x05\x16\x07\0.\0\0Mac OS X        ',
     XCF  => 'gimp xcf ',
     XMP  => '\0{0,3}(\xfe\xff|\xff\xfe|\xef\xbb\xbf)?\0{0,3}\s*<',
     ZIP  => 'PK\x03\x04',
 );
+
+# file types with weak magic number recognition
+my %weakMagic = ( MP3 => 1 );
+
+# file types that are determined by the process proc when FastScan == 3
+# (when done, the process proc must exit after SetFileType if FastScan is 3)
+my %processType = map { $_ => 1 } qw(JPEG TIFF XMP AIFF EXE Font PS Real VCard TXT);
+
+# Compact/XMPShorthand option settings
+my %compactOpt = (
+    nopadding => 'NoPadding', noindent => 'NoIndent', nonewline => 'NoNewline',
+    shorthand => 'Shorthand', onedesc => 'OneDesc',
+    all => ['NoPadding','NoIndent','NoNewline','Shorthand','OneDesc'],
+    allspace => ['NoPadding','NoIndent','NoNewline'], allformat => ['Shorthand','OneDesc'],
+    # aliases to cover anticipated user typos
+    nonewlines => 'NoNewline', nospace => 'NoIndent', nospaces => 'NoIndent',
+    nopad => 'NoPadding', onedescr => 'OneDesc',
+    # allow numerical settings for backward compatibility
+    0 => 'None',
+    1 => 'NoPadding',
+    2 => ['NoPadding','NoIndent'],
+    3 => ['NoPadding','NoIndent','OneDesc'],
+    4 => ['NoPadding','NoIndent','OneDesc','NoNewline'],
+    5 => ['NoPadding','NoIndent','OneDesc','NoNewline','Shorthand'],
+);
+my %xmpShorthandOpt = ( 0 => 'None', 1 => 'Shorthand', 2 => ['Shorthand','OneDesc'] );
 
 # lookup for valid character set names (keys are all lower case)
@@ -694 +995 @@
     vietnam     => 'Vietnam',     cp1258  => 'Vietnam',
     thai        => 'Thai',        cp874   => 'Thai',
+    doslatinus  => 'DOSLatinUS',  cp437   => 'DOSLatinUS',
+    doslatin1   => 'DOSLatin1',   cp850   => 'DOSLatin1',
+    doscyrillic => 'DOSCyrillic', cp866   => 'DOSCyrillic',
     macroman    => 'MacRoman',    cp10000 => 'MacRoman', mac => 'MacRoman', roman => 'MacRoman',
     maclatin2   => 'MacLatin2',   cp10029 => 'MacLatin2',
@@ -704 +1008 @@
 );
 
-# default group priority for writing
-my @defaultWriteGroups = qw(EXIF IPTC XMP MakerNotes Photoshop ICC_Profile CanonVRD);
+# default family 0 group priority for writing
+# (NOTE: tags in groups not specified here will not be written unless
+#  overridden by the module or specified when writing)
+my @defaultWriteGroups = qw(
+    EXIF IPTC XMP MakerNotes QuickTime Photoshop ICC_Profile CanonVRD Adobe
+);
 
 # group hash for ExifTool-generated tags
@@ -711 +1019 @@
 
 # special tag names (not used for tag info)
-my %specialTags = (
-    TABLE_NAME=>1, SHORT_NAME=>1, PROCESS_PROC=>1, WRITE_PROC=>1, CHECK_PROC=>1,
-    GROUPS=>1, FORMAT=>1, FIRST_ENTRY=>1, TAG_PREFIX=>1, PRINT_CONV=>1,
-    WRITABLE=>1, TABLE_DESC=>1, NOTES=>1, IS_OFFSET=>1, EXTRACT_UNKNOWN=>1,
-    NAMESPACE=>1, PREFERRED=>1, SRC_TABLE=>1, PRIORITY=>1, WRITE_GROUP=>1,
-    LANG_INFO=>1, VARS=>1, DATAMEMBER=>1, IS_SUBDIR=>1, SET_GROUP1=>1,
+%specialTags = map { $_ => 1 } qw(
+    TABLE_NAME       SHORT_NAME  PROCESS_PROC  WRITE_PROC  CHECK_PROC
+    GROUPS           FORMAT      FIRST_ENTRY   TAG_PREFIX  PRINT_CONV
+    WRITABLE         TABLE_DESC  NOTES         IS_OFFSET   IS_SUBDIR
+    EXTRACT_UNKNOWN  NAMESPACE   PREFERRED     SRC_TABLE   PRIORITY
+    AVOID            WRITE_GROUP LANG_INFO     VARS        DATAMEMBER
+    SET_GROUP1       PERMANENT   INIT_TABLE
 );
 
@@ -734 +1043 @@
 @Image::ExifTool::pluginTags = ( );
 %Image::ExifTool::pluginTags = ( );
+
+my %systemTagsNotes = (
+    Notes => q{
+        extracted only if specifically requested or the L<SystemTags|../ExifTool.html#SystemTags> or L<RequestAll|../ExifTool.html#RequestAll> API
+        option is set
+    },
+);
 
 # tag information for preview image -- this should be used for all
@@ -746 +1062 @@
     RawConv => '$self->ValidateImage(ref $val ? $val : \$val, $tag)',
     # we allow preview image to be set to '', but we don't want a zero-length value
-    # in the IFD, so set it temorarily to 'none'.  Note that the length is <= 4,
+    # in the IFD, so set it temporarily to 'none'.  Note that the length is <= 4,
     # so this value will fit in the IFD so the preview fixup won't be generated.
     ValueConvInv => '$val eq "" and $val="none"; $val',
@@ -758 +1074 @@
     VARS => { NO_ID => 1 }, # tag ID's aren't meaningful for these tags
     WRITE_PROC => \&DummyWriteProc,
-    Error   => { Priority => 0, Groups => \%allGroupsExifTool },
-    Warning => { Priority => 0, Groups => \%allGroupsExifTool },
+    Error   => {
+        Priority => 0,
+        Groups => \%allGroupsExifTool,
+        Notes => q{
+            returns errors that may have occurred while reading or writing a file.  Any
+            Error will prevent the file from being processed.  Minor errors may be
+            downgraded to warnings with the -m or L<IgnoreMinorErrors|../ExifTool.html#IgnoreMinorErrors> option
+        },
+    },
+    Warning => {
+        Priority => 0,
+        Groups => \%allGroupsExifTool,
+        Notes => q{
+            returns warnings that may have occurred while reading or writing a file.
+            Use the -a or L<Duplicates|../ExifTool.html#Duplicates> option to see all warnings if more than one
+            occurred. Minor warnings may be ignored with the -m or L<IgnoreMinorErrors|../ExifTool.html#IgnoreMinorErrors>
+            option.  Minor warnings with a capital "M" in the "[Minor]" designation
+            indicate that the processing is affected by ignoring the warning
+        },
+    },
     Comment => {
         Notes => 'comment embedded in JPEG, GIF89a or PPM/PGM/PBM image',
@@ -767 +1101 @@
     },
     Directory => {
-        Groups => { 1 => 'System' },
+        Groups => { 1 => 'System', 2 => 'Other' },
+        Notes => q{
+            the directory of the file as specified in the call to ExifTool, or "." if no
+            directory was specified.  May be written to move the file to another
+            directory that will be created if doesn't already exist
+        },
         Writable => 1,
+        WritePseudo => 1,
+        DelCheck => q{"Can't delete"},
         Protected => 1,
+        RawConv => '$self->ConvertFileName($val)',
         # translate backslashes in directory names and add trailing '/'
-        ValueConvInv => '$_=$val; tr/\\\\/\//; m{[^/]$} and $_ .= "/"; $_',
+        ValueConvInv => '$_ = $self->InverseFileName($val); m{[^/]$} and $_ .= "/"; $_',
     },
     FileName => {
-        Groups => { 1 => 'System' },
+        Groups => { 1 => 'System', 2 => 'Other' },
         Writable => 1,
+        WritePseudo => 1,
+        DelCheck => q{"Can't delete"},
         Protected => 1,
         Notes => q{
             may be written with a full path name to set FileName and Directory in one
-            operation.  See L<filename.html|../filename.html> for more information on
-            writing the FileName and Directory tags
+            operation.  This is such a powerful feature that a TestName tag is provided
+            to allow dry-run tests before actually writing the file name.  See
+            L<filename.html|../filename.html> for more information on writing the
+            FileName, Directory and TestName tags
         },
-        ValueConvInv => '$val=~tr/\\\\/\//; $val',
+        RawConv => '$self->ConvertFileName($val)',
+        ValueConvInv => '$self->InverseFileName($val)',
+    },
+    FilePath => {
+        Groups => { 1 => 'System', 2 => 'Other' },
+        Notes => q{
+            absolute path of source file. Not generated unless specifically requested or
+            the L<RequestAll|../ExifTool.html#RequestAll> API option is set.  Does not support Windows Unicode file
+            names
+        },
+    },
+    TestName => {
+        Writable => 1,
+        WritePseudo => 1,
+        DelCheck => q{"Can't delete"},
+        Protected => 1,
+        WriteOnly => 1,
+        Notes => q{
+            this write-only tag may be used instead of FileName for dry-run tests of the
+            file renaming feature.  Writing this tag prints the old and new file names
+            to the console, but does not affect the file itself
+        },
+        ValueConvInv => '$self->InverseFileName($val)',
+    },
+    FileSequence => {
+        Groups => { 0 => 'ExifTool', 1 => 'ExifTool', 2 => 'Other' },
+        Notes => q{
+            sequence number for each source file when extracting or copying information,
+            including files that fail the -if condition of the command-line application,
+            beginning at 0 for the first file.  Not generated unless specifically
+            requested or the L<RequestAll|../ExifTool.html#RequestAll> API option is set
+        },
     },
     FileSize => {
-        Groups => { 1 => 'System' },
+        Groups => { 1 => 'System', 2 => 'Other' },
+        Notes => q{
+            note that the print conversion for this tag uses historic prefixes: 1 kB =
+            1024 bytes, etc.
+        },
         PrintConv => \&ConvertFileSize,
     },
     ResourceForkSize => {
-        Groups => { 1 => 'System' },
+        Groups => { 1 => 'System', 2 => 'Other' },
         Notes => q{
-            [Mac OS only] size of the file's resource fork if it contains data.  If this
-            tag is generated the ExtractEmbedded option may be used to extract
-            resource-fork information as a sub-document
+            size of the file's resource fork if it contains data.  Mac OS only.  If this
+            tag is generated the L<ExtractEmbedded|../ExifTool.html#ExtractEmbedded> option may be used to extract
+            resource-fork information as a sub-document.  When writing, the resource
+            fork is preserved by default, but it may be deleted with C<-rsrc:all=> on
+            the command line
         },
         PrintConv => \&ConvertFileSize,
     },
-    FileType    => { },
+    FileType => {
+        Groups => { 2 => 'Other' },
+        Notes => q{
+            a short description of the file type.  For many file types this is the just
+            the uppercase file extension
+        },
+    },
+    FileTypeExtension => {
+        Groups => { 2 => 'Other' },
+        Notes => q{
+            a common lowercase extension for this file type, or uppercase with the -n
+            option
+        },
+        PrintConv => 'lc $val',
+    },
     FileModifyDate => {
         Description => 'File Modification Date/Time',
-        Notes => 'the filesystem modification time',
+        Notes => q{
+            the filesystem modification date/time.  Note that ExifTool may not be able
+            to handle filesystem dates before 1970 depending on the limitations of the
+            system's standard libraries
+        },
         Groups => { 1 => 'System', 2 => 'Time' },
         Writable => 1,
-        # all pseudo-tags must be protected so -tagsfromfile fails with
+        WritePseudo => 1,
+        DelCheck => q{"Can't delete"},
+        # all writable pseudo-tags must be protected so -tagsfromfile fails with
         # unrecognized files unless a pseudo tag is specified explicitly
         Protected => 1,
@@ -812 +1215 @@
         PrintConvInv => '$self->InverseDateTime($val)',
     },
+    FileAccessDate => {
+        Description => 'File Access Date/Time',
+        Notes => q{
+            the date/time of last access of the file.  Note that this access time is
+            updated whenever any software, including ExifTool, reads the file
+        },
+        Groups => { 1 => 'System', 2 => 'Time' },
+        ValueConv => 'ConvertUnixTime($val,1)',
+        PrintConv => '$self->ConvertDateTime($val)',
+    },
+    FileCreateDate => {
+        Description => 'File Creation Date/Time',
+        Notes => q{
+            the filesystem creation date/time.  Windows/Mac only.  In Windows, the file
+            creation date/time is preserved by default when writing if Win32API::File
+            and Win32::API are available.  On Mac, this tag is extracted only if it or
+            the MacOS group is specifically requested or the L<RequestAll|../ExifTool.html#RequestAll> API option is
+            set to 2 or higher.  Requires "setfile" for writing on Mac, which may be
+            installed by typing C<xcode-select --install> in the Terminal
+        },
+        Groups => { 1 => 'System', 2 => 'Time' },
+        Writable => 1,
+        WritePseudo => 1,
+        DelCheck => q{"Can't delete"},
+        Protected => 1, # all writable pseudo-tags must be protected!
+        Shift => 'Time',
+        ValueConv => '$^O eq "darwin" ? $val : ConvertUnixTime($val,1)',
+        ValueConvInv => q{
+            return GetUnixTime($val,1) if $^O eq 'MSWin32';
+            return $val if $^O eq 'darwin';
+            warn "This tag is Windows/Mac only\n";
+            return undef;
+        },
+        PrintConv => '$self->ConvertDateTime($val)',
+        PrintConvInv => '$self->InverseDateTime($val)',
+    },
+    FileInodeChangeDate => {
+        Description => 'File Inode Change Date/Time',
+        Notes => q{
+            the date/time when the file's directory information was last changed.
+            Non-Windows systems only
+        },
+        Groups => { 1 => 'System', 2 => 'Time' },
+        ValueConv => 'ConvertUnixTime($val,1)',
+        PrintConv => '$self->ConvertDateTime($val)',
+    },
     FilePermissions => {
-        Groups => { 1 => 'System' },
+        Groups => { 1 => 'System', 2 => 'Other' },
         Notes => q{
             r=read, w=write and x=execute permissions for the file owner, group and
             others.  The ValueConv value is an octal number so bit test operations on
-            this value should be done in octal, ie. "oct($filePermissions) & 0200"
+            this value should be done in octal, eg. 'oct($filePermissions#) & 0200'
         },
-        ValueConv => 'sprintf("%.3o", $val & 0777)',
+        Writable => 1,
+        WritePseudo => 1,
+        DelCheck => q{"Can't delete"},
+        Protected => 1, # all writable pseudo-tags must be protected!
+        ValueConv => 'sprintf("%.3o", $val)',
+        ValueConvInv => 'oct($val & 07777)',
         PrintConv => sub {
-            my ($mask, $str, $val) = (0400, '', oct(shift));
+            my ($mask, $val) = (0400, oct(shift));
+            my %types = (
+                0010000 => 'p',
+                0020000 => 'c',
+                0040000 => 'd',
+                0060000 => 'b',
+                0120000 => 'l',
+                0140000 => 's',
+            );
+            my $str = $types{$val & 0170000} || '-';
             while ($mask) {
                 foreach (qw(r w x)) {
@@ -830 +1293 @@
             return $str;
         },
+        PrintConvInv => sub {
+            my ($bit, $val, $str) = (8, 0, shift);
+            $str = substr($str, 1) if length($str) == 10;
+            return undef if length($str) != 9;
+            while ($bit >= 0) {
+                foreach (qw(r w x)) {
+                    $val |= (1 << $bit) if substr($str, 8-$bit, 1) eq $_;
+                    --$bit;
+                }
+            }
+            return sprintf('%.3o', $val);
+        },
     },
-    MIMEType    => { },
-    ImageWidth  => { },
-    ImageHeight => { },
-    XResolution => { },
-    YResolution => { },
-    MaxVal      => { }, # max pixel value in PPM or PGM image
+    FileAttributes => {
+        Groups => { 1 => 'System', 2 => 'Other' },
+        Notes => q{
+            extracted only if specifically requested or the L<SystemTags|../ExifTool.html#SystemTags> or L<RequestAll|../ExifTool.html#RequestAll> API
+            option is set.  2 or 3 values: 0. File type, 1. Attribute bits, 2. Windows
+            attribute bits if Win32API::File is available
+        },
+        PrintHex => 1,
+        PrintConvColumns => 2,
+        PrintConv => [{ # stat device types (bitmask 0xf000)
+            0x0000 => 'Unknown',
+            0x1000 => 'FIFO',
+            0x2000 => 'Character',
+            0x3000 => 'Mux Character',
+            0x4000 => 'Directory',
+            0x5000 => 'XENIX Named',
+            0x6000 => 'Block',
+            0x7000 => 'Mux Block',
+            0x8000 => 'Regular',
+            0x9000 => 'VxFS Compressed',
+            0xa000 => 'Symbolic Link',
+            0xb000 => 'Solaris Shadow Inode',
+            0xc000 => 'Socket',
+            0xd000 => 'Solaris Door',
+            0xe000 => 'BSD Whiteout',
+        },{ BITMASK => { # stat attribute bits (bitmask 0x0e00)
+            9 => 'Sticky',
+            10 => 'Set Group ID',
+            11 => 'Set User ID',
+        }},{ BITMASK => { # Windows attribute bits
+            0 => 'Read Only',
+            1 => 'Hidden',
+            2 => 'System',
+            3 => 'Volume Label',
+            4 => 'Directory',
+            5 => 'Archive',
+            6 => 'Device',
+            7 => 'Normal',
+            8 => 'Temporary',
+            9 => 'Sparse File',
+            10 => 'Reparse Point',
+            11 => 'Compressed',
+            12 => 'Offline',
+            13 => 'Not Content Indexed',
+            14 => 'Encrypted',
+        }}],
+    },
+    FileDeviceID => {
+        Groups => { 1 => 'System', 2 => 'Other' },
+        %systemTagsNotes,
+        PrintConv => '(($val >> 24) & 0xff) . "." . ($val & 0xffffff)', # (major.minor)
+    },
+    FileDeviceNumber => { Groups => { 1 => 'System', 2 => 'Other' }, %systemTagsNotes },
+    FileInodeNumber  => { Groups => { 1 => 'System', 2 => 'Other' }, %systemTagsNotes },
+    FileHardLinks    => { Groups => { 1 => 'System', 2 => 'Other' }, %systemTagsNotes },
+    FileUserID => {
+        Groups => { 1 => 'System', 2 => 'Other' },
+        Notes => q{
+            extracted only if specifically requested or the L<SystemTags|../ExifTool.html#SystemTags> or L<RequestAll|../ExifTool.html#RequestAll> API
+            option is set.  Returns user ID number with the -n option, or name
+            otherwise.  May be written with either user name or number
+        },
+        Writable => 1,
+        WritePseudo => 1,
+        DelCheck => q{"Can't delete"},
+        Protected => 1, # all writable pseudo-tags must be protected!
+        PrintConv => 'eval { getpwuid($val) } || $val',
+        PrintConvInv => 'eval { getpwnam($val) } || ($val=~/[^0-9]/ ? undef : $val)',
+    },
+    FileGroupID => {
+        Groups => { 1 => 'System', 2 => 'Other' },
+        Notes => q{
+            extracted only if specifically requested or the L<SystemTags|../ExifTool.html#SystemTags> or L<RequestAll|../ExifTool.html#RequestAll> API
+            option is set.  Returns group ID number with the -n option, or name
+            otherwise.  May be written with either group name or number
+        },
+        Writable => 1,
+        WritePseudo => 1,
+        DelCheck => q{"Can't delete"},
+        Protected => 1, # all writable pseudo-tags must be protected!
+        PrintConv => 'eval { getgrgid($val) } || $val',
+        PrintConvInv => 'eval { getgrnam($val) } || ($val=~/[^0-9]/ ? undef : $val)',
+    },
+    FileBlockSize    => { Groups => { 1 => 'System', 2 => 'Other' }, %systemTagsNotes },
+    FileBlockCount   => { Groups => { 1 => 'System', 2 => 'Other' }, %systemTagsNotes },
+    HardLink => {
+        Writable => 1,
+        DelCheck => q{"Can't delete"},
+        WriteOnly => 1,
+        WritePseudo => 1,
+        Protected => 1,
+        Notes => q{
+            this write-only tag is used to create a hard link with the specified name to
+            the source file.  If the source file is edited, copied, renamed or moved in
+            the same operation as writing HardLink, then the link is made to the updated
+            file.  Note that subsequent editing of either hard-linked file by exiftool
+            will break the link unless the -overwrite_original_in_place option is used
+        },
+        ValueConvInv => '$val=~tr/\\\\/\//; $val',
+    },
+    SymLink => {
+        Writable => 1,
+        DelCheck => q{"Can't delete"},
+        WriteOnly => 1,
+        WritePseudo => 1,
+        Protected => 1,
+        Notes => q{
+            this write-only tag is used to create a symbolic link with the specified
+            name to the source file.  If the source file is edited, copied, renamed or
+            moved in the same operation as writing SymLink, then the link is made to the
+            updated file.  The link uses an absolute path unless it is created in the
+            current working directory.  Valid only for file systems that support
+            symbolic links.  Note that subsequent editing of the file via the symbolic
+            link by exiftool will cause the link to be replaced by the edited file
+            without changing the original unless the -overwrite_original_in_place option
+            is used
+        },
+        ValueConvInv => '$val=~tr/\\\\/\//; $val',
+    },
+    MIMEType    => { Notes => 'the MIME type of the source file', Groups => { 2 => 'Other' } },
+    ImageWidth  => { Notes => 'the width of the image in number of pixels' },
+    ImageHeight => { Notes => 'the height of the image in number of pixels' },
+    XResolution => { Notes => 'the horizontal pixel resolution' },
+    YResolution => { Notes => 'the vertical pixel resolution' },
+    MaxVal      => { Notes => 'maximum pixel value in PPM or PGM image' },
     EXIF => {
-        Notes => 'the full EXIF data block from JPEG, PNG, JP2, MIE and MIFF images',
+        Notes => q{
+            the full EXIF data block from JPEG, PNG, JP2, MIE and MIFF images. This tag
+            is generated only if specifically requested
+        },
         Groups => { 0 => 'EXIF', 1 => 'EXIF' },
-        Flags => ['Writable' ,'Protected', 'Binary'],
+        Flags => ['Writable' ,'Protected', 'Binary', 'DelGroup'],
         WriteCheck => q{
             return undef if $val =~ /^(II\x2a\0|MM\0\x2a)/;
@@ -846 +1443 @@
         },
     },
-    ICC_Profile => {
-        Notes => 'the full ICC_Profile data block',
-        Groups => { 0 => 'ICC_Profile', 1 => 'ICC_Profile' },
-        Flags => ['Writable' ,'Protected', 'Binary'],
+    IPTC => {
+        Notes => q{
+            the full IPTC data block.  This tag is generated only if specifically
+            requested
+        },
+        Groups => { 0 => 'IPTC', 1 => 'IPTC' },
+        Flags => ['Writable', 'Protected', 'Binary', 'DelGroup'],
+        Priority => 0,  # so main IPTC (which hopefully comes first) takes priority
         WriteCheck => q{
-            require Image::ExifTool::ICC_Profile;
-            return Image::ExifTool::ICC_Profile::ValidateICC(\$val);
+            return undef if $val =~ /^(\x1c|\0+$)/;
+            return 'Invalid IPTC data';
         },
     },
     XMP => {
-        Notes => 'the full XMP data block',
+        Notes => q{
+            the XMP data block, but note that extended XMP in JPEG images may be split
+            into multiple blocks.  This tag is generated only if specifically requested
+        },
         Groups => { 0 => 'XMP', 1 => 'XMP' },
-        Flags => ['Writable', 'Protected', 'Binary'],
+        Flags => ['Writable', 'Protected', 'Binary', 'DelGroup'],
         Priority => 0,  # so main xmp (which usually comes first) takes priority
         WriteCheck => q{
@@ -865 +1469 @@
         },
     },
+    XML => {
+        Notes => 'the XML data block, extracted for some file types',
+        Groups => { 0 => 'XML', 1 => 'XML' },
+        Binary => 1,
+    },
+    ICC_Profile => {
+        Notes => q{
+            the full ICC_Profile data block.  This tag is generated only if specifically
+            requested
+        },
+        Groups => { 0 => 'ICC_Profile', 1 => 'ICC_Profile' },
+        Flags => ['Writable' ,'Protected', 'Binary', 'DelGroup'],
+        WriteCheck => q{
+            require Image::ExifTool::ICC_Profile;
+            return Image::ExifTool::ICC_Profile::ValidateICC(\$val);
+        },
+    },
     CanonVRD => {
-        Notes => 'the full Canon DPP VRD trailer block',
+        Notes => q{
+            the full Canon DPP VRD trailer block.  This tag is generated only if
+            specifically requested
+        },
         Groups => { 0 => 'CanonVRD', 1 => 'CanonVRD' },
-        Flags => ['Writable' ,'Protected', 'Binary'],
+        Flags => ['Writable' ,'Protected', 'Binary', 'DelGroup'],
         Permanent => 0, # (this is 1 by default for MakerNotes tags)
         WriteCheck => q{
@@ -874 +1498 @@
             return 'Invalid CanonVRD data';
         },
+    },
+    CanonDR4 => {
+        Notes => q{
+            the full Canon DPP version 4 DR4 block.  This tag is generated only if
+            specifically requested
+        },
+        Groups => { 0 => 'CanonVRD', 1 => 'CanonVRD' },
+        Flags => ['Writable' ,'Protected', 'Binary'],
+        Permanent => 0, # (this is 1 by default for MakerNotes tags)
+        WriteCheck => q{
+            return undef if $val =~ /^IIII\x04\0\x04\0/;
+            return 'Invalid CanonDR4 data';
+        },
+    },
+    Adobe => {
+        Notes => q{
+            the JPEG APP14 Adobe segment.  Extracted only if specified. See the
+            L<JPEG Adobe Tags|JPEG.html#Adobe> for more information
+        },
+        Groups => { 0 => 'APP14', 1 => 'Adobe' },
+        WriteGroup => 'Adobe',
+        Flags => ['Writable' ,'Protected', 'Binary'],
     },
     CurrentIPTCDigest => {
@@ -881 +1527 @@
             specified by the L<MWG|http://www.metadataworkinggroup.org/>.  ExifTool
             automates the handling of this tag in the MWG module -- see the
-            L<MWG Tag Name documentation|MWG.html> for details
+            L<MWG Composite Tags|MWG.html> for details
         },
         ValueConv => 'unpack("H*", $val)',
     },
     PreviewImage => {
+        Notes => 'JPEG-format embedded preview image',
+        Groups => { 2 => 'Preview' },
         Writable => 1,
         WriteCheck => '$self->CheckImage(\$val)',
+        WriteGroup => 'All',
         # can't delete, so set to empty string and return no error
         DelCheck => '$val = ""; return undef',
@@ -893 +1542 @@
         RawConv => '$self->ValidateImage(ref $val ? $val : \$val, $tag)',
     },
-    PreviewPNG  => { Binary => 1 },
+    ThumbnailImage => {
+        Groups => { 2 => 'Preview' },
+        Notes => 'JPEG-format embedded thumbnail image',
+        RawConv => '$self->ValidateImage(ref $val ? $val : \$val, $tag)',
+    },
+    OtherImage => {
+        Groups => { 2 => 'Preview' },
+        Notes => 'other JPEG-format embedded image',
+        RawConv => '$self->ValidateImage(ref $val ? $val : \$val, $tag)',
+    },
+    PreviewPNG => {
+        Groups => { 2 => 'Preview' },
+        Notes => 'PNG-format embedded preview image',
+        Binary => 1,
+    },
+    PreviewWMF => {
+        Groups => { 2 => 'Preview' },
+        Notes => 'WMF-format embedded preview image',
+        Binary => 1,
+    },
+    PreviewTIFF => {
+        Groups => { 2 => 'Preview' },
+        Notes => 'TIFF-format embedded preview image',
+        Binary => 1,
+    },
+    PreviewPDF => {
+        Groups => { 2 => 'Preview' },
+        Notes => 'PDF-format embedded preview image',
+        Binary => 1,
+    },
     ExifByteOrder => {
         Writable => 1,
-        Notes => 'only writable for newly created EXIF segments',
+        DelCheck => q{"Can't delete"},
+        Notes => q{
+            represents the byte order of EXIF information.  May be written to set the
+            byte order only for newly created EXIF segments
+        },
         PrintConv => {
             II => 'Little-endian (Intel, II)',
@@ -904 +1586 @@
     ExifUnicodeByteOrder => {
         Writable => 1,
+        WriteOnly => 1,
+        DelCheck => q{"Can't delete"},
         Notes => q{
-            the EXIF specification is particularly vague about the byte ordering for
-            Unicode text, and different applications use different conventions.  By
-            default ExifTool writes Unicode text in EXIF byte order, but this write-only
-            tag may be used to force a specific byte order
+            specifies the byte order to use when writing EXIF Unicode text.  The EXIF
+            specification is particularly vague about this byte ordering, and different
+            applications use different conventions.  By default ExifTool writes Unicode
+            text in EXIF byte order, but this write-only tag may be used to force a
+            specific order.  Applies to the EXIF UserComment tag when writing special
+            characters
         },
         PrintConv => {
@@ -918 +1604 @@
         Description => 'ExifTool Version Number',
         Groups => \%allGroupsExifTool,
+        Notes => 'the version of ExifTool currently running',
     },
-    RAFVersion => { },
+    ProcessingTime => {
+        Groups => { 0 => 'ExifTool', 1 => 'ExifTool', 2 => 'Other' },
+        Notes => q{
+            the clock time in seconds taken by ExifTool to extract information from this
+            file.  Not generated unless specifically requested or the L<RequestAll|../ExifTool.html#RequestAll> API
+            option is set.  Requires Time::HiRes
+        },
+        PrintConv => 'sprintf("%.3g s", $val)',
+    },
+    RAFVersion => { Notes => 'RAF file version number' },
     JPEGDigest => {
         Notes => q{
@@ -926 +1622 @@
             compared to known values in an attempt to deduce the originating software
             based only on the JPEG image data.  For performance reasons, this tag is
-            generated only if specifically requested
+            generated only if specifically requested or the L<RequestAll|../ExifTool.html#RequestAll> API option is set
+            to 3 or higher
         },
     },
+    JPEGQualityEstimate => {
+        Notes => q{
+            an estimate of the IJG JPEG quality setting for the image, calculated from
+            the quantization tables.  For performance reasons, this tag is generated
+            only if specifically requested or the L<RequestAll|../ExifTool.html#RequestAll> API option is set to 3 or
+            higher
+        },
+    },
+    JPEGImageLength => {
+        Notes => q{
+            byte length of JPEG image without metadata.  For performance reasons, this
+            tag is generated only if specifically requested or the L<RequestAll|../ExifTool.html#RequestAll> API option
+            is set to 3 or higher
+        },
+    },
+    # Validate (added from Validate.pm)
     Now => {
         Groups => { 0 => 'ExifTool', 1 => 'ExifTool', 2 => 'Time' },
         Notes => q{
-            the current date/time.  Useful when setting the tag values, ie.
-            C<"-modifydate<now">.  Not generated unless specifically requested
-        },
-        ValueConv => sub {
-            my $time = shift;
-            my @tm = localtime $time;
-            my $tz = Image::ExifTool::TimeZoneString(\@tm, $time);
-            sprintf("%4d:%.2d:%.2d %.2d:%.2d:%.2d%s", $tm[5]+1900, $tm[4]+1, $tm[3],
-                    $tm[2], $tm[1], $tm[0], $tz);
+            the current date/time.  Useful when setting the tag values, eg.
+            C<"-modifydate<now">.  Not generated unless specifically requested or the
+            L<RequestAll|../ExifTool.html#RequestAll> API option is set
         },
         PrintConv => '$self->ConvertDateTime($val)',
     },
-    ID3Size     => { },
+    NewGUID => {
+        Groups => { 0 => 'ExifTool', 1 => 'ExifTool', 2 => 'Other' },
+        Notes => q{
+            generates a new, random GUID with format
+            YYYYmmdd-HHMM-SSNN-PPPP-RRRRRRRRRRRR, where Y=year, m=month, d=day, H=hour,
+            M=minute, S=second, N=file sequence number in hex, P=process ID in hex, and
+            R=random hex number; without dashes with the -n option.  Not generated
+            unless specifically requested or the L<RequestAll|../ExifTool.html#RequestAll> API option is set
+        },
+        PrintConv => '$val =~ s/(.{8})(.{4})(.{4})(.{4})/$1-$2-$3-$4-/; $val',
+    },
+    ID3Size     => { Notes => 'size of the ID3 data block' },
     Geotag => {
         Writable => 1,
+        WriteOnly => 1,
+        WriteNothing => 1,
         AllowGroup => '(exif|gps|xmp|xmp-exif)',
         Notes => q{
             this write-only tag is used to define the GPS track log data or track log
             file name.  Currently supported track log formats are GPX, NMEA RMC/GGA/GLL,
-            KML, IGC, Garmin XML and TCX, and Magellan PMGNTRK.  See
-            L<geotag.html|../geotag.html> for details
+            KML, IGC, Garmin XML and TCX, Magellan PMGNTRK, Honeywell PTNTHPR, Winplus
+            Beacon text, and Bramor gEO log files.  May be set to the special value of
+            "DATETIMEONLY" (all caps) to set GPS date/time tags if no input track points
+            are available.  See L<geotag.html|../geotag.html> for details
         },
         DelCheck => q{
@@ -970 +1692 @@
     Geotime => {
         Writable => 1,
+        WriteOnly => 1,
         AllowGroup => '(exif|gps|xmp|xmp-exif)',
         Notes => q{
             this write-only tag is used to define a date/time for interpolating a
             position in the GPS track specified by the Geotag tag.  Writing this tag
-            causes the following 8 tags to be written:  GPSLatitude, GPSLatitudeRef,
-            GPSLongitude, GPSLongitudeRef, GPSAltitude, GPSAltitudeRef, GPSDateStamp and
-            GPSTimeStamp.  The local system timezone is assumed if the date/time value
-            does not contain a timezone.  May be deleted to delete associated GPS tags.
-            A group name of 'EXIF' or 'XMP' may be specified to write or delete only
-            EXIF or XMP GPS tags.  The value of Geotag must be assigned before this tag
+            causes GPS information to be written into the EXIF or XMP of the target
+            files.  The local system timezone is assumed if the date/time value does not
+            contain a timezone.  May be deleted to delete associated GPS tags.  A group
+            name of "EXIF" or "XMP" may be specified to write or delete only EXIF or XMP
+            GPS tags
         },
         DelCheck => q{
@@ -994 +1716 @@
     Geosync => {
         Writable => 1,
+        WriteOnly => 1,
+        WriteNothing => 1,
         AllowGroup => '(exif|gps|xmp|xmp-exif)',
         Shift => 'Time', # enables "+=" syntax as well as "=+"
@@ -1000 +1724 @@
             synchronization with the GPS clock.  For example, set this to "-12" if the
             camera clock is 12 seconds faster than GPS time.  Input format is
-            "[+-][[[DD ]HH:]MM:]SS[.ss]".  Must be set before Geotime to be effective.
-            Additional features allow calculation of time differences and time drifts,
-            and extraction of synchronization times from image files. See the
-            L<geotagging documentation|../geotag.html> for details
+            "[+-][[[DD ]HH:]MM:]SS[.ss]".  Additional features allow calculation of time
+            differences and time drifts, and extraction of synchronization times from
+            image files.  See the L<geotagging documentation|../geotag.html> for details
         },
         ValueConvInv => q{
@@ -1010 +1733 @@
         },
     },
+    ForceWrite => {
+        Groups => { 0 => '*', 1 => '*', 2 => '*' },
+        Writable => 1,
+        WriteOnly => 1,
+        Notes => q{
+            write-only tag used to force metadata in a file to be rewritten even if no
+            tag values are changed.  May be set to "EXIF", "IPTC", "XMP" or "PNG" to
+            force the corresponding metadata type to be rewritten, "FixBase" to cause
+            EXIF to be rewritten only if the MakerNotes offset base was fixed, or "All"
+            to rewrite all of these metadata types.  Values are case insensitive, and
+            multiple values may be separated with commas, eg. C<-ForceWrite=exif,xmp>
+        },
+    },
+    EmbeddedVideo => { Groups => { 0 => 'Trailer', 2 => 'Video' } },
+    Trailer => {
+        Groups => { 0 => 'Trailer' },
+        Notes => 'the full JPEG trailer data block.  Extracted only if specifically requested',
+        Writable => 1,
+        Protected => 1,
+    },
+);
+
+# tags defined by UserParam option (added at runtime)
+%Image::ExifTool::UserParam = (
+    GROUPS => { 0 => 'UserParam', 1 => 'UserParam', 2 => 'Other' },
+    PRIORITY => 0,
 );
 
@@ -1071 +1820 @@
         Format => 'int8u[2]',
         PrintConv => 'sprintf("%d.%.2d", split(" ",$val))',
+        Mandatory => 1,
     },
     2 => {
@@ -1082 +1832 @@
         },
         Priority => -1,
+        Mandatory => 1,
     },
     3 => {
@@ -1089 +1840 @@
         Priority => -1,
         RawConv => '$$self{JFIFXResolution} = $val',
+        Mandatory => 1,
     },
     5 => {
@@ -1096 +1848 @@
         Priority => -1,
         RawConv => '$$self{JFIFYResolution} = $val',
+        Mandatory => 1,
+    },
+    7 => {
+        Name => 'ThumbnailWidth',
+        RawConv => '$val ? $$self{JFIFThumbnailWidth} = $val : undef',
+    },
+    8 => {
+        Name => 'ThumbnailHeight',
+        RawConv => '$val ? $$self{JFIFThumbnailHeight} = $val : undef',
+    },
+    9 => {
+        Name => 'ThumbnailTIFF',
+        Groups => { 2 => 'Preview' },
+        Format => 'undef[3*($val{7}||0)*($val{8}||0)]',
+        Notes => 'raw RGB thumbnail data, extracted as a TIFF image',
+        RawConv => 'length($val) ? $val : undef',
+        ValueConv => sub {
+            my ($val, $et) = @_;
+            my $len = length $val;
+            return \ "Binary data $len bytes" unless $et->Options('Binary');
+            my $img = MakeTiffHeader($$et{JFIFThumbnailWidth},$$et{JFIFThumbnailHeight},3,8) . $val;
+            return \$img;
+        },
     },
 );
 %Image::ExifTool::JFIF::Extension = (
-    GROUPS => { 0 => 'JFIF', 1 => 'JFIF', 2 => 'Image' },
+    GROUPS => { 0 => 'JFIF', 1 => 'JFXX', 2 => 'Image' },
+    NOTES => 'Thumbnail images extracted from the JFXX segment.',
     0x10 => {
         Name => 'ThumbnailImage',
+        Groups => { 2 => 'Preview' },
+        Notes => 'JPEG-format thumbnail image',
         RawConv => '$self->ValidateImage(\$val,$tag)',
+    },
+    0x11 => { # (untested)
+        Name => 'ThumbnailTIFF',
+        Groups => { 2 => 'Preview' },
+        Notes => 'raw palette-color thumbnail data, extracted as a TIFF image',
+        RawConv => '(length $val > 770 and $val !~ /^\0\0/) ? $val : undef',
+        ValueConv => sub {
+            my ($val, $et) = @_;
+            my $len = length $val;
+            return \ "Binary data $len bytes" unless $et->Options('Binary');
+            my ($w, $h) = unpack('CC', $val);
+            my $img = MakeTiffHeader($w,$h,1,8,undef,substr($val,2,768)) . substr($val,770);
+            return \$img;
+        },
+    },
+    0x13 => {
+        Name => 'ThumbnailTIFF',
+        Groups => { 2 => 'Preview' },
+        Notes => 'raw RGB thumbnail data, extracted as a TIFF image',
+        RawConv => '(length $val > 2 and $val !~ /^\0\0/) ? $val : undef',
+        ValueConv => sub {
+            my ($val, $et) = @_;
+            my $len = length $val;
+            return \ "Binary data $len bytes" unless $et->Options('Binary');
+            my ($w, $h) = unpack('CC', $val);
+            my $img = MakeTiffHeader($w,$h,3,8) . substr($val,2);
+            return \$img;
+        },
     },
 );
@@ -1115 +1921 @@
 );
 
+my %compositeID;    # lookup for new ID's of Composite tags based on original ID
+
 # static private ExifTool variables
 
@@ -1131 +1939 @@
 
 # Clean unnecessary information (line number, LF) from warning
-# Inputs: 0) warning string or undef to use current warning
+# Inputs: 0) warning string or undef to use $evalWarning
 # Returns: cleaned warning
 sub CleanWarning(;$)
@@ -1160 +1968 @@
 
     $self->ClearOptions();      # create default options hash
-    $self->{VALUE} = { };       # must initialize this for warning messages
-    $self->{DEL_GROUP} = { };   # lookup for groups to delete when writing
+    $$self{VALUE} = { };        # must initialize this for warning messages
+    $$self{PATH} = [ ];         # (this too)
+    $$self{DEL_GROUP} = { };    # lookup for groups to delete when writing
+    $$self{SAVE_COUNT} = 0;     # count calls to SaveNewValues()
+    $$self{FILE_SEQUENCE} = 0;  # sequence number for files when reading
 
     # initialize our new groups for writing
@@ -1187 +1998 @@
 #   my $info = ImageInfo($file, 'DateTimeOriginal', 'ImageSize');
 #    - or -
-#   my $exifTool = new Image::ExifTool;
-#   my $info = $exifTool->ImageInfo($file, \@tagList, {Sort=>'Group0'} );
+#   my $et = new Image::ExifTool;
+#   my $info = $et->ImageInfo($file, \@tagList, {Sort=>'Group0'} );
 sub ImageInfo($;@)
 {
@@ -1199 +2010 @@
         $self = new Image::ExifTool;
     }
-    my %saveOptions = %{$self->{OPTIONS}};  # save original options
+    my %saveOptions = %{$$self{OPTIONS}};   # save original options
 
     # initialize file information
-    $self->{FILENAME} = $self->{RAF} = undef;
+    $$self{FILENAME} = $$self{RAF} = undef;
 
     $self->ParseArguments(@_);              # parse our function arguments
@@ -1208 +2019 @@
     my $info = $self->GetInfo(undef);       # get requested information
 
-    $self->{OPTIONS} = \%saveOptions;       # restore original options
+    $$self{OPTIONS} = \%saveOptions;        # restore original options
 
     return $info;   # return requested information
@@ -1216 +2027 @@
 # Get/set ExifTool options
 # Inputs: 0) ExifTool object reference,
-#         1) Parameter name, 2) Value to set the option
+#         1) Parameter name (case insensitive), 2) Value to set the option
 #         3-N) More parameter/value pairs
 # Returns: original value of last option specified
@@ -1228 +2039 @@
     while (@_) {
         my $param = shift;
+        # fix parameter case if necessary
+        unless (exists $$options{$param}) {
+            my ($fixed) = grep /^$param$/i, keys %$options;
+            if ($fixed) {
+                $param = $fixed;
+            } else {
+                $param =~ s/^Group(\d*)$/Group$1/i;
+            }
+        }
         $oldVal = $$options{$param};
+        if (ref $oldVal eq 'HASH' and ($param eq 'Compact' or $param eq 'XMPShorthand')) {
+            # get previous Compact/XMPShorthand setting
+            $oldVal = $$oldVal{$param};
+        }
         last unless @_;
         my $newVal = shift;
@@ -1267 +2091 @@
                     warn "Invalid Charset $newVal\n";
                 }
-            }
+            } elsif ($param eq 'CharsetEXIF' or $param eq 'CharsetFileName' or $param eq 'CharsetRIFF') {
+                $$options{$param} = $newVal;    # only these may be set to a false value
+            } elsif ($param eq 'CharsetQuickTime') {
+                $$options{$param} = 'MacRoman'; # QuickTime defaults to MacRoman
+            } else {
+                $$options{$param} = 'Latin';    # all others default to Latin
+            }
+        } elsif ($param eq 'UserParam') {
+            # clear options if $newVal is undef
+            defined $newVal or $$options{$param} = {}, next;
+            my $table = GetTagTable('Image::ExifTool::UserParam');
+            # allow initialization of entire UserParam hash
+            if (ref $newVal eq 'HASH') {
+                my %newParams;
+                foreach (sort keys %$newVal) {
+                    my $lcTag = lc $_;
+                    $newParams{$lcTag} = $$newVal{$_};
+                    delete $$table{$lcTag};
+                    AddTagToTable($table, $lcTag, $_);
+                }
+                $$options{$param} = \%newParams;
+                next;
+            }
+            my ($force, $paramName);
+            # set/reset single UserParam parameter
+            if ($newVal =~ /(.*?)=(.*)/s) {
+                $paramName = $1;
+                $newVal = $2;
+                $force = 1 if $paramName =~ s/\^$//;
+                $paramName =~ tr/-_a-zA-Z0-9#//dc;
+                $param = lc $paramName;
+            } else {
+                ($param = lc $newVal) =~ tr/-_a-zA-Z0-9#//dc;
+                undef $newVal;
+            }
+            delete $$table{$param};
+            $oldVal = $$options{UserParam}{$param};
+            if (defined $newVal) {
+                if (length $newVal or $force) {
+                    $$options{UserParam}{$param} = $newVal;
+                    AddTagToTable($table, $param, $paramName);
+                } else {
+                    delete $$options{UserParam}{$param};
+                }
+            }
+            # remove alternate version of tag
+            $param .= '#' unless $param =~ s/#$//;
+            delete $$table{$param};
+            delete $$options{UserParam}{$param};
+        } elsif ($param eq 'RequestTags') {
+            if (defined $newVal) {
+                # parse list from delimited string if necessary
+                my @reqList = (ref $newVal eq 'ARRAY') ? @$newVal : ($newVal =~ /[-\w?*:]+/g);
+                ExpandShortcuts(\@reqList);
+                # add to existing list
+                $$options{$param} or $$options{$param} = [ ];
+                foreach (@reqList) {
+                    /^(.*:)?([-\w?*]*)#?$/ or next;
+                    push @{$$options{$param}}, lc($2) if $2;
+                    next unless $1;
+                    # add requested groups with trailing colon
+                    push @{$$options{$param}}, lc($_).':' foreach split /:/, $1;
+                }
+            } else {
+                $$options{$param} = undef;  # clear the list
+            }
+        } elsif ($param eq 'ListJoin') {
+            $$options{$param} = $newVal;
+            # set the old List and ListSep options for backward compatibility
+            if (defined $newVal) {
+                $$options{List} = 0;
+                $$options{ListSep} = $newVal;
+            } else {
+                $$options{List} = 1;
+                # (ListSep must be defined)
+            }
+        } elsif ($param eq 'List') {
+            $$options{$param} = $newVal;
+            # set the new ListJoin option for forward compatibility
+            $$options{ListJoin} = $newVal ? undef : $$options{ListSep};
+        } elsif ($param eq 'Compact' or $param eq 'XMPShorthand') {
+            # set Compact and XMPShorthand options, preserving backward compatibility
+            my ($p, %compact);
+            foreach $p ('Compact','XMPShorthand') {
+                my $val = $param eq $p ? $newVal : $$options{Compact}{$p};
+                if (defined $val) {
+                    my @v = ($val =~ /\w+/g);
+                    my $opt = ($p eq 'Compact') ? \%compactOpt : \%xmpShorthandOpt;
+                    foreach (@v) {
+                        my $set = $$opt{lc $_} or warn("Invalid $p setting '${_}'\n"), return $oldVal;
+                        ref $set or $compact{$set} = 1, next;
+                        $compact{$_} = 1 foreach @$set;
+                    }
+                }
+                $compact{$p} = $val; # preserve most recent setting
+            }
+            $$options{Compact} = $$options{XMPShorthand} = \%compact;
         } else {
             if ($param eq 'Escape') {
@@ -1281 +2201 @@
                 }
                 # must forget saved values since they depend on Escape method
-                $self->{BOTH} = { };
+                $$self{BOTH} = { };
+            } elsif ($param eq 'GlobalTimeShift') {
+                delete $$self{GLOBAL_TIME_OFFSET};  # reset our calculated offset
+            } elsif ($param eq 'TimeZone' and defined $newVal and length $newVal) {
+                $ENV{TZ} = $newVal;
+                eval { require POSIX; POSIX::tzset() };
+            } elsif ($param eq 'Validate') {
+                # load Validate module if Validate option enabled
+                $newVal and require Image::ExifTool::Validate;
             }
             $$options{$param} = $newVal;
@@ -1298 +2226 @@
 
     # create options hash with default values
-    # (commented out options don't need initializing)
     # +-----------------------------------------------------+
     # ! DON'T FORGET!!  When adding any new option, must    !
     # ! decide how it is handled in SetNewValuesFromFile()  !
     # +-----------------------------------------------------+
-    $self->{OPTIONS} = {
-    #   Binary      => undef,   # flag to extract binary values even if tag not specified
-    #   ByteOrder   => undef,   # default byte order when creating EXIF information
+    # (Note: All options must exist in this lookup, even if undefined,
+    # to facilitate case-insensitive options. 'Group#' is handled specially)
+    $$self{OPTIONS} = {
+        Binary      => undef,   # flag to extract binary values even if tag not specified
+        ByteOrder   => undef,   # default byte order when creating EXIF information
         Charset     => 'UTF8',  # character set for converting Unicode characters
+        CharsetEXIF => undef,   # internal EXIF "ASCII" string encoding
+        CharsetFileName => undef,   # external encoding for file names
         CharsetID3  => 'Latin', # internal ID3v1 character set
         CharsetIPTC => 'Latin', # fallback IPTC character set if no CodedCharacterSet
-    #   Compact     => undef,   # compact XMP and IPTC data
+        CharsetPhotoshop => 'Latin', # internal encoding for Photoshop resource names
+        CharsetQuickTime => 'MacRoman', # internal QuickTime string encoding
+        CharsetRIFF => 0,       # internal RIFF string encoding (0=default to Latin)
+        Compact     => { },     # write compact XMP
         Composite   => 1,       # flag to calculate Composite tags
-    #   Compress    => undef,   # flag to write new values as compressed if possible
-    #   CoordFormat => undef,   # GPS lat/long coordinate format
-    #   DateFormat  => undef,   # format for date/time
+        Compress    => undef,   # flag to write new values as compressed if possible
+        CoordFormat => undef,   # GPS lat/long coordinate format
+        DateFormat  => undef,   # format for date/time
         Duplicates  => 1,       # flag to save duplicate tag values
-    #   Escape      => undef,   # escape special characters
-    #   Exclude     => undef,   # tags to exclude
-    #   ExtractEmbedded =>undef,# flag to extract information from embedded documents
-    #   FastScan    => undef,   # flag to avoid scanning for trailer
-    #   FixBase     => undef,   # fix maker notes base offsets
-    #   GeoMaxIntSecs => undef, # geotag maximum interpolation time (secs)
-    #   GeoMaxExtSecs => undef, # geotag maximum extrapolation time (secs)
-    #   GeoMaxHDOP  => undef,   # geotag maximum HDOP
-    #   GeoMaxPDOP  => undef,   # geotag maximum PDOP
-    #   GeoMinSats  => undef,   # geotag minimum satellites
+        Escape      => undef,   # escape special characters
+        Exclude     => undef,   # tags to exclude
+        ExtendedXMP => 1,       # strategy for reading extended XMP
+        ExtractEmbedded =>undef,# flag to extract information from embedded documents
+        FastScan    => undef,   # flag to avoid scanning for trailer
+        Filter      => undef,   # output filter for all tag values
+        FilterW     => undef,   # input filter when writing tag values
+        FixBase     => undef,   # fix maker notes base offsets
+        GeoMaxIntSecs => 1800,  # geotag maximum interpolation time (secs)
+        GeoMaxExtSecs => 1800,  # geotag maximum extrapolation time (secs)
+        GeoMaxHDOP  => undef,   # geotag maximum HDOP
+        GeoMaxPDOP  => undef,   # geotag maximum PDOP
+        GeoMinSats  => undef,   # geotag minimum satellites
+        GeoSpeedRef => undef,   # geotag GPSSpeedRef
+        GlobalTimeShift => undef,   # apply time shift to all extracted date/time values
     #   Group#      => undef,   # return tags for specified groups in family #
+        HexTagIDs   => 0,       # use hex tag ID's in family 7 group names
         HtmlDump    => 0,       # HTML dump (0-3, higher # = bigger limit)
-    #   HtmlDumpBase => undef,  # base address for HTML dump
-    #   IgnoreMinorErrors => undef, # ignore minor errors when reading/writing
+        HtmlDumpBase => undef,  # base address for HTML dump
+        IgnoreMinorErrors => undef, # ignore minor errors when reading/writing
         Lang        => $defaultLang,# localized language for descriptions etc
-    #   LargeFileSupport => undef,  # flag indicating support of 64-bit file offsets
-    #   List        => undef,   # extract lists of PrintConv values into arrays
-        ListSep     => ', ',    # list item separator
-    #   ListSplit   => undef,   # regex for splitting list-type tag values when writing
-    #   MakerNotes  => undef,   # extract maker notes as a block
-    #   MissingTagValue =>undef,# value for missing tags when expanded in expressions
-    #   Password    => undef,   # password for password-protected PDF documents
+        LargeFileSupport => undef,  # flag indicating support of 64-bit file offsets
+        List        => undef,   # extract lists of PrintConv values into arrays [no longer documented]
+        ListItem    => undef,   # used to return a specific item from lists
+        ListJoin    => ', ',    # join lists together with this separator
+        ListSep     => ', ',    # list item separator [no longer documented]
+        ListSplit   => undef,   # regex for splitting list-type tag values when writing
+        MakerNotes  => undef,   # extract maker notes as a block
+        MDItemTags  => undef,   # extract MacOS metadata item tags
+        MissingTagValue =>undef,# value for missing tags when expanded in expressions
+        NoMultiExif => undef,   # raise error when writing multi-segment EXIF
+        NoPDFList   => undef,   # flag to avoid splitting PDF List-type tag values
+        Password    => undef,   # password for password-protected PDF documents
         PrintConv   => 1,       # flag to enable print conversion
-    #   SavePath    => undef,   # (undocumented) save family 5 location path
-    #   ScanForXMP  => undef,   # flag to scan for XMP information in all files
-        Sort        => 'Input', # order to sort found tags (Input, File, Alpha, Group#)
-    #   StrictDate  => undef,   # flag to return undef for invalid date conversions
-    #   Struct      => undef,   # return structures as hash references
+        QuickTimeHandler => 1,  # flag to add mdir Handler to newly created Meta box
+        QuickTimeUTC=> undef,   # assume that QuickTime date/time tags are stored as UTC
+        RequestAll  => undef,   # extract all tags that must be specifically requested
+        RequestTags => undef,   # extra tags to request (on top of those in the tag list)
+        SaveFormat  => undef,   # save family 6 tag TIFF format
+        SavePath    => undef,   # save family 5 location path
+        ScanForXMP  => undef,   # flag to scan for XMP information in all files
+        Sort        => 'Input', # order to sort found tags (Input, File, Tag, Descr, Group#)
+        Sort2       => 'File',  # secondary sort order for tags in a group (File, Tag, Descr)
+        StrictDate  => undef,   # flag to return undef for invalid date conversions
+        Struct      => undef,   # return structures as hash references
+        SystemTags  => undef,   # extract additional File System tags
         TextOut     => \*STDOUT,# file for Verbose/HtmlDump output
+        TimeZone    => undef,   # local time zone
         Unknown     => 0,       # flag to get values of unknown tags (0-2)
+        UserParam   => { },     # user parameters for additional user-defined tag values
+        Validate    => undef,   # perform additional validation
         Verbose     => 0,       # print verbose messages (0-5, higher # = more verbose)
+        WriteMode   => 'wcg',   # enable all write modes by default
+        XAttrTags   => undef,   # extract MacOS extended attribute tags
+        XMPAutoConv => 1,       # automatic conversion of unknown XMP tag values
+        XMPShorthand=> 0,       # (unused, but needed for backward compatibility)
     };
     # keep necessary member variables in sync with options
@@ -1370 +2329 @@
     local $_;
     my $self = shift;
-    my $options = $self->{OPTIONS};     # pointer to current options
-    my (%saveOptions, $reEntry, $rsize);
+    my $options = $$self{OPTIONS};      # pointer to current options
+    my $fast = $$options{FastScan} || 0;
+    my $req = $$self{REQ_TAG_LOOKUP};
+    my $reqAll = $$options{RequestAll} || 0;
+    my (%saveOptions, $reEntry, $rsize, $type, @startTime, $saveOrder, $isDir);
 
     # check for internal ReEntry option to allow recursive calls to ExtractInfo
@@ -1385 +2347 @@
             FILE_TYPE => $$self{FILE_TYPE},
         };
-        $self->{RAF} = new File::RandomAccess($_[0]);
+        $saveOrder = GetByteOrder(),
+        $$self{RAF} = new File::RandomAccess($_[0]);
         $$self{PROCESSED} = { };
         delete $$self{EXIF_DATA};
         delete $$self{EXIF_POS};
     } else {
-        if (defined $_[0] or $options->{HtmlDump}) {
+        if (defined $_[0] or $$options{HtmlDump} or $$req{validate}) {
             %saveOptions = %$options;       # save original options
-    
+
             # require duplicates for html dump
-            $self->Options(Duplicates => 1) if $options->{HtmlDump};
-    
+            $self->Options(Duplicates => 1) if $$options{HtmlDump};
+            # enable Validate option if Validate tag is requested
+            $self->Options(Validate => 1) if $$req{validate};
+
             if (defined $_[0]) {
                 # only initialize filename if called with arguments
-                $self->{FILENAME} = undef;  # name of file (or '' if we didn't open it)
-                $self->{RAF} = undef;       # RandomAccess object reference
-    
+                $$self{FILENAME} = undef;   # name of file (or '' if we didn't open it)
+                $$self{RAF} = undef;        # RandomAccess object reference
+
                 $self->ParseArguments(@_);  # initialize from our arguments
             }
@@ -1407 +2372 @@
         $self->Init();
 
-        delete $self->{MAKER_NOTE_FIXUP};   # fixup information for extracted maker notes
-        delete $self->{MAKER_NOTE_BYTE_ORDER};
+        delete $$self{MAKER_NOTE_FIXUP};    # fixup information for extracted maker notes
+        delete $$self{MAKER_NOTE_BYTE_ORDER};
 
         # return our version number
         $self->FoundTag('ExifToolVersion', "$VERSION$RELEASE");
-        $self->FoundTag('Now', time()) if $self->{REQ_TAG_LOOKUP}{now} or $self->{TAGS_FROM_FILE};
-    }
-    my $filename = $self->{FILENAME};   # image file name ('' if already open)
-    my $raf = $self->{RAF};             # RandomAccess object
+        $self->FoundTag('Now', $self->TimeNow()) if $$req{now} or $reqAll;
+        $self->FoundTag('NewGUID', NewGUID()) if $$req{newguid} or $reqAll;
+        # generate sequence number if necessary
+        $self->FoundTag('FileSequence', $$self{FILE_SEQUENCE}) if $$req{filesequence} or $reqAll;
+
+        if ($$req{processingtime} or $reqAll) {
+            eval { require Time::HiRes; @startTime = Time::HiRes::gettimeofday() };
+            if (not @startTime and $$req{processingtime}) {
+                $self->WarnOnce('Install Time::HiRes to generate ProcessingTime');
+            }
+        }
+
+        ++$$self{FILE_SEQUENCE};        # count files read
+    }
+
+    my $filename = $$self{FILENAME};    # image file name ('' if already open)
+    my $raf = $$self{RAF};              # RandomAccess object
 
     local *EXIFTOOL_FILE;   # avoid clashes with global namespace
@@ -1425 +2403 @@
             unless ($filename eq '-') {
                 # extract file name from pipe if necessary
-                $realname =~ /\|$/ and $realname =~ s/.*?"(.*?)".*/$1/;
-                my ($dir, $name);
-                if (eval 'require File::Basename') {
-                    $dir = File::Basename::dirname($realname);
-                    $name = File::Basename::basename($realname);
-                } else {
-                    ($name = $realname) =~ tr/\\/\//;
-                    # remove path
-                    $dir = length($1) ? $1 : '/' if $name =~ s/(.*)\///;
-                }
+                $realname =~ /\|$/ and $realname =~ s/^.*?"(.*?)".*/$1/s;
+                my ($dir, $name) = SplitFileName($realname);
                 $self->FoundTag('FileName', $name);
                 $self->FoundTag('Directory', $dir) if defined $dir and length $dir;
+                if ($$req{filepath} or
+                   ($reqAll and not $$self{EXCL_TAG_LOOKUP}{filepath}))
+                {
+                    local $SIG{'__WARN__'} = \&SetWarning;
+                    if (eval { require Cwd }) {
+                        my $path = eval { Cwd::abs_path($filename) };
+                        $self->FoundTag('FilePath', $path) if defined $path;
+                    } elsif ($$req{filepath}) {
+                        $self->WarnOnce('The Perl Cwd module must be installed to use FilePath');
+                    }
+                }
                 # get size of resource fork on Mac OS
-                $rsize = -s "$filename/rsrc" if $^O eq 'darwin' and not $$self{IN_RESOURCE};
+                $rsize = -s "$filename/..namedfork/rsrc" if $^O eq 'darwin' and not $$self{IN_RESOURCE};
             }
             # open the file
-            if (open(EXIFTOOL_FILE, $filename)) {
+            if ($self->Open(\*EXIFTOOL_FILE, $filename)) {
                 # create random access file object
                 $raf = new File::RandomAccess(\*EXIFTOOL_FILE);
                 # patch to force pipe to be buffered because seek returns success
                 # in Windows cmd shell pipe even though it really failed
-                $raf->{TESTED} = -1 if $filename eq '-' or $filename =~ /\|$/;
-                $self->{RAF} = $raf;
+                $$raf{TESTED} = -1 if $filename eq '-' or $filename =~ /\|$/;
+                $$self{RAF} = $raf;
+            } elsif ($self->IsDirectory($filename)) {
+                $isDir = 1;
             } else {
                 $self->Error('Error opening file');
@@ -1456 +2439 @@
     }
 
-    if ($raf) {
+    while ($raf or $isDir) {
+        my (@stat, $plainFile);
         if ($reEntry) {
             # we already set these tags
-        } elsif (not $raf->{FILE_PT}) {
+        } elsif (not $raf) {
+            @stat = stat $filename;
+        } elsif (not $$raf{FILE_PT}) {
             # get file size from image in memory
-            $self->FoundTag('FileSize', length ${$raf->{BUFF_PT}});
-        } elsif (-f $raf->{FILE_PT}) {
-            # get file size and last modified time if this is a plain file
-            my $fileSize = -s _;
-            my $fileTime = -M _;
-            my @stat = stat _;
-            $self->FoundTag('FileSize', $fileSize) if defined $fileSize;
-            $self->FoundTag('ResourceForkSize', $rsize) if $rsize;
-            $self->FoundTag('FileModifyDate', $^T - $fileTime*(24*3600)) if defined $fileTime;
-            $self->FoundTag('FilePermissions', $stat[2]) if defined $stat[2];
-        }
-
+            $self->FoundTag('FileSize', length ${$$raf{BUFF_PT}});
+        } elsif (-f $$raf{FILE_PT}) {
+            # get file tags if this is a plain file
+            @stat = stat _;
+            $plainFile = 1;
+        } else {
+            @stat = stat $$raf{FILE_PT};
+        }
+        my $fileSize = $stat[7];
+        $self->FoundTag('FileSize', $stat[7]) if defined $stat[7];
+        $self->FoundTag('ResourceForkSize', $rsize) if $rsize;
+        $self->FoundTag('FileModifyDate', $stat[9]) if defined $stat[9];
+        $self->FoundTag('FileAccessDate', $stat[8]) if defined $stat[8];
+        my $cTag = $^O eq 'MSWin32' ? 'FileCreateDate' : 'FileInodeChangeDate';
+        $self->FoundTag($cTag, $stat[10]) if defined $stat[10];
+        $self->FoundTag('FilePermissions', $stat[2]) if defined $stat[2];
+        # extract more system info if SystemTags option is set
+        if (@stat) {
+            my $sys = $$options{SystemTags} || ($reqAll and not defined $$options{SystemTags});
+            if ($sys or $$req{fileattributes}) {
+                my @attr = ($stat[2] & 0xf000, $stat[2] & 0x0e00);
+                # add Windows file attributes if available
+                if ($^O eq 'MSWin32' and defined $filename and $filename ne '' and $filename ne '-') {
+                    local $SIG{'__WARN__'} = \&SetWarning;
+                    if (eval { require Win32API::File }) {
+                        my $wattr;
+                        my $file = $filename;
+                        if ($self->EncodeFileName($file)) {
+                            $wattr = eval { Win32API::File::GetFileAttributesW($file) };
+                        } else {
+                            $wattr = eval { Win32API::File::GetFileAttributes($file) };
+                        }
+                        push @attr, $wattr if defined $wattr and $wattr != 0xffffffff;
+                    }
+                }
+                $self->FoundTag('FileAttributes', "@attr");
+            }
+            $self->FoundTag('FileDeviceNumber', $stat[0]) if $sys or $$req{filedevicenumber};
+            $self->FoundTag('FileInodeNumber', $stat[1])  if $sys or $$req{fileinodenumber};
+            $self->FoundTag('FileHardLinks', $stat[3])    if $sys or $$req{filehardlinks};
+            $self->FoundTag('FileUserID', $stat[4])       if $sys or $$req{fileuserid};
+            $self->FoundTag('FileGroupID', $stat[5])      if $sys or $$req{filegroupid};
+            $self->FoundTag('FileDeviceID', $stat[6])     if $sys or $$req{filedeviceid};
+            $self->FoundTag('FileBlockSize', $stat[11])   if $sys or $$req{fileblocksize};
+            $self->FoundTag('FileBlockCount', $stat[12])  if $sys or $$req{fileblockcount};
+        }
+        # extract MDItem tags if requested (only on plain files)
+        if ($^O eq 'darwin' and defined $filename and $filename ne '' and defined $fileSize) {
+            my $reqMacOS = ($reqAll > 1 or $$req{'macos:'});
+            my $crDate = ($reqMacOS || $$req{filecreatedate});
+            my $mdItem = ($reqMacOS || $$options{MDItemTags} || grep /^mditem/, keys %$req);
+            my $xattr  = ($reqMacOS || $$options{XAttrTags}  || grep /^xattr/,  keys %$req);
+            if ($crDate or $mdItem or $xattr) {
+                require Image::ExifTool::MacOS;
+                Image::ExifTool::MacOS::GetFileCreateDate($self, $filename) if $crDate;
+                Image::ExifTool::MacOS::ExtractMDItemTags($self, $filename) if $mdItem and $plainFile;
+                Image::ExifTool::MacOS::ExtractXAttrTags($self, $filename) if $xattr;
+            }
+        }
+        # do whatever else we can with directories, then return
+        if ($isDir or (defined $stat[2] and ($stat[2] & 0170000) == 0040000)) {
+            $self->FoundTag('FileType', 'DIR');
+            $self->FoundTag('FileTypeExtension', '');
+            $self->BuildCompositeTags() if $$options{Composite};
+            $raf->Close() if $raf;
+            return 1;
+        }
         # get list of file types to check
-        my ($tiffType, %noMagic);
-        $self->{FILE_EXT} = GetFileExtension($realname);
+        my ($tiffType, %noMagic, $recognizedExt);
+        my $ext = $$self{FILE_EXT} = GetFileExtension($realname);
+        # set $recognizedExt if this file type is recognized by extension only
+        $recognizedExt = $ext if defined $ext and not defined $magicNumber{$ext} and
+                                 defined $moduleName{$ext} and not $moduleName{$ext};
         my @fileTypeList = GetFileType($realname);
+        if ($fast >= 4) {
+            if (@fileTypeList) {
+                $type = shift @fileTypeList;
+                $self->SetFileType($$self{FILE_TYPE} = $type);
+            } else {
+                $self->Error('Unknown file type');
+            }
+            $self->BuildCompositeTags() if $fast == 4 and $$options{Composite};
+            last;   # don't read the file
+        }
         if (@fileTypeList) {
             # add remaining types to end of list so we test them all
             my $pat = join '|', @fileTypeList;
             push @fileTypeList, grep(!/^($pat)$/, @fileTypes);
-            $tiffType = $self->{FILE_EXT};
-            $noMagic{MXF} = 1;  # don't do magic number test on MXF or DV files
-            $noMagic{DV} = 1;
+            $tiffType = $$self{FILE_EXT};
+            unless ($fast == 3) {
+                $noMagic{MXF} = 1;  # don't do magic number test on MXF or DV files
+                $noMagic{DV} = 1;
+            }
         } else {
             # scan through all recognized file types
@@ -1493 +2549 @@
         $raf->BinMode();    # set binary mode before we start reading
         my $pos = $raf->Tell(); # get file position so we can rewind
-        my %dirInfo = ( RAF => $raf, Base => $pos );
         # loop through list of file types to test
-        my ($type, $buff, $seekErr);
-        # read first 1024 bytes of file for testing
-        $raf->Read($buff, 1024) or $buff = '';
+        my ($buff, $seekErr);
+        my %dirInfo = ( RAF => $raf, Base => $pos, TestBuff => \$buff );
+        # read start of file for testing
+        $raf->Read($buff, $testLen) or $buff = '';
         $raf->Seek($pos, 0) or $seekErr = 1;
         until ($seekErr) {
+            my $unkHeader;
             $type = shift @fileTypeList;
             if ($type) {
-                # do quick test for this file type to avoid loading module unnecessarily
-                next if $magicNumber{$type} and $buff !~ /^$magicNumber{$type}/s and
-                        not $noMagic{$type};
+                if ($magicNumber{$type}) {
+                    # do quick test for this file type to avoid loading module unnecessarily
+                    next if $buff !~ /^$magicNumber{$type}/s and not $noMagic{$type};
+                } else {
+                    # keep checking for other types if we recognize this file only by extension
+                    next if defined $moduleName{$type} and not $moduleName{$type};
+                    next if $fast > 2;  # keep checking if we aren't processing the file
+                }
+                next if $weakMagic{$type} and defined $recognizedExt;
+            } elsif (not defined $type) {
+                last;
+            } elsif ($recognizedExt) {
+                $type = $recognizedExt; # set type from recognized file extension only
             } else {
-                last unless defined $type;
                 # last ditch effort to scan past unknown header for JPEG/TIFF
                 next unless $buff =~ /(\xff\xd8\xff|MM\0\x2a|II\x2a\0)/g;
@@ -1513 +2579 @@
                 $dirInfo{Base} = $pos + $skip;
                 $raf->Seek($pos + $skip, 0) or $seekErr = 1, last;
-                $self->Warn("Skipped unknown $skip byte header");
+                $self->Warn("Processing $type-like data after unknown $skip-byte header");
+                $unkHeader = 1 unless $$self{DOC_NUM};
             }
             # save file type in member variable
-            $self->{FILE_TYPE} = $self->{PATH}[0] = $type;
+            $$self{FILE_TYPE} = $type;
             $dirInfo{Parent} = ($type eq 'TIFF') ? $tiffType : $type;
+            # don't process the file when FastScan == 3
+            if ($fast == 3 and not $processType{$type}) {
+                unless ($weakMagic{$type} and (not $ext or $ext ne $type)) {
+                    $self->SetFileType($dirInfo{Parent});
+                }
+                last;
+            }
             my $module = $moduleName{$type};
             $module = $type unless defined $module;
@@ -1531 +2605 @@
                 last;
             }
+            push @{$$self{PATH}}, $type;    # save file type in metadata PATH
+
             # process the file
             no strict 'refs';
-            &$func($self, \%dirInfo) and last;
+            my $result = &$func($self, \%dirInfo);
             use strict 'refs';
 
+            pop @{$$self{PATH}};
+
+            if ($result) {  # all done if successful
+                if ($unkHeader) {
+                    $self->DeleteTag('FileType');
+                    $self->DeleteTag('FileTypeExtension');
+                    $self->DeleteTag('MIMEType');
+                    $self->VPrint(0,"Reset file type due to unknown header\n");
+                }
+                last;
+            }
             # seek back to try again from the same position in the file
             $raf->Seek($pos, 0) or $seekErr = 1, last;
+        }
+        if (not defined $type and not $$self{DOC_NUM}) {
+            # if we were given a single image with a known type there
+            # must be a format error since we couldn't read it, otherwise
+            # it is likely we don't support images of this type
+            my $fileType = GetFileType($realname) || '';
+            my $err;
+            if (not length $buff) {
+                $err = 'File is empty';
+            } else {
+                my $ch = substr($buff, 0, 1);
+                if (length $buff < 16 or $buff =~ /[^\Q$ch\E]/) {
+                    if ($fileType eq 'RAW') {
+                        $err = 'Unsupported RAW file type';
+                    } elsif ($fileType) {
+                        $err = 'File format error';
+                    } else {
+                        $err = 'Unknown file type';
+                    }
+                } else {
+                    # provide some insight into the content of some corrupted files
+                    if ($$self{OPTIONS}{FastScan}) {
+                        $err = 'File header is all';
+                    } else {
+                        my $num = 0;
+                        for (;;) {
+                            $raf->Read($buff, 65536) or undef($num), last;
+                            $buff =~ /[^\Q$ch\E]/g and $num += pos($buff) - 1, last;
+                            $num += length($buff);
+                        }
+                        if ($num) {
+                            $err = 'First ' . ConvertFileSize($num) . ' of file is';
+                        } else {
+                            $err = 'Entire file is';
+                        }
+                    }
+                    if ($ch eq "\0") {
+                        $err .= ' binary zeros';
+                    } elsif ($ch eq ' ') {
+                        $err .= ' ASCII spaces';
+                    } elsif ($ch =~ /[a-zA-Z0-9]/) {
+                        $err .= " ASCII '${ch}' characters";
+                    } else {
+                        $err .= sprintf(" binary 0x%.2x's", ord $ch);
+                    }
+                }
+            }
+            $self->Error($err);
         }
         if ($seekErr) {
             $self->Error('Error seeking in file');
         } elsif ($self->Options('ScanForXMP') and (not defined $type or
-            (not $self->Options('FastScan') and not $$self{FoundXMP})))
+            (not $fast and not $$self{FoundXMP})))
         {
             # scan for XMP
@@ -1549 +2684 @@
             Image::ExifTool::XMP::ScanForXMP($self, $raf) and $type = '';
         }
-        unless (defined $type) {
-            # if we were given a single image with a known type there
-            # must be a format error since we couldn't read it, otherwise
-            # it is likely we don't support images of this type
-            my $fileType = GetFileType($realname);
-            my $err;
-            if (not $fileType) {
-                $err = 'Unknown file type';
-            } elsif ($fileType eq 'RAW') {
-                $err = 'Unsupported RAW file type';
-            } else {
-                $err = 'File format error';
-            }
-            $self->Error($err);
-        }
         # extract binary EXIF data block only if requested
-        if (defined $self->{EXIF_DATA} and length $$self{EXIF_DATA} > 16 and
-            ($self->{REQ_TAG_LOOKUP}{exif} or $self->{OPTIONS}{Binary}))
+        if (defined $$self{EXIF_DATA} and length $$self{EXIF_DATA} > 16 and
+            ($$req{exif} or
+            # (not extracted normally, so check TAGS_FROM_FILE)
+            ($$self{TAGS_FROM_FILE} and not $$self{EXCL_TAG_LOOKUP}{exif})))
         {
-            $self->FoundTag('EXIF', $self->{EXIF_DATA});
+            $self->FoundTag('EXIF', $$self{EXIF_DATA});
         }
         unless ($reEntry) {
-            $self->{PATH} = [ ];    # reset PATH
+            $$self{PATH} = [ ];     # reset PATH
             # calculate Composite tags
-            $self->BuildCompositeTags() if $options->{Composite};
+            $self->BuildCompositeTags() if $$options{Composite};
             # do our HTML dump if requested
-            if ($self->{HTML_DUMP}) {
+            if ($$self{HTML_DUMP}) {
                 $raf->Seek(0, 2);   # seek to end of file
-                $self->{HTML_DUMP}->FinishTiffDump($self, $raf->Tell());
-                my $pos = $options->{HtmlDumpBase};
-                $pos = ($self->{FIRST_EXIF_POS} || 0) unless defined $pos;
-                my $dataPt = defined $self->{EXIF_DATA} ? \$self->{EXIF_DATA} : undef;
-                undef $dataPt if defined $self->{EXIF_POS} and $pos != $self->{EXIF_POS};
-                my $success = $self->{HTML_DUMP}->Print($raf, $dataPt, $pos,
-                    $options->{TextOut}, $options->{HtmlDump},
-                    $self->{FILENAME} ? "HTML Dump ($self->{FILENAME})" : 'HTML Dump');
-                $self->Warn("Error reading $self->{HTML_DUMP}{ERROR}") if $success < 0;
+                $$self{HTML_DUMP}->FinishTiffDump($self, $raf->Tell());
+                my $pos = $$options{HtmlDumpBase};
+                $pos = ($$self{FIRST_EXIF_POS} || 0) unless defined $pos;
+                my $dataPt = defined $$self{EXIF_DATA} ? \$$self{EXIF_DATA} : undef;
+                undef $dataPt if defined $$self{EXIF_POS} and $pos != $$self{EXIF_POS};
+                undef $dataPt if $$self{ExtendedEXIF}; # can't use EXIF block if not contiguous
+                my $success = $$self{HTML_DUMP}->Print($raf, $dataPt, $pos,
+                    $$options{TextOut}, $$options{HtmlDump},
+                    $$self{FILENAME} ? "HTML Dump ($$self{FILENAME})" : 'HTML Dump');
+                $self->Warn("Error reading $$self{HTML_DUMP}{ERROR}") if $success < 0;
             }
         }
@@ -1591 +2714 @@
             $raf->Close();  # close the file if we opened it
             # process the resource fork as an embedded file on Mac filesystems
-            if ($rsize and $options->{ExtractEmbedded}) {
+            if ($rsize and $$options{ExtractEmbedded}) {
                 local *RESOURCE_FILE;
-                if (open(RESOURCE_FILE, "$filename/rsrc")) {
+                if ($self->Open(\*RESOURCE_FILE, "$filename/..namedfork/rsrc")) {
                     $$self{DOC_NUM} = $$self{DOC_COUNT} + 1;
                     $$self{IN_RESOURCE} = 1;
@@ -1604 +2727 @@
             }
         }
+        last;   # (loop was a cheap "goto")
+    }
+
+    # generate Validate tag if requested
+    if ($$options{Validate} and not $reEntry) {
+        Image::ExifTool::Validate::FinishValidate($self, $$req{validate});
+    }
+
+    @startTime and $self->FoundTag('ProcessingTime', Time::HiRes::tv_interval(\@startTime));
+
+    # add user-defined parameters that ended with '!'
+    if (%{$$options{UserParam}}) {
+        my $doMsg = $$options{Verbose};
+        my $table = GetTagTable('Image::ExifTool::UserParam');
+        foreach (sort keys %{$$options{UserParam}}) {
+            next unless /#$/;
+            if ($doMsg) {
+                $self->VPrint(0, "UserParam tags:\n");
+                undef $doMsg;
+            }
+            $self->HandleTag($table, $_, $$options{UserParam}{$_});
+        }
     }
 
     # restore original options
-    %saveOptions and $self->{OPTIONS} = \%saveOptions;
+    %saveOptions and $$self{OPTIONS} = \%saveOptions;
 
     if ($reEntry) {
         # restore necessary members when exiting re-entrant code
         $$self{$_} = $$reEntry{$_} foreach keys %$reEntry;
-    }
-
-    return exists $self->{VALUE}{Error} ? 0 : 1;
+        SetByteOrder($saveOrder);
+    }
+
+    # ($type may be undef without an Error when processing sub-documents)
+    return 0 if not defined $type or exists $$self{VALUE}{Error};
+    return 1;
 }
 
@@ -1633 +2781 @@
 
     unless (@_ and not defined $_[0]) {
-        %saveOptions = %{$self->{OPTIONS}}; # save original options
+        %saveOptions = %{$$self{OPTIONS}}; # save original options
         # must set FILENAME so it isn't parsed from the arguments
-        $self->{FILENAME} = '' unless defined $self->{FILENAME};
+        $$self{FILENAME} = '' unless defined $$self{FILENAME};
         $self->ParseArguments(@_);
     }
 
     # get reference to list of tags for which we will return info
-    my ($rtnTags, $byValue) = $self->SetFoundTags();
+    my ($rtnTags, $byValue, $wildTags) = $self->SetFoundTags();
 
     # build hash of tag information
     my (%info, %ignored);
-    my $conv = $self->{OPTIONS}{PrintConv} ? 'PrintConv' : 'ValueConv';
+    my $conv = $$self{OPTIONS}{PrintConv} ? 'PrintConv' : 'ValueConv';
     foreach (@$rtnTags) {
         my $val = $self->GetValue($_, $conv);
@@ -1652 +2800 @@
 
     # override specified tags with ValueConv value if necessary
-    if (@$byValue and $conv ne 'ValueConv') {
+    if (@$byValue) {
         # first determine the number of times each non-ValueConv value is used
         my %nonVal;
@@ -1665 +2813 @@
             # generate a new tag key like "Tag #" or "Tag #(1)"
             $vtag =~ s/( |$)/ #/;
-            unless (defined $self->{VALUE}->{$vtag}) {
-                $self->{VALUE}{$vtag} = $self->{VALUE}{$tag};
-                $self->{TAG_INFO}{$vtag} = $self->{TAG_INFO}{$tag};
-                $self->{TAG_EXTRA}{$vtag} = $self->{TAG_EXTRA}{$tag};
-                $self->{FILE_ORDER}{$vtag} = $self->{FILE_ORDER}{$tag};
+            unless (defined $$self{VALUE}{$vtag}) {
+                $$self{VALUE}{$vtag} = $$self{VALUE}{$tag};
+                $$self{TAG_INFO}{$vtag} = $$self{TAG_INFO}{$tag};
+                $$self{TAG_EXTRA}{$vtag} = $$self{TAG_EXTRA}{$tag};
+                $$self{FILE_ORDER}{$vtag} = $$self{FILE_ORDER}{$tag};
                 # remove existing PrintConv entry unless we are using it too
                 delete $info{$tag} unless $nonVal{$tag};
@@ -1679 +2827 @@
 
     # remove ignored tags from the list
-    my $reqTags = $self->{REQUESTED_TAGS} || [ ];
-    if (%ignored and not @$reqTags) {
-        my @goodTags;
-        foreach (@$rtnTags) {
-            push @goodTags, $_ unless $ignored{$_};
-        }
-        $rtnTags = $self->{FOUND_TAGS} = \@goodTags;
+    my $reqTags = $$self{REQUESTED_TAGS} || [ ];
+    if (%ignored) {
+        if (not @$reqTags) {
+            my @goodTags;
+            foreach (@$rtnTags) {
+                push @goodTags, $_ unless $ignored{$_};
+            }
+            $rtnTags = $$self{FOUND_TAGS} = \@goodTags;
+        } elsif (@$wildTags) {
+            # only remove tags specified by wildcard
+            my @goodTags;
+            my $i = 0;
+            foreach (@$rtnTags) {
+                if (@$wildTags and $i == $$wildTags[0]) {
+                    shift @$wildTags;
+                    push @goodTags, $_ unless $ignored{$_};
+                } else {
+                    push @goodTags, $_;
+                }
+                ++$i;
+            }
+            $rtnTags = $$self{FOUND_TAGS} = \@goodTags;
+        }
     }
 
     # return sorted tag list if provided with a list reference
-    if ($self->{IO_TAG_LIST}) {
+    if ($$self{IO_TAG_LIST}) {
         # use file order by default if no tags specified
         # (no such thing as 'Input' order in this case)
-        my $sortOrder = $self->{OPTIONS}{Sort};
-        unless (@$reqTags or ($sortOrder and $sortOrder ne 'Input')) {
-            $sortOrder = 'File';
-        }
+        my $sort = $$self{OPTIONS}{Sort};
+        $sort = 'File' unless @$reqTags or ($sort and $sort ne 'Input');
         # return tags in specified sort order
-        @{$self->{IO_TAG_LIST}} = $self->GetTagList($rtnTags, $sortOrder);
+        @{$$self{IO_TAG_LIST}} = $self->GetTagList($rtnTags, $sort, $$self{OPTIONS}{Sort2});
     }
 
     # restore original options
-    %saveOptions and $self->{OPTIONS} = \%saveOptions;
+    %saveOptions and $$self{OPTIONS} = \%saveOptions;
 
     return \%info;
-}
-
-#------------------------------------------------------------------------------
-# Combine information from a list of info hashes
-# Unless Duplicates is enabled, first entry found takes priority
-# Inputs: 0) ExifTool object reference, 1-N) list of info hash references
-# Returns: Combined information hash reference
-sub CombineInfo($;@)
-{
-    local $_;
-    my $self = shift;
-    my (%combinedInfo, $info, $tag, %haveInfo);
-
-    if ($self->{OPTIONS}{Duplicates}) {
-        while ($info = shift) {
-            foreach $tag (keys %$info) {
-                $combinedInfo{$tag} = $$info{$tag};
-            }
-        }
-    } else {
-        while ($info = shift) {
-            foreach $tag (keys %$info) {
-                my $tagName = GetTagName($tag);
-                next if $haveInfo{$tagName};
-                $haveInfo{$tagName} = 1;
-                $combinedInfo{$tag} = $$info{$tag};
-            }
-        }
-    }
-    return \%combinedInfo;
 }
 
@@ -1740 +2872 @@
 #         1) [optional] reference to info hash or tag list ref (default is found tags)
 #         2) [optional] sort order ('File', 'Input', ...)
+#         3) [optional] secondary sort order
 # Returns: List of tags in specified order
-sub GetTagList($;$$)
+sub GetTagList($;$$$)
 {
     local $_;
-    my ($self, $info, $sortOrder) = @_;
+    my ($self, $info, $sort, $sort2) = @_;
 
     my $foundTags;
@@ -1753 +2886 @@
         $foundTags = $info;
     }
-    my $fileOrder = $self->{FILE_ORDER};
+    my $fileOrder = $$self{FILE_ORDER};
 
     if ($foundTags) {
@@ -1763 +2896 @@
         }
     } else {
-        $sortOrder = $info if $info and not $sortOrder;
-        $foundTags = $self->{FOUND_TAGS} || $self->SetFoundTags() or return undef;
-    }
-    $sortOrder or $sortOrder = $self->{OPTIONS}{Sort};
+        $sort = $info if $info and not $sort;
+        $foundTags = $$self{FOUND_TAGS} || $self->SetFoundTags() or return undef;
+    }
+    $sort or $sort = $$self{OPTIONS}{Sort};
 
     # return original list if no sort order specified
-    return @$foundTags unless $sortOrder and $sortOrder ne 'Input';
-
-    if ($sortOrder eq 'Alpha') {
+    return @$foundTags unless $sort and $sort ne 'Input';
+
+    if ($sort eq 'Tag' or $sort eq 'Alpha') {
         return sort @$foundTags;
-    } elsif ($sortOrder =~ /^Group(\d*(:\d+)*)/) {
+    } elsif ($sort =~ /^Group(\d*(:\d+)*)/) {
         my $family = $1 || 0;
         # want to maintain a basic file order with the groups
@@ -1786 +2919 @@
             $groupOrder{$tag} = $num;
         }
+        $sort2 or $sort2 = $$self{OPTIONS}{Sort2};
+        if ($sort2) {
+            if ($sort2 eq 'Tag' or $sort2 eq 'Alpha') {
+                return sort { $groupOrder{$a} <=> $groupOrder{$b} or $a cmp $b } @$foundTags;
+            } elsif ($sort2 eq 'Descr') {
+                my $desc = $self->GetDescriptions($foundTags);
+                return sort { $groupOrder{$a} <=> $groupOrder{$b} or
+                              $$desc{$a} cmp $$desc{$b} } @$foundTags;
+            }
+        }
         return sort { $groupOrder{$a} <=> $groupOrder{$b} or
                       $$fileOrder{$a} <=> $$fileOrder{$b} } @$foundTags;
+    } elsif ($sort eq 'Descr') {
+        my $desc = $self->GetDescriptions($foundTags);
+        return sort { $$desc{$a} cmp $$desc{$b} } @$foundTags;
     } else {
         return sort { $$fileOrder{$a} <=> $$fileOrder{$b} } @$foundTags;
@@ -1796 +2942 @@
 # Get list of found tags in specified sort order
 # Inputs: 0) ExifTool object reference, 1) sort order ('File', 'Input', ...)
+#         2) secondary sort order
 # Returns: List of tag keys in specified order
 # Notes: If not specified, sort order is taken from OPTIONS
-sub GetFoundTags($;$)
+sub GetFoundTags($;$$)
 {
     local $_;
-    my ($self, $sortOrder) = @_;
-    my $foundTags = $self->{FOUND_TAGS} || $self->SetFoundTags() or return undef;
-    return $self->GetTagList($foundTags, $sortOrder);
+    my ($self, $sort, $sort2) = @_;
+    my $foundTags = $$self{FOUND_TAGS} || $self->SetFoundTags() or return undef;
+    return $self->GetTagList($foundTags, $sort, $sort2);
 }
 
@@ -1819 +2966 @@
 # Get tag value
 # Inputs: 0) ExifTool object reference
-#         1) tag key (or flattened tagInfo for getting field values, not part of public API)
-#         2) [optional] Value type: PrintConv, ValueConv, Both or Raw, the default
+#         1) tag key or tag name with optional group names (case sensitive)
+#            (or flattened tagInfo for getting field values, not part of public API)
+#         2) [optional] Value type: PrintConv, ValueConv, Both, Raw or Rational, the default
 #            is PrintConv or ValueConv, depending on the PrintConv option setting
 #         3) raw field value (not part of public API)
@@ -1830 +2978 @@
     my ($self, $tag, $type) = @_; # plus: ($fieldValue)
     my (@convTypes, $tagInfo, $valueConv, $both);
-
+    my $rawValue = $$self{VALUE};
+
+    # get specific tag key if tag has a group name
+    if ($tag =~ /^(.*):(.+)/) {
+        my ($gp, $tg) = ($1, $2);
+        my ($i, $key, @keys);
+        # build list of tag keys in the order of priority (no index
+        # is top priority, otherwise higher index is higher priority)
+        for ($key=$tg, $i=$$self{DUPL_TAG}{$tg} || 0; ; --$i) {
+            push @keys, $key if defined $$rawValue{$key};
+            last if $i <= 0;
+            $key = "$tg ($i)";
+        }
+        if (@keys) {
+            $key = $self->GroupMatches($gp, \@keys);
+            $tag = $key if $key;
+        }
+    }
     # figure out what conversions to do
-    $type or $type = $self->{OPTIONS}{PrintConv} ? 'PrintConv' : 'ValueConv';
+    if ($type) {
+        return $$self{RATIONAL}{$tag} if $type eq 'Rational';
+    } else {
+        $type = $$self{OPTIONS}{PrintConv} ? 'PrintConv' : 'ValueConv';
+    }
 
     # start with the raw value
-    my $value = $self->{VALUE}{$tag};
+    my $value = $$rawValue{$tag};
     if (not defined $value) {
-        return wantarray ? () : undef unless ref $tag;
+        return () unless ref $tag;
         # get the value of a structure field
         $tagInfo = $tag;
@@ -1848 +3017 @@
         }
     } else {
-        $tagInfo = $self->{TAG_INFO}{$tag};
+        $tagInfo = $$self{TAG_INFO}{$tag};
         if ($$tagInfo{Struct} and ref $value) {
             # must load XMPStruct.pl just in case (should already be loaded if
@@ -1856 +3025 @@
             # convert strucure field values
             unless ($type eq 'Both') {
-                # (note: ConvertStruct handles the escape too if necessary)
+                # (note: ConvertStruct handles the filtering and escaping too if necessary)
                 return Image::ExifTool::XMP::ConvertStruct($self,$tagInfo,$value,$type);
             }
@@ -1866 +3035 @@
         if ($type ne 'Raw') {
             # use values we calculated already if we stored them
-            $both = $self->{BOTH}{$tag};
+            $both = $$self{BOTH}{$tag};
             if ($both) {
                 if ($type eq 'PrintConv') {
@@ -1887 +3056 @@
     foreach $convType (@convTypes) {
         # don't convert a scalar reference or structure
-        last if ref $value eq 'SCALAR';
+        last if ref $value eq 'SCALAR' and not $$tagInfo{ConvertBinary};
         my $conv = $$tagInfo{$convType};
         unless (defined $conv) {
@@ -1895 +3064 @@
             } else {
                 # use PRINT_CONV from tag table if PrintConv doesn't exist
-                next unless defined($conv = $tagInfo->{Table}{PRINT_CONV});
+                next unless defined($conv = $$tagInfo{Table}{PRINT_CONV});
                 next if exists $$tagInfo{$convType};
             }
@@ -1906 +3075 @@
             $convList = $conv;
             $conv = $$convList[0];
-            my @valList = split ' ', $value;
+            my @valList = (ref $value eq 'ARRAY') ? @$value : split ' ', $value;
             # reorganize list if specified (Note: The writer currently doesn't
             # relist values, so they may be grouped but the order must not change)
@@ -1928 +3097 @@
                 $value = \@valList;
             }
+            return () unless @$value;
         }
         # initialize array so we can iterate over values in list
         if (ref $value eq 'ARRAY') {
-            $i = 0;
-            $vals = $value;
-            $val = $$vals[0];
+            if (defined $$tagInfo{RawJoin}) {
+                $val = join ' ', @$value;
+            } else {
+                $i = 0;
+                $vals = $value;
+                $val = $$vals[0];
+            }
         } else {
             $val = $value;
@@ -1945 +3119 @@
                     my $oldEscape = $$self{ESCAPE_PROC};
                     delete $$self{ESCAPE_PROC};
+                    # temporarily delete filter so it isn't applied to the Require'd values
+                    my $oldFilter = $$self{OPTIONS}{Filter};
+                    delete $$self{OPTIONS}{Filter};
                     foreach (keys %$val) {
-                        $raw[$_] = $self->{VALUE}{$$val{$_}};
+                        next unless defined $$val{$_};
+                        $raw[$_] = $$rawValue{$$val{$_}};
                         ($val[$_], $prt[$_]) = $self->GetValue($$val{$_}, 'Both');
-                        next if defined $val[$_] or not $tagInfo->{Require}{$_};
+                        next if defined $val[$_] or not $$tagInfo{Require}{$_};
+                        $$self{OPTIONS}{Filter} = $oldFilter if defined $oldFilter;
                         $$self{ESCAPE_PROC} = $oldEscape;
-                        return wantarray ? () : undef;
+                        return ();
                     }
+                    $$self{OPTIONS}{Filter} = $oldFilter if defined $oldFilter;
                     $$self{ESCAPE_PROC} = $oldEscape;
                     # set $val to $val[0], or \@val for a CODE ref conversion
@@ -1958 +3138 @@
                 if (ref $conv eq 'HASH') {
                     # look up converted value in hash
-                    my $lc;
-                    if (defined($value = $$conv{$val})) {
-                        # override with our localized language PrintConv if available
-                        if ($$self{CUR_LANG} and $convType eq 'PrintConv' and
-                            # (no need to check for lang-alt tag names -- they won't have a PrintConv)
-                            ref($lc = $self->{CUR_LANG}{$$tagInfo{Name}}) eq 'HASH' and
-                            ($lc = $$lc{PrintConv}) and ($lc = $$lc{$value}))
-                        {
-                            $value = $self->Decode($lc, 'UTF8');
+                    if (not defined($value = $$conv{$val})) {
+                        if ($$conv{BITMASK}) {
+                            $value = DecodeBits($val, $$conv{BITMASK}, $$tagInfo{BitsPerWord});
+                        } else {
+                             # use alternate conversion routine if available
+                            if ($$conv{OTHER}) {
+                                local $SIG{'__WARN__'} = \&SetWarning;
+                                undef $evalWarning;
+                                $value = &{$$conv{OTHER}}($val, undef, $conv);
+                                $self->Warn("$convType $tag: " . CleanWarning()) if $evalWarning;
+                            }
+                            if (not defined $value) {
+                                if ($$tagInfo{PrintHex} and $val and IsInt($val) and
+                                    $convType eq 'PrintConv')
+                                {
+                                    $value = sprintf('Unknown (0x%x)',$val);
+                                } else {
+                                    $value = "Unknown ($val)";
+                                }
+                            }
                         }
-                    } else {
-                        if ($$conv{BITMASK}) {
-                            $value = DecodeBits($val, $$conv{BITMASK});
-                            # override with localized language strings
-                            if (defined $value and $$self{CUR_LANG} and $convType eq 'PrintConv' and
-                                ref($lc = $self->{CUR_LANG}{$$tagInfo{Name}}) eq 'HASH' and
-                                ($lc = $$lc{PrintConv}))
-                            {
-                                my @vals = split ', ', $value;
-                                foreach (@vals) {
-                                    $_ = $$lc{$_} if defined $$lc{$_};
-                                }
-                                $value = join ', ', @vals;
+                    }
+                    # override with our localized language PrintConv if available
+                    my $tmp;
+                    if ($$self{CUR_LANG} and $convType eq 'PrintConv' and
+                        # (no need to check for lang-alt tag names -- they won't have a PrintConv)
+                        ref($tmp = $$self{CUR_LANG}{$$tagInfo{Name}}) eq 'HASH' and
+                        ($tmp = $$tmp{PrintConv}))
+                    {
+                        if ($$conv{BITMASK} and not defined $$conv{$val}) {
+                            my @vals = split ', ', $value;
+                            foreach (@vals) {
+                                $_ = $$tmp{$_} if defined $$tmp{$_};
                             }
-                        } elsif (not $$conv{OTHER} or
-                                 # use alternate conversion routine if available
-                                 not defined($value = &{$$conv{OTHER}}($val, undef, $conv)))
-                        {
-                            if (($$tagInfo{PrintHex} or
-                                ($$tagInfo{Mask} and not defined $$tagInfo{PrintHex}))
-                                and $val and IsInt($val) and $convType eq 'PrintConv')
-                            {
-                                $val = sprintf('0x%x',$val);
-                            }
-                            $value = "Unknown ($val)";
+                            $value = join ', ', @vals;
+                        } elsif (defined($tmp = $$tmp{$value})) {
+                            $value = $self->Decode($tmp, 'UTF8');
                         }
                     }
@@ -2012 +3194 @@
             }
             last unless $vals;
+            # must store a separate copy of each binary data value in the list
+            if (ref $value eq 'SCALAR') {
+                my $tval = $$value;
+                $value = \$tval;
+            }
             # save this converted value and step to next value in list
             push @values, $value if defined $value;
@@ -2019 +3206 @@
             }
             $val = $$vals[$i];
-            $conv = $$convList[$i] if $convList;
+            if ($convList) {
+                my $nextConv = $$convList[$i];
+                if ($nextConv and $nextConv eq 'REPEAT') {
+                    undef $convList;
+                } else {
+                    $conv = $nextConv;
+                }
+            }
         }
         # return undefined now if no value
-        return wantarray ? () : undef unless defined $value;
+        return () unless defined $value;
         # join back into single value if split for conversion list
         if ($convList and ref $value eq 'ARRAY') {
@@ -2031 +3225 @@
         # save both (unescaped) values because we often need them again
         # (Composite tags need "Both" and often Require one tag for various Composite tags)
-        $self->{BOTH}{$tag} = [ $valueConv, $value ] unless $both;
+        $$self{BOTH}{$tag} = [ $valueConv, $value ] unless $both;
         # escape values if necessary
         if ($$self{ESCAPE_PROC}) {
@@ -2044 +3238 @@
             $valueConv = $value;
         }
+        $self->Filter($$self{OPTIONS}{Filter}, \$value);
         # return Both values as a list (ValueConv, PrintConv)
         return ($valueConv, $value);
@@ -2050 +3245 @@
     DoEscape($value, $$self{ESCAPE_PROC}) if $$self{ESCAPE_PROC};
 
+    # filter if necessary
+    $self->Filter($$self{OPTIONS}{Filter}, \$value) if $$self{OPTIONS}{Filter} and $type eq 'PrintConv';
+
     if (ref $value eq 'ARRAY') {
-        # return array if requested
-        return @$value if wantarray;
-        # return list reference for Raw, ValueConv or if List or not a list of scalars
-        return $value if $type ne 'PrintConv' or $self->{OPTIONS}{List} or ref $$value[0];
-        # otherwise join in comma-separated string
-        $value = join $self->{OPTIONS}{ListSep}, @$value;
+        if (defined $$self{OPTIONS}{ListItem}) {
+            $value = $$value[$$self{OPTIONS}{ListItem}];
+        } elsif (wantarray) {
+            # return array if requested
+            return @$value;
+        } elsif ($type eq 'PrintConv' and not $$self{OPTIONS}{List} and not ref $$value[0]) {
+            # join PrintConv values in comma-separated string if List option not used
+            # and list contains simple scalars (otherwise return ARRAY ref)
+            $value = join $$self{OPTIONS}{ListSep}, @$value;
+        }
     }
     return $value;
@@ -2064 +3266 @@
 # Get tag identification number
 # Inputs: 0) ExifTool object reference, 1) tag key
-# Returns: Scalar context: Tag ID if available, otherwise ''
-#          List context: 0) Tag ID (or ''), 1) language code (or undef)
+# Returns: Scalar context: tag ID if available, otherwise ''
+#          List context: 0) tag ID (or ''), 1) language code (or undef)
 sub GetTagID($$)
 {
     my ($self, $tag) = @_;
-    my $tagInfo = $self->{TAG_INFO}{$tag};
+    my $tagInfo = $$self{TAG_INFO}{$tag};
     return '' unless $tagInfo and defined $$tagInfo{TagID};
     return ($$tagInfo{TagID}, $$tagInfo{LangCode}) if wantarray;
     return $$tagInfo{TagID};
-}
-
-#------------------------------------------------------------------------------
-# Get tag table name
-# Inputs: 0) ExifTool object reference, 1) tag key
-# Returns: Table name if available, otherwise ''
-sub GetTableName($$)
-{
-    my ($self, $tag) = @_;
-    my $tagInfo = $self->{TAG_INFO}{$tag} or return '';
-    return $tagInfo->{Table}{SHORT_NAME};
-}
-
-#------------------------------------------------------------------------------
-# Get tag index number
-# Inputs: 0) ExifTool object reference, 1) tag key
-# Returns: Table index number, or undefined if this tag isn't indexed
-sub GetTagIndex($$)
-{
-    my ($self, $tag) = @_;
-    my $tagInfo = $self->{TAG_INFO}{$tag} or return undef;
-    return $$tagInfo{Index};
 }
 
@@ -2107 +3287 @@
     my ($self, $tag) = @_;
     my ($desc, $name);
-    my $tagInfo = $self->{TAG_INFO}{$tag};
+    my $tagInfo = $$self{TAG_INFO}{$tag};
     # ($tagInfo won't be defined for missing tags extracted with -f)
     if ($tagInfo) {
         # use alternate language description if available
         while ($$self{CUR_LANG}) {
-            $desc = $self->{CUR_LANG}{$$tagInfo{Name}};
+            $desc = $$self{CUR_LANG}{$$tagInfo{Name}};
             if ($desc) {
                 # must look up Description if this tag also has a PrintConv
@@ -2120 +3300 @@
                 last unless $$tagInfo{LangCode} and
                     ($name = $$tagInfo{Name}) =~ s/-$$tagInfo{LangCode}$// and
-                    $desc = $self->{CUR_LANG}{$name};
+                    $desc = $$self{CUR_LANG}{$name};
                 $desc = $$desc{Description} or last if ref $desc;
                 $desc .= " ($$tagInfo{LangCode})";
@@ -2144 +3324 @@
 # Inputs: 0) ExifTool object reference
 #         1) tag key (or reference to tagInfo hash, not part of the public API)
-#         2) [optional] group family (-1 to get extended group list)
-# Returns: Scalar context: Group name (for family 0 if not otherwise specified)
-#          Array context: Group name if family specified, otherwise list of
+#         2) [optional] group family (-1 to get extended group list, or multiple
+#            families separated by colons to return multiple groups as a string)
+# Returns: Scalar context: group name (for family 0 if not otherwise specified)
+#          List context: group name if family specified, otherwise list of
 #          group names for each family.  Returns '' for undefined tag.
-# Notes: Mutiple families may be specified with ':' in family argument (ie. '1:2')
+# Notes: Multiple families may be specified with ':' in family argument (eg. '1:2')
 sub GetGroup($$;$)
 {
     local $_;
     my ($self, $tag, $family) = @_;
-    my ($tagInfo, @groups, @families, $simplify, $byTagInfo);
+    my ($tagInfo, @groups, @families, $simplify, $byTagInfo, $ex, $noID);
     if (ref $tag eq 'HASH') {
         $tagInfo = $tag;
@@ -2160 +3341 @@
         $byTagInfo = 1;
     } else {
-        $tagInfo = $self->{TAG_INFO}{$tag} or return '';
+        $tagInfo = $$self{TAG_INFO}{$tag} || { };
+        $ex = $$self{TAG_EXTRA}{$tag};
     }
     my $groups = $$tagInfo{Groups};
@@ -2166 +3348 @@
     # (after this, Groups 0-2 in tagInfo are guaranteed to be defined)
     unless ($$tagInfo{GotGroups}) {
-        my $tagTablePtr = $$tagInfo{Table};
-        if ($tagTablePtr) {
-            # construct our group list
-            $groups or $groups = $$tagInfo{Groups} = { };
-            # fill in default groups
-            foreach (keys %{$$tagTablePtr{GROUPS}}) {
-                $$groups{$_} or $$groups{$_} = $tagTablePtr->{GROUPS}{$_};
-            }
+        my $tagTablePtr = $$tagInfo{Table} || { GROUPS => { } };
+        # construct our group list
+        $groups or $groups = $$tagInfo{Groups} = { };
+        # fill in default groups
+        foreach (0..2) {
+            $$groups{$_} = $$tagTablePtr{GROUPS}{$_} || '' unless $$groups{$_};
         }
         # set flag indicating group list was built
@@ -2181 +3361 @@
         if ($family =~ /[^\d]/) {
             @families = ($family =~ /\d+/g);
-            return $$groups{0} unless @families;
+            return(($ex && $$ex{G0}) || $$groups{0}) unless @families;
             $simplify = 1 unless $family =~ /^:/;
             undef $family;
             foreach (0..2) { $groups[$_] = $$groups{$_}; }
+            $noID = 1 if @families == 1 and $families[0] != 7;
         } else {
-            return $$groups{$family} if $family == 0 or $family == 2;
+            return(($ex && $$ex{"G$family"}) || $$groups{$family}) if $family == 0 or $family == 2;
             $groups[1] = $$groups{1};
         }
     } else {
-        return $$groups{0} unless wantarray;
+        return(($ex && $$ex{G0}) || $$groups{0}) unless wantarray;
         foreach (0..2) { $groups[$_] = $$groups{$_}; }
     }
@@ -2196 +3377 @@
     $groups[4] = ($tag =~ /\((\d+)\)$/) ? "Copy$1" : '';
     # handle dynamic group names if necessary
-    my $ex = $self->{TAG_EXTRA}{$tag};
-    if ($ex and not $byTagInfo) {
-        $groups[0] = $$ex{G0} if $$ex{G0};
-        $groups[1] = $$ex{G1} =~ /^\+(.*)/ ? "$groups[1]$1" : $$ex{G1} if $$ex{G1};
-        $groups[3] = 'Doc' . $$ex{G3} if $$ex{G3};
-        $groups[5] = $$ex{G5} || $groups[1] if defined $$ex{G5};
+    unless ($byTagInfo) {
+        if ($ex) {
+            $groups[0] = $$ex{G0} if $$ex{G0};
+            $groups[1] = $$ex{G1} =~ /^\+(.*)/ ? "$groups[1]$1" : $$ex{G1} if $$ex{G1};
+            $groups[3] = 'Doc' . $$ex{G3} if $$ex{G3};
+            $groups[5] = $$ex{G5} || $groups[1] if defined $$ex{G5};
+            if (defined $$ex{G6}) {
+                $groups[5] = '' unless defined $groups[5];  # (can't leave a hole in the array)
+                $groups[6] = $$ex{G6};
+            }
+        }
+        # generate tag ID group names unless obviously not needed
+        unless ($noID) {
+            my $id = $$tagInfo{TagID};
+            if (not defined $id) {
+                $id = '';   # (just to be safe)
+            } elsif ($id =~ /^\d+$/) {
+                $id = sprintf('0x%x', $id) if $$self{OPTIONS}{HexTagIDs};
+            } else {
+                $id =~ s/([^-_A-Za-z0-9])/sprintf('%.2x',ord $1)/ge;
+            }
+            $groups[7] = 'ID-' . $id;
+            defined $groups[$_] or $groups[$_] = '' foreach (5,6);
+        }
     }
     if ($family) {
         return $groups[$family] || '' if $family > 0;
         # add additional matching group names to list
-        # ie) for MIE-Doc, also add MIE1, MIE1-Doc, MIE-Doc1 and MIE1-Doc1
+        # eg) for MIE-Doc, also add MIE1, MIE1-Doc, MIE-Doc1 and MIE1-Doc1
         # and for MIE2-Doc3, also add MIE2, MIE-Doc3, MIE2-Doc and MIE-Doc
         if ($groups[1] =~ /^MIE(\d*)-(.+?)(\d*)$/) {
@@ -2219 +3418 @@
         # create list of group names (without identical adjacent groups if simplifying)
         foreach (@families) {
-            my $grp = $groups[$_] or next;
+            my $grp = $groups[$_];
+            unless ($grp) {
+                next if $simplify;
+                $grp = '';
+            }
             push @grps, $grp unless $simplify and @grps and $grp eq $grps[-1];
         }
@@ -2246 +3449 @@
     if (ref $info ne 'HASH') {
         $family = $info;
-        $info = $self->{VALUE};
+        $info = $$self{VALUE};
     } else {
         $family = shift;
@@ -2264 +3467 @@
 # Inputs: 0) ExifTool object reference,
 #         1-N) group names (reset to default if no groups specified)
+# - used when new tag values are set (ie. before files are written)
 sub SetNewGroups($;@)
 {
@@ -2269 +3473 @@
     my ($self, @groups) = @_;
     @groups or @groups = @defaultWriteGroups;
-    my $count = @groups;
+    my $count = @groups * 10;
     my %priority;
     foreach (@groups) {
-        $priority{lc($_)} = $count--;
-    }
-    $priority{file} = 10;       # 'File' group is always written (Comment)
-    $priority{composite} = 10;  # 'Composite' group is always written
+        $priority{lc($_)} = $count;
+        $count -= 10;
+    }
+    $priority{file} = 500;      # 'File' group is always written (Comment)
+    $priority{composite} = 500; # 'Composite' group is always written
     # set write priority (higher # is higher priority)
-    $self->{WRITE_PRIORITY} = \%priority;
-    $self->{WRITE_GROUPS} = \@groups;
+    $$self{WRITE_PRIORITY} = \%priority;
+    $$self{WRITE_GROUPS} = \@groups;
 }
 
@@ -2293 +3498 @@
 
     $$self{BuildingComposite} = 1;
-    # first, add user-defined Composite tags if necessary
-    if (%UserDefined and $UserDefined{'Image::ExifTool::Composite'}) {
-        AddCompositeTags($UserDefined{'Image::ExifTool::Composite'}, 1);
-        delete $UserDefined{'Image::ExifTool::Composite'};
-    }
-    my @tagList = sort keys %Image::ExifTool::Composite;
-    my %tagsUsed;
-
-    my $rawValue = $self->{VALUE};
+
+    my $compTable = GetTagTable('Image::ExifTool::Composite');
+    my @tagList = sort keys %$compTable;
+    my $rawValue = $$self{VALUE};
+    my $compKeys = $$self{COMP_KEYS};
+    my (%cache, $allBuilt);
+
     for (;;) {
-        my %notBuilt;
-        $notBuilt{$_} = 1 foreach @tagList;
-        my @deferredTags;
-        my $tag;
+        my (%notBuilt, $tag, @deferredTags);
+        foreach (@tagList) {
+            $notBuilt{$$compTable{$_}{Name}} = 1 unless $specialTags{$_};
+        }
 COMPOSITE_TAG:
         foreach $tag (@tagList) {
             next if $specialTags{$tag};
-            my $tagInfo = $self->GetTagInfo(\%Image::ExifTool::Composite, $tag);
+            my $tagInfo = $self->GetTagInfo($compTable, $tag);
             next unless $tagInfo;
+            my $tagName = $$compTable{$tag}{Name};
             # put required tags into array and make sure they all exist
             my $subDoc = ($$tagInfo{SubDoc} and $$self{DOC_COUNT});
             my $require = $$tagInfo{Require} || { };
-            my $desire = $$tagInfo{Desire} || { };
+            my $desire  = $$tagInfo{Desire}  || { };
+            my $inhibit = $$tagInfo{Inhibit} || { };
             # loop through sub-documents if necessary
-            my $doc;
+            my $docNum = 0;
             for (;;) {
                 my (%tagKey, $found, $index);
                 # save Require'd and Desire'd tag values in list
                 for ($index=0; ; ++$index) {
-                    my $reqTag = $$require{$index} || $$desire{$index} or last;
-                    # add family 3 group if generating Composite tags for sub-documents
-                    # (unless tag already begins with family 3 group name)
-                    if ($subDoc and $reqTag !~ /^(Main|Doc\d+):/) {
-                        $reqTag = ($doc ? "Doc$doc:" : 'Main:') . $reqTag;
+                    my $reqTag = $$require{$index} || $$desire{$index} || $$inhibit{$index};
+                    unless ($reqTag) {
+                        # allow Composite with no Require'd or Desire'd tags
+                        $found = 1 if $index == 0;
+                        last;
                     }
-                    # allow tag group to be specified
-                    if ($reqTag =~ /^(.*):(.+)/) {
+                    if ($subDoc) {
+                        # handle SubDoc tags specially to cache tag keys for faster
+                        # processing when there are a large number of sub-documents
+                        # - get document number from the tag groups if specified,
+                        #   otherwise we are looping through all documents for this tag
+                        my $doc = $reqTag =~ s/\b(Main|Doc(\d+)):// ? ($2 || 0) : $docNum;
+                        # make fast lookup for keys of this tag with specified groups other than doc group
+                        # (similar to code in InsertTagValues(), but this is case-sensitive)
+                        my $cacheTag = $cache{$reqTag};
+                        unless ($cacheTag) {
+                            $cacheTag = $cache{$reqTag} = [ ];
+                            my $reqGroup;
+                            $reqTag =~ s/^(.*):// and $reqGroup = $1;
+                            my ($i, $key, @keys);
+                            # build list of tag keys in order of precedence
+                            for ($key=$reqTag, $i=$$self{DUPL_TAG}{$reqTag} || 0; ; --$i) {
+                                push @keys, $key if defined $$rawValue{$key};
+                                last if $i <= 0;
+                                $key = "$reqTag ($i)";
+                            }
+                            @keys = $self->GroupMatches($reqGroup, \@keys) if defined $reqGroup;
+                            if (@keys) {
+                                my $ex = $$self{TAG_EXTRA};
+                                # loop through tags in reverse order of precedence so the higher
+                                # priority tag will win in the case of duplicates within a doc
+                                $$cacheTag[$$ex{$_} ? $$ex{$_}{G3} || 0 : 0] = $_ foreach reverse @keys;
+                            }
+                        }
+                        # (set $reqTag to a bogus key if not found)
+                        $reqTag = $$cacheTag[$doc] || "$reqTag (0)";
+                    } elsif ($reqTag =~ /^(.*):(.+)/) {
                         my ($reqGroup, $name) = ($1, $2);
                         if ($reqGroup eq 'Composite' and $notBuilt{$name}) {
-                            push @deferredTags, $tag;
-                            next COMPOSITE_TAG;
+                            # defer only until all other tags are built if
+                            # we are inhibiting based on another Composite tag
+                            unless ($$inhibit{$index} and $allBuilt) {
+                                push @deferredTags, $tag;
+                                next COMPOSITE_TAG;
+                            }
                         }
+                        # (CAREFUL! keys may not be sequential if one was deleted)
                         my ($i, $key, @keys);
-                        for ($i=0; ; ++$i) {
-                            $key = $name;
-                            $key .= " ($i)" if $i;
-                            last unless defined $$rawValue{$key};
-                            push @keys, $key;
+                        for ($key=$name, $i=$$self{DUPL_TAG}{$name} || 0; ; --$i) {
+                            push @keys, $key if defined $$rawValue{$key};
+                            last if $i <= 0;
+                            $key = "$name ($i)";
                         }
                         # find first matching tag
                         $key = $self->GroupMatches($reqGroup, \@keys);
-                        $reqTag = $key if $key;
-                    } elsif ($notBuilt{$reqTag}) {
+                        $reqTag = $key || "$name (0)";
+                    } elsif ($notBuilt{$reqTag} and not $$inhibit{$index}) {
                         # calculate this tag later if it relies on another
                         # Composite tag which hasn't been calculated yet
@@ -2352 +3590 @@
                     }
                     if (defined $$rawValue{$reqTag}) {
-                        $found = 1;
+                        if ($$inhibit{$index}) {
+                            $found = 0;
+                            last;
+                        } else {
+                            $found = 1;
+                        }
                     } elsif ($$require{$index}) {
                         $found = 0;
@@ -2359 +3602 @@
                     $tagKey{$index} = $reqTag;
                 }
-                if ($doc) {
+                if ($docNum) {
                     if ($found) {
-                        $self->{DOC_NUM} = $doc;
+                        $$self{DOC_NUM} = $docNum;
+                        # save pointers to all used tag keys
+                        foreach (keys %tagKey) {
+                            $$compKeys{$_} or $$compKeys{$_} = [ ];
+                            push @{$$compKeys{$tagKey{$_}}}, [ \%tagKey, $_ ];
+                        }
                         $self->FoundTag($tagInfo, \%tagKey);
-                        delete $self->{DOC_NUM};
+                        delete $$self{DOC_NUM};
                     }
-                    next if ++$doc <= $self->{DOC_COUNT};
+                    next if ++$docNum <= $$self{DOC_COUNT};
                     last;
                 } elsif ($found) {
-                    delete $notBuilt{$tag}; # this tag is OK to build now
+                    delete $notBuilt{$tagName}; # this tag is OK to build now
                     # keep track of all Require'd tag keys
                     foreach (keys %tagKey) {
@@ -2374 +3622 @@
                         # can be replaced (also eliminates keys with
                         # instance numbers which can't be replaced either)
-                        next unless $Image::ExifTool::Composite{$tagKey{$_}};
-                        my $keyRef = \$tagKey{$_};
-                        $tagsUsed{$$keyRef} or $tagsUsed{$$keyRef} = [ ];
-                        push @{$tagsUsed{$$keyRef}}, $keyRef;
+                        next unless $compositeID{$tagKey{$_}};
+                    }
+                    # save pointers to all used tag keys
+                    foreach (keys %tagKey) {
+                        $$compKeys{$_} or $$compKeys{$_} = [ ];
+                        push @{$$compKeys{$tagKey{$_}}}, [ \%tagKey, $_ ];
                     }
                     # save reference to tag key lookup as value for Composite tag
                     my $key = $self->FoundTag($tagInfo, \%tagKey);
-                    # check to see if we just replaced one of the tag keys we Require'd
-                    if (defined $key and $tagsUsed{$key}) {
-                        foreach (@{$tagsUsed{$key}}) {
-                            $$_ = $self->{MOVED_KEY};   # replace with new tag key
-                        }
-                        delete $tagsUsed{$key};         # can't be replaced again
+                } elsif (not defined $found) {
+                    delete $notBuilt{$tagName}; # tag can't be built anyway
+                }
+                last unless $subDoc;
+                # don't process sub-documents if there is no chance to build this tag
+                # (can be very time-consuming if there are many docs)
+                if (%$require) {
+                    foreach (keys %$require) {
+                        my $reqTag = $$require{$_};
+                        $reqTag =~ s/.*://;
+                        next COMPOSITE_TAG unless defined $$rawValue{$reqTag};
                     }
-                } elsif (not defined $found) {
-                    delete $notBuilt{$tag}; # tag can't be built anyway
-                }
-                last unless $subDoc;
-                $doc = 1;   # continue to process the 1st sub-document
+                    $docNum = 1;   # go ahead and process the 1st sub-document
+                } else {
+                    my @try = ref $$tagInfo{SubDoc} ? @{$$tagInfo{SubDoc}} : keys %$desire;
+                    # at least one of the specified desire tags must exist
+                    foreach (@try) {
+                        my $desTag = $$desire{$_} or next;
+                        $desTag =~ s/.*://;
+                        defined $$rawValue{$desTag} and $docNum = 1, last;
+                    }
+                    last unless $docNum;
+                }
             }
         }
         last unless @deferredTags;
         if (@deferredTags == @tagList) {
-            # everything was deferred in the last pass,
-            # must be a circular dependency
-            warn "Circular dependency in Composite tags\n";
-            last;
+            if ($allBuilt) {
+                # everything was deferred in the last pass,
+                # must be a circular dependency
+                warn "Circular dependency in Composite tags\n";
+                last;
+            }
+            $allBuilt = 1;  # try once more, ignoring Composite Inhibit tags
         }
         @tagList = @deferredTags; # calculate deferred tags now
     }
     delete $$self{BuildingComposite};
+}
+
+#------------------------------------------------------------------------------
+# Get reference to Composite tag info hash
+# Inputs: 0) case-sensitive Composite tag name
+# Returns: tagInfo hash or undef
+sub GetCompositeTagInfo($)
+{
+    my $tag = shift;
+    return undef unless $compositeID{$tag};
+    return $Image::ExifTool::Composite{$compositeID{$tag}[0]};
 }
 
@@ -2434 +3709 @@
 #         1) flag to return long description instead of type ('0' to return any recognized type)
 # Returns: File type (or desc) or undef if extension not supported or if
-#          description is the same as the input FileType.  In array
-#          context, may return more than one file type if the file may be
-#          different formats.  Returns list of all supported extensions if no
-#          file specified
+#          description is the same as the input FileType.  In list context,
+#          may return more than one file type if the file may be different formats.
+#          Returns list of all supported extensions if no file specified
 sub GetFileType(;$$)
 {
@@ -2450 +3724 @@
             # return all supported types
             foreach (sort keys %fileTypeLookup) {
-                push @types, $_ unless defined $moduleName{$_} and $moduleName{$_} eq '0';
+                my $module = $moduleName{$_};
+                $module = $moduleName{$fileTypeLookup{$_}} unless defined $module;
+                push @types, $_ unless defined $module and $module eq '0';
             }
         }
         return @types;
     }
-    my $fileType;
+    my ($fileType, $subType);
     my $fileExt = GetFileExtension($file);
-    $fileExt = uc($file) unless $fileExt;
+    unless ($fileExt) {
+        if ($file =~ s/ \((.*)\)$//) {
+            $subType = $1;
+            $fileExt = GetFileExtension($file);
+        }
+        $fileExt = uc($file) unless $fileExt;
+    }
     $fileExt and $fileType = $fileTypeLookup{$fileExt}; # look up the file type
-    $fileType = $fileTypeLookup{$fileType} unless ref $fileType or not $fileType;
+    $fileType = $fileTypeLookup{$fileType} while $fileType and not ref $fileType;
     # return description if specified
     # (allow input $file to be a FileType for this purpose)
     if ($desc) {
-        return $fileType ? $$fileType[1] : $fileDescription{$file};
+        $desc = $fileType ? $$fileType[1] : $fileDescription{$file};
+        $desc .= ", $subType" if $subType;
+        return $desc;
     } elsif ($fileType and (not defined $desc or $desc ne '0')) {
         # return only supported file types
@@ -2469 +3753 @@
         undef $fileType if defined $mod and $mod eq '0';
     }
-    $fileType or return wantarray ? () : undef;
+    $fileType or return ();
     $fileType = $$fileType[0];      # get file type (or list of types)
     if (wantarray) {
@@ -2487 +3771 @@
     local $_;
     my $file = shift or return undef;
-    my $type = GetFileType($file) or return undef;
+    my ($type) = GetFileType($file) or return undef;
     if ($noWriteFile{$type}) {
         # can't write TIFF files with certain extensions (various RAW formats)
@@ -2493 +3777 @@
         return grep(/^$ext$/, @{$noWriteFile{$type}}) ? 0 : 1 if $ext;
     }
-    return scalar(grep /^$type$/, @writeTypes);
+    unless (%writeTypes) {
+        $writeTypes{$_} = 1 foreach @writeTypes;
+    }
+    return $writeTypes{$type};
 }
 
@@ -2513 +3800 @@
 # Functions below this are not part of the public API
 
-# Initialize member variables
+# Initialize member variables for reading or writing a new file
 # Inputs: 0) ExifTool object reference
 sub Init($)
@@ -2521 +3808 @@
     # delete all DataMember variables (lower-case names)
     foreach (keys %$self) {
-        /[a-z]/ and delete $self->{$_};
-    }
-    delete $self->{FOUND_TAGS};     # list of found tags
-    delete $self->{EXIF_DATA};      # the EXIF data block
-    delete $self->{EXIF_POS};       # EXIF position in file
-    delete $self->{FIRST_EXIF_POS}; # position of first EXIF in file
-    delete $self->{HTML_DUMP};      # html dump information
-    delete $self->{SET_GROUP1};     # group1 name override
-    delete $self->{DOC_NUM};        # current embedded document number
-    $self->{DOC_COUNT}  = 0;        # count of embedded documents processed
-    $self->{BASE}       = 0;        # base for offsets from start of file
-    $self->{FILE_ORDER} = { };      # * hash of tag order in file
-    $self->{VALUE}      = { };      # * hash of raw tag values
-    $self->{BOTH}       = { };      # * hash for Value/PrintConv values of Require'd tags
-    $self->{TAG_INFO}   = { };      # * hash of tag information
-    $self->{TAG_EXTRA}  = { };      # * hash of extra tag information (dynamic group names)
-    $self->{PRIORITY}   = { };      # * priority of current tags
-    $self->{LIST_TAGS}  = { };      # hash of tagInfo refs for active List-type tags
-    $self->{PROCESSED}  = { };      # hash of processed directory start positions
-    $self->{DIR_COUNT}  = { };      # count various types of directories
-    $self->{DUPL_TAG}   = { };      # last-used index for duplicate-tag keys
-    $self->{WARNED_ONCE}= { };      # WarnOnce() warnings already issued
-    $self->{PATH}       = [ ];      # current subdirectory path in file when reading
-    $self->{NUM_FOUND}  = 0;        # total number of tags found (incl. duplicates)
-    $self->{CHANGED}    = 0;        # number of tags changed (writer only)
-    $self->{INDENT}     = '  ';     # initial indent for verbose messages
-    $self->{PRIORITY_DIR} = '';     # the priority directory name
-    $self->{LOW_PRIORITY_DIR} = { PreviewIFD => 1 }; # names of priority 0 directories
-    $self->{TIFF_TYPE}  = '';       # type of TIFF data (APP1, TIFF, NEF, etc...)
-    $self->{Make}       = '';       # camera make
-    $self->{Model}      = '';       # camera model
-    $self->{CameraType} = '';       # Olympus camera type
+        /[a-z]/ and delete $$self{$_};
+    }
+    delete $$self{FOUND_TAGS};      # list of found tags
+    delete $$self{EXIF_DATA};       # the EXIF data block
+    delete $$self{EXIF_POS};        # EXIF position in file
+    delete $$self{FIRST_EXIF_POS};  # position of first EXIF in file
+    delete $$self{HTML_DUMP};       # html dump information
+    delete $$self{SET_GROUP0};      # group0 name override
+    delete $$self{SET_GROUP1};      # group1 name override
+    delete $$self{DOC_NUM};         # current embedded document number
+    $$self{DOC_COUNT}  = 0;         # count of embedded documents processed
+    $$self{BASE}       = 0;         # base for offsets from start of file
+    $$self{FILE_ORDER} = { };       # * hash of tag order in file ('*' = based on tag key)
+    $$self{VALUE}      = { };       # * hash of raw tag values
+    $$self{BOTH}       = { };       # * hash for Value/PrintConv values of Require'd tags
+    $$self{RATIONAL}   = { };       # * hash of original rational components
+    $$self{TAG_INFO}   = { };       # * hash of tag information
+    $$self{TAG_EXTRA}  = { };       # * hash of extra tag information (dynamic group names)
+    $$self{PRIORITY}   = { };       # * priority of current tags
+    $$self{LIST_TAGS}  = { };       # hash of tagInfo refs for active List-type tags
+    $$self{PROCESSED}  = { };       # hash of processed directory start positions
+    $$self{DIR_COUNT}  = { };       # count various types of directories
+    $$self{DUPL_TAG}   = { };       # last-used index for duplicate-tag keys
+    $$self{WARNED_ONCE}= { };       # WarnOnce() warnings already issued
+    $$self{WRITTEN}    = { };       # list of tags written (selected tags only)
+    $$self{FORCE_WRITE}= { };       # ForceWrite lookup (set from ForceWrite tag)
+    $$self{FOUND_DIR}  = { };       # hash of directory names found in file
+    $$self{COMP_KEYS}  = { };       # lookup for tag keys used in Composite tags
+    $$self{PATH}       = [ ];       # current subdirectory path in file when reading
+    $$self{NUM_FOUND}  = 0;         # total number of tags found (incl. duplicates)
+    $$self{CHANGED}    = 0;         # number of tags changed (writer only)
+    $$self{INDENT}     = '  ';      # initial indent for verbose messages
+    $$self{PRIORITY_DIR} = '';      # the priority directory name
+    $$self{LOW_PRIORITY_DIR} = { PreviewIFD => 1 }; # names of priority 0 directories
+    $$self{TIFF_TYPE}  = '';        # type of TIFF data (APP1, TIFF, NEF, etc...)
+    $$self{FMT_EXPR}   = undef;     # current advanced formatting expression
+    $$self{Make}       = '';        # camera make
+    $$self{Model}      = '';        # camera model
+    $$self{CameraType} = '';        # Olympus camera type
+    $$self{FileType}   = '';        # identified file type
     if ($self->Options('HtmlDump')) {
         require Image::ExifTool::HtmlDump;
-        $self->{HTML_DUMP} = new Image::ExifTool::HtmlDump;
+        $$self{HTML_DUMP} = new Image::ExifTool::HtmlDump;
     }
     # make sure our TextOut is a file reference
-    $self->{OPTIONS}{TextOut} = \*STDOUT unless ref $self->{OPTIONS}{TextOut};
+    $$self{OPTIONS}{TextOut} = \*STDOUT unless ref $$self{OPTIONS}{TextOut};
+}
+
+#------------------------------------------------------------------------------
+# Combine information from a list of info hashes
+# Unless Duplicates is enabled, first entry found takes priority
+# Inputs: 0) ExifTool object reference, 1-N) list of info hash references
+# Returns: Combined information hash reference
+sub CombineInfo($;@)
+{
+    local $_;
+    my $self = shift;
+    my (%combinedInfo, $info, $tag, %haveInfo);
+
+    if ($$self{OPTIONS}{Duplicates}) {
+        while ($info = shift) {
+            foreach $tag (keys %$info) {
+                $combinedInfo{$tag} = $$info{$tag};
+            }
+        }
+    } else {
+        while ($info = shift) {
+            foreach $tag (keys %$info) {
+                my $tagName = GetTagName($tag);
+                next if $haveInfo{$tagName};
+                $haveInfo{$tagName} = 1;
+                $combinedInfo{$tag} = $$info{$tag};
+            }
+        }
+    }
+    return \%combinedInfo;
+}
+
+#------------------------------------------------------------------------------
+# Get tag table name
+# Inputs: 0) ExifTool object reference, 1) tag key
+# Returns: Table name if available, otherwise ''
+sub GetTableName($$)
+{
+    my ($self, $tag) = @_;
+    my $tagInfo = $$self{TAG_INFO}{$tag} or return '';
+    return $$tagInfo{Table}{SHORT_NAME};
+}
+
+#------------------------------------------------------------------------------
+# Get tag index number
+# Inputs: 0) ExifTool object reference, 1) tag key
+# Returns: Table index number, or undefined if this tag isn't indexed
+sub GetTagIndex($$)
+{
+    my ($self, $tag) = @_;
+    my $tagInfo = $$self{TAG_INFO}{$tag} or return undef;
+    return $$tagInfo{Index};
+}
+
+#------------------------------------------------------------------------------
+# Find value for specified tag
+# Inputs: 0) ExifTool ref, 1) tag name, 2) tag group (family 1)
+# Returns: value or undef
+sub FindValue($$$)
+{
+    my ($et, $tag, $grp) = @_;
+    my ($i, $val);
+    my $value = $$et{VALUE};
+    for ($i=0; ; ++$i) {
+        my $key = $tag . ($i ? " ($i)" : '');
+        last unless defined $$value{$key};
+        if ($et->GetGroup($key, 1) eq $grp) {
+            $val = $$value{$key};
+            last;
+        }
+    }
+    return $val;
+}
+
+#------------------------------------------------------------------------------
+# Get tag key for next existing tag
+# Inputs: 0) ExifTool ref, 1) tag key or case-sensitive tag name
+# Returns: Key of next existing tag, or undef if no more
+# Notes: This routine is provided for iterating through duplicate tags in the
+#        ValueConv of Composite tags.
+sub NextTagKey($$)
+{
+    my ($self, $tag) = @_;
+    my $i = ($tag =~ s/ \((\d+)\)$//) ? $1 + 1 : 1;
+    $tag = "$tag ($i)";
+    return $tag if defined $$self{VALUE}{$tag};
+    return undef;
+}
+
+#------------------------------------------------------------------------------
+# Split file name into directory and name parts
+# Inptus: 0) file name
+# Returns: 0) directory, 1) filename
+sub SplitFileName($)
+{
+    my $file = shift;
+    my ($dir, $name);
+    if (eval { require File::Basename }) {
+        $dir = File::Basename::dirname($file);
+        $name = File::Basename::basename($file);
+    } else {
+        ($name = $file) =~ tr/\\/\//;
+        # remove path
+        $dir = length($1) ? $1 : '/' if $name =~ s/(.*)\///;
+    }
+    return ($dir, $name);
+}
+
+#------------------------------------------------------------------------------
+# Encode file name for calls to system i/o routines
+# Inputs: 0) ExifTool ref, 1) file name in CharSetFileName, 2) flag to force conversion
+# Returns: true if Windows Unicode routines should be used (in which case
+#          the file name will be encoded as a null-terminated UTF-16LE string)
+sub EncodeFileName($$;$)
+{
+    my ($self, $file, $force) = @_;
+    my $enc = $$self{OPTIONS}{CharsetFileName};
+    if ($enc) {
+        if ($file =~ /[\x80-\xff]/ or $force) {
+            # encode for use in Windows Unicode functions if necessary
+            if ($^O eq 'MSWin32') {
+                local $SIG{'__WARN__'} = \&SetWarning;
+                if (eval { require Win32API::File }) {
+                    # recode as UTF-16LE and add null terminator
+                    $_[1] = $self->Decode($file, $enc, undef, 'UTF16', 'II') . "\0\0";
+                    return 1;
+                }
+                $self->WarnOnce('Install Win32API::File for Windows Unicode file support');
+            } else {
+                # recode as UTF-8 for other platforms if necessary
+                $_[1] = $self->Decode($file, $enc, undef, 'UTF8') unless $enc eq 'UTF8';
+            }
+        }
+    } elsif ($^O eq 'MSWin32' and $file =~ /[\x80-\xff]/ and not defined $enc) {
+        require Image::ExifTool::XMP;
+        if (Image::ExifTool::XMP::IsUTF8(\$file) < 0) {
+            $self->WarnOnce('FileName encoding not specified');
+        }
+    }
+    return 0;
+}
+
+#------------------------------------------------------------------------------
+# Modified perl open() routine to properly handle special characters in file names
+# Inputs: 0) ExifTool ref, 1) filehandle, 2) filename,
+#         3) mode: '<' or undef = read, '>' = write, '+<' = update
+# Returns: true on success
+# Note: Must call like "$et->Open(\*FH,$file)", not "$et->Open(FH,$file)" to avoid
+#       "unopened filehandle" errors due to a change in scope of the filehandle
+sub Open($*$;$)
+{
+    my ($self, $fh, $file, $mode) = @_;
+
+    $file =~ s/^([\s&])/.\/$1/; # protect leading whitespace or ampersand
+    # default to read mode ('<') unless input is a pipe
+    $mode = ($file =~ /\|$/ ? '' : '<') unless $mode;
+    if ($mode) {
+        if ($self->EncodeFileName($file)) {
+            # handle Windows Unicode file name
+            local $SIG{'__WARN__'} = \&SetWarning;
+            my ($access, $create);
+            if ($mode eq '>') {
+                eval {
+                    $access  = Win32API::File::GENERIC_WRITE();
+                    $create  = Win32API::File::CREATE_ALWAYS();
+                }
+            } else {
+                eval {
+                    $access  = Win32API::File::GENERIC_READ();
+                    $access |= Win32API::File::GENERIC_WRITE() if $mode eq '+<'; # update
+                    $create  = Win32API::File::OPEN_EXISTING();
+                }
+            }
+            my $share = 0;
+            eval {
+                unless ($access & Win32API::File::GENERIC_WRITE()) {
+                    $share = Win32API::File::FILE_SHARE_READ() | Win32API::File::FILE_SHARE_WRITE();
+                }
+            };
+            my $wh = eval { Win32API::File::CreateFileW($file, $access, $share, [], $create, 0, []) };
+            return undef unless $wh;
+            my $fd = eval { Win32API::File::OsFHandleOpenFd($wh, 0) };
+            if (not defined $fd or $fd < 0) {
+                eval { Win32API::File::CloseHandle($wh) };
+                return undef;
+            }
+            $file = "&=$fd";    # specify file by descriptor
+        } else {
+            # add leading space to protect against leading characters like '>'
+            # in file name, and trailing "\0" to protect trailing spaces
+            $file = " $file\0";
+        }
+    }
+    return open $fh, "$mode$file";
+}
+
+#------------------------------------------------------------------------------
+# Check to see if a file exists (with Windows Unicode support)
+# Inputs: 0) ExifTool ref, 1) file name
+# Returns: true if file exists
+sub Exists($$)
+{
+    my ($self, $file) = @_;
+
+    if ($self->EncodeFileName($file)) {
+        local $SIG{'__WARN__'} = \&SetWarning;
+        my $wh = eval { Win32API::File::CreateFileW($file,
+                        Win32API::File::GENERIC_READ(),
+                        Win32API::File::FILE_SHARE_READ(), [],
+                        Win32API::File::OPEN_EXISTING(), 0, []) };
+        return 0 unless $wh;
+        eval { Win32API::File::CloseHandle($wh) };
+    } else {
+        return -e $file;
+    }
+    return 1;
+}
+
+#------------------------------------------------------------------------------
+# Return true if file is a directory (with Windows Unicode support)
+# Inputs: 0) ExifTool ref, 1) file name
+# Returns: true if file is a directory (false if file isn't, or doesn't exist)
+sub IsDirectory($$)
+{
+    my ($et, $file) = @_;
+    if ($et->EncodeFileName($file)) {
+        local $SIG{'__WARN__'} = \&SetWarning;
+        my $attrs = eval { Win32API::File::GetFileAttributesW($file) };
+        my $dirBit = eval { Win32API::File::FILE_ATTRIBUTE_DIRECTORY() } || 0;
+        return 1 if $attrs and $attrs != 0xffffffff and $attrs & $dirBit;
+    } else {
+        return -d $file;
+    }
+    return 0;
+}
+
+#------------------------------------------------------------------------------
+# Get file times (Unix seconds since the epoch)
+# Inputs: 0) ExifTool ref, 1) file name or ref
+# Returns: 0) access time, 1) modification time, 2) creation time (or undefs on error)
+my $k32GetFileTime;
+sub GetFileTime($$)
+{
+    my ($self, $file) = @_;
+
+    # open file by name if necessary
+    unless (ref $file) {
+        local *FH;
+        unless ($self->Open(\*FH, $file)) {
+            if ($self->IsDirectory($file)) {
+                my @rtn = (stat $file)[8, 9, 10];
+                return @rtn if defined $rtn[0];
+            }
+            $self->Warn("GetFileTime error for '${file}'");
+            return ();
+        }
+        $file = *FH;  # (not \*FH, so *FH will be kept open until $file goes out of scope)
+    }
+    # on Windows, try to work around incorrect file times when daylight saving time is in effect
+    if ($^O eq 'MSWin32') {
+        if (not eval { require Win32::API }) {
+            $self->WarnOnce('Install Win32::API for proper handling of Windows file times');
+        } elsif (not eval { require Win32API::File }) {
+            $self->WarnOnce('Install Win32API::File for proper handling of Windows file times');
+        } else {
+            # get Win32 handle, needed for GetFileTime
+            my $win32Handle = eval { Win32API::File::GetOsFHandle($file) };
+            unless ($win32Handle) {
+                $self->Warn("Win32API::File::GetOsFHandle returned invalid handle");
+                return ();
+            }
+            # get FILETIME structs
+            my ($atime, $mtime, $ctime, $time);
+            $atime = $mtime = $ctime = pack 'LL', 0, 0;
+            unless ($k32GetFileTime) {
+                return () if defined $k32GetFileTime;
+                $k32GetFileTime = new Win32::API('KERNEL32', 'GetFileTime', 'NPPP', 'I');
+                unless ($k32GetFileTime) {
+                    $self->Warn('Error calling Win32::API::GetFileTime');
+                    $k32GetFileTime = 0;
+                    return ();
+                }
+            }
+            unless ($k32GetFileTime->Call($win32Handle, $ctime, $atime, $mtime)) {
+                $self->Warn("Win32::API::GetFileTime returned " . Win32::GetLastError());
+                return ();
+            }
+            # convert FILETIME structs to Unix seconds
+            foreach $time ($atime, $mtime, $ctime) {
+                my ($lo, $hi) = unpack 'LL', $time; # unpack FILETIME struct
+                # FILETIME is in 100 ns intervals since 0:00 UTC Jan 1, 1601
+                # (89 leap years between 1601 and 1970)
+                $time = ($hi * 4294967296 + $lo) * 1e-7 - (((1970-1601)*365+89)*24*3600);
+            }
+            return ($atime, $mtime, $ctime);
+        }
+    }
+    # other os (or Windows fallback)
+    return (stat $file)[8, 9, 10];
 }
 
@@ -2568 +4162 @@
 {
     my $self = shift;
-    my $options = $self->{OPTIONS};
-    my @exclude;
-    my @oldGroupOpts = grep /^Group/, keys %{$self->{OPTIONS}};
-    my $wasExcludeOpt;
-
-    $self->{REQUESTED_TAGS} = [ ];
-    $self->{REQ_TAG_LOOKUP} = { };
-    $self->{IO_TAG_LIST} = undef;
+    my $options = $$self{OPTIONS};
+    my @oldGroupOpts = grep /^Group/, keys %{$$self{OPTIONS}};
+    my (@exclude, $wasExcludeOpt);
+
+    $$self{REQUESTED_TAGS}  = [ ];
+    $$self{REQ_TAG_LOOKUP}  = { };
+    $$self{EXCL_TAG_LOOKUP} = { };
+    $$self{IO_TAG_LIST} = undef;
 
     # handle our input arguments
     while (@_) {
         my $arg = shift;
-        if (ref $arg) {
+        if (ref $arg and not overload::Method($arg, q[""])) {
             if (ref $arg eq 'ARRAY') {
-                $self->{IO_TAG_LIST} = $arg;
+                $$self{IO_TAG_LIST} = $arg;
                 foreach (@$arg) {
                     if (/^-(.*)/) {
                         push @exclude, $1;
                     } else {
-                        push @{$self->{REQUESTED_TAGS}}, $_;
+                        push @{$$self{REQUESTED_TAGS}}, $_;
                     }
                 }
@@ -2596 +4190 @@
                     if (@oldGroupOpts and $opt =~ /^Group/) {
                         foreach (@oldGroupOpts) {
-                            delete $options->{$_};
+                            delete $$options{$_};
                         }
                         undef @oldGroupOpts;
@@ -2604 +4198 @@
                 }
             } elsif (ref $arg eq 'SCALAR' or UNIVERSAL::isa($arg,'GLOB')) {
-                next if defined $self->{RAF};
+                next if defined $$self{RAF};
                 # convert image data from UTF-8 to character stream if necessary
                 # (patches RHEL 3 UTF8 LANG problem)
                 if (ref $arg eq 'SCALAR' and $] >= 5.006 and
-                    (eval 'require Encode; Encode::is_utf8($$arg)' or $@))
+                    (eval { require Encode; Encode::is_utf8($$arg) } or $@))
                 {
                     # repack by hand if Encode isn't available
-                    my $buff = $@ ? pack('C*',unpack('U0C*',$$arg)) : Encode::encode('utf8',$$arg);
+                    my $buff = $@ ? pack('C*',unpack($] < 5.010000 ? 'U0C*' : 'C0C*',$$arg)) : Encode::encode('utf8',$$arg);
                     $arg = \$buff;
                 }
-                $self->{RAF} = new File::RandomAccess($arg);
+                $$self{RAF} = new File::RandomAccess($arg);
                 # set filename to empty string to indicate that
                 # we have a file but we didn't open it
-                $self->{FILENAME} = '';
+                $$self{FILENAME} = '';
             } elsif (UNIVERSAL::isa($arg, 'File::RandomAccess')) {
-                $self->{RAF} = $arg;
-                $self->{FILENAME} = '';
+                $$self{RAF} = $arg;
+                $$self{FILENAME} = '';
             } else {
                 warn "Don't understand ImageInfo argument $arg\n";
             }
-        } elsif (defined $self->{FILENAME}) {
+        } elsif (defined $$self{FILENAME}) {
             if ($arg =~ /^-(.*)/) {
                 push @exclude, $1;
             } else {
-                push @{$self->{REQUESTED_TAGS}}, $arg;
+                push @{$$self{REQUESTED_TAGS}}, $arg;
             }
         } else {
-            $self->{FILENAME} = $arg;
-        }
+            $$self{FILENAME} = $arg;
+        }
+    }
+    # add additional requested tags to lookup
+    if ($$options{RequestTags}) {
+        $$self{REQ_TAG_LOOKUP}{$_} = 1 foreach @{$$options{RequestTags}};
     }
     # expand shortcuts in tag arguments if provided
-    if (@{$self->{REQUESTED_TAGS}}) {
-        ExpandShortcuts($self->{REQUESTED_TAGS});
+    if (@{$$self{REQUESTED_TAGS}}) {
+        ExpandShortcuts($$self{REQUESTED_TAGS});
         # initialize lookup for requested tags
-        foreach (@{$self->{REQUESTED_TAGS}}) {
-            $self->{REQ_TAG_LOOKUP}{lc(/.+:(.+)/ ? $1 : $_)} = 1;
-        }
-    }
-
+        foreach (@{$$self{REQUESTED_TAGS}}) {
+            /^(.*:)?([-\w?*]*)#?$/ or next;
+            $$self{REQ_TAG_LOOKUP}{lc($2)} = 1 if $2;
+            next unless $1;
+            $$self{REQ_TAG_LOOKUP}{lc($_).':'} = 1 foreach split /:/, $1;
+        }
+    }
     if (@exclude or $wasExcludeOpt) {
         # must add existing excluded tags
-        if ($options->{Exclude}) {
-            if (ref $options->{Exclude} eq 'ARRAY') {
-                push @exclude, @{$options->{Exclude}};
-            } else {
-                push @exclude, $options->{Exclude};
-            }
-        }
-        $options->{Exclude} = \@exclude;
+        push @exclude, @{$$options{Exclude}} if $$options{Exclude};
+        $$options{Exclude} = \@exclude;
         # expand shortcuts in new exclude list
-        ExpandShortcuts($options->{Exclude}, 1); # (also remove '#' suffix)
-    }
+        ExpandShortcuts($$options{Exclude}, 1); # (also remove '#' suffix)
+    }
+    # generate lookup for excluded tags
+    if ($$options{Exclude}) {
+        foreach (@{$$options{Exclude}}) {
+            /([-\w]+)#?$/ and $$self{EXCL_TAG_LOOKUP}{lc($1)} = 1;
+        }
+        # exclude list is used only for EXCL_TAG_LOOKUP when TAGS_FROM_FILE is set
+        undef $$options{Exclude} if $$self{TAGS_FROM_FILE};
+    }
+}
+
+#------------------------------------------------------------------------------
+# Does group name match the tag ID?
+# Inputs: 0) tag ID, 1) group name (with "ID-" removed)
+# Returns: true on success
+sub IsSameID($$)
+{
+    my ($id, $grp) = @_;
+    return 1 if $grp eq $id;    # decimal ID's or raw ID's
+    if ($id =~ /^\d+$/) {       # numerical numerical ID's may be in hex
+        return 1 if $grp =~ s/^0x0*// and $grp eq sprintf('%x', $id);
+    } else {                    # other ID's may conform to ExifTool group name conventions
+        return 1 if $id =~ s/([^-_A-Za-z0-9])/sprintf('%.2x',ord $1)/ge and $grp eq $id;
+    }
+    return 0;
 }
 
@@ -2669 +4287 @@
     $tagList = [ $tagList ] unless ref $tagList;
     my ($tag, @matches);
-    if ($group =~ /:/) {
-        # check each group name individually (ie. "Author:1IPTC")
-        my @grps = split ':', lc $group;
-        my (@fmys, $g);
+    # check each group name individually (eg. "Author:1IPTC")
+    my @grps = split ':', $group;
+    my (@fmys, $g);
+    for ($g=0; $g<@grps; ++$g) {
+        if ($grps[$g] =~ s/^(\d*)(id-)?//i) {
+            $fmys[$g] = $1 if length $1;
+            if ($2) {
+                $fmys[$g] = 7;
+                next;   # (don't convert tag ID's to lower case)
+            }
+        }
+        $grps[$g] = lc $grps[$g];
+        $grps[$g] = '' if $grps[$g] eq 'copy0'; # accept 'Copy0' for primary tag
+    }
+    foreach $tag (@$tagList) {
+        my @groups = $self->GetGroup($tag, -1);
         for ($g=0; $g<@grps; ++$g) {
-            $fmys[$g] = $1 if $grps[$g] =~ s/^(\d+)//;
-        }
-        foreach $tag (@$tagList) {
-            my @groups = $self->GetGroup($tag, -1);
-            for ($g=0; $g<@grps; ++$g) {
-                my $grp = $grps[$g];
-                next if $grp eq '*' or $grp eq 'all';
-                if (defined $fmys[$g]) {
-                    my $f = $fmys[$g];
-                    last unless $groups[$f] and $grps[$g] eq lc $groups[$f];
+            my $grp = $grps[$g];
+            next if $grp eq '*' or $grp eq 'all';
+            my $f;
+            if (defined($f = $fmys[$g])) {
+                last unless defined $groups[$f];
+                if ($f == 7) {
+                    next if IsSameID($self->GetTagID($tag), $grp);
                 } else {
-                    last unless grep /^$grps[$g]$/i, @groups;
-                }
-            }
-            push @matches, $tag if $g == @grps;
+                    next if $grp eq lc $groups[$f];
+                }
+                last;
+            } else {
+                last unless grep /^$grp$/i, @groups;
+            }
+        }
+        if ($g == @grps) {
+            return $tag unless wantarray;
+            push @matches, $tag;
+        }
+    }
+    return wantarray ? @matches : $matches[0];
+}
+
+#------------------------------------------------------------------------------
+# Remove specified tags from returned tag list, updating indices in other lists
+# Inputs: 0) tag list ref, 1) index list ref, 2) index list ref, 3) hash ref,
+#         4) true to include tags from hash instead of excluding
+# Returns: nothing, but updates input lists
+sub RemoveTagsFromList($$$$;$)
+{
+    local $_;
+    my ($tags, $list1, $list2, $exclude, $inv) = @_;
+    my @filteredTags;
+
+    if (@$list1 or @$list2) {
+        while (@$tags) {
+            my $tag = pop @$tags;
+            my $i = @$tags;
+            if ($$exclude{$tag} xor $inv) {
+                # remove index of excluded tag from each list
+                @$list1 = map { $_ < $i ? $_ : $_ == $i ? () : $_ - 1 } @$list1;
+                @$list2 = map { $_ < $i ? $_ : $_ == $i ? () : $_ - 1 } @$list2;
+            } else {
+                unshift @filteredTags, $tag;
+            }
         }
     } else {
-        my $family = ($group =~ s/^(\d+)//) ? $1 : -1;
-        foreach $tag (@$tagList) {
-            my @groups = $self->GetGroup($tag, $family);
-            push @matches, $tag if grep(/^$group$/i, @groups);
-        }
-    }
-    return wantarray ? @matches : $matches[0];
+        foreach (@$tags) {
+            push @filteredTags, $_ unless $$exclude{$_} xor $inv;
+        }
+    }
+    $_[0] = \@filteredTags;     # update tag list
 }
 
@@ -2705 +4363 @@
 # Returns: 0) Reference to list of found tag keys (in order of requested tags)
 #          1) Reference to list of indices for tags requested by value
+#          2) Reference to list of indices for tags specified by wildcard or "all"
+# Notes: index lists are returned in increasing order
 sub SetFoundTags($)
 {
     my $self = shift;
-    my $options = $self->{OPTIONS};
-    my $reqTags = $self->{REQUESTED_TAGS} || [ ];
-    my $duplicates = $options->{Duplicates};
-    my $exclude = $options->{Exclude};
-    my $fileOrder = $self->{FILE_ORDER};
+    my $options = $$self{OPTIONS};
+    my $reqTags = $$self{REQUESTED_TAGS} || [ ];
+    my $duplicates = $$options{Duplicates};
+    my $exclude = $$options{Exclude};
+    my $fileOrder = $$self{FILE_ORDER};
     my @groupOptions = sort grep /^Group/, keys %$options;
     my $doDups = $duplicates || $exclude || @groupOptions;
-    my ($tag, $rtnTags, @byValue);
+    my ($tag, $rtnTags, @byValue, @wildTags);
 
     # only return requested tags if specified
@@ -2721 +4381 @@
         $rtnTags or $rtnTags = [ ];
         # scan through the requested tags and generate a list of tags we found
-        my $tagHash = $self->{VALUE};
+        my $tagHash = $$self{VALUE};
         my $reqTag;
         foreach $reqTag (@$reqTags) {
@@ -2730 +4390 @@
                     $allGrp = 1;
                 } elsif ($group !~ /^[-\w:]*$/) {
-                    $self->Warn("Invalid group name '$group'");
+                    $self->Warn("Invalid group name '${group}'");
                     $group = 'invalid';
                 }
@@ -2736 +4396 @@
                 $tag = $reqTag;
             }
-            $byValue = 1 if $tag =~ s/#$//;
-            if (defined $tagHash->{$reqTag} and not $doDups) {
+            $byValue = 1 if $tag =~ s/#$// and $$options{PrintConv};
+            if (defined $$tagHash{$reqTag} and not $doDups) {
                 $matches[0] = $tag;
             } elsif ($tag =~ /^(\*|all)$/i) {
                 # tag name of '*' or 'all' matches all tags
                 if ($doDups or $allGrp) {
-                    @matches = keys %$tagHash;
+                    @matches = grep(!/#/, keys %$tagHash);
                 } else {
                     @matches = grep(!/ /, keys %$tagHash);
@@ -2752 +4412 @@
                 $tag =~ s/\*/[-\\w]*/g;
                 $tag =~ s/\?/[-\\w]/g;
-                $tag .= '( .*)?' if $doDups or $allGrp;
+                $tag .= '( \\(.*)?' if $doDups or $allGrp;
                 @matches = grep(/^$tag$/i, keys %$tagHash);
                 next unless @matches;   # don't want entry in list for wildcard tags
@@ -2758 +4418 @@
             } elsif ($doDups or defined $group) {
                 # must also look for tags like "Tag (1)"
-                @matches = grep(/^$tag( |$)/i, keys %$tagHash);
+                # (but be sure not to match temporary ValueConv entries like "Tag #")
+                @matches = grep(/^$tag( \(|$)/i, keys %$tagHash);
             } elsif ($tag =~ /^[-\w]+$/) {
                 # find first matching value
@@ -2765 +4426 @@
                 defined $matches[0] or undef @matches;
             } else {
-                $self->Warn("Invalid tag name '$tag'");
+                $self->Warn("Invalid tag name '${tag}'");
             }
             if (defined $group and not $allGrp) {
@@ -2778 +4439 @@
                 unless ($doDups or $allTag or $allGrp) {
                     $tag = shift @matches;
-                    my $oldPriority = $self->{PRIORITY}{$tag} || 1;
+                    my $oldPriority = $$self{PRIORITY}{$tag} || 1;
                     foreach (@matches) {
-                        my $priority = $self->{PRIORITY}{$_};
+                        my $priority = $$self{PRIORITY}{$_};
                         $priority = 1 unless defined $priority;
                         next unless $priority >= $oldPriority;
@@ -2790 +4451 @@
             } elsif (not @matches) {
                 # put entry in return list even without value (value is undef)
-                $matches[0] = "$tag (0)";
+                $matches[0] = $byValue ? "$tag #(0)" : "$tag (0)";
                 # bogus file order entry to avoid warning if sorting in file order
-                $self->{FILE_ORDER}{$matches[0]} = 999;
+                $$self{FILE_ORDER}{$matches[0]} = 9999;
             }
             # save indices of tags extracted by value
             push @byValue, scalar(@$rtnTags) .. (scalar(@$rtnTags)+scalar(@matches)-1) if $byValue;
+            # save indices of wildcard tags
+            push @wildTags, scalar(@$rtnTags) .. (scalar(@$rtnTags)+scalar(@matches)-1) if $allTag;
             push @$rtnTags, @matches;
         }
@@ -2802 +4465 @@
         my @allTags;
         if ($doDups) {
-            @allTags = keys %{$self->{VALUE}};
+            @allTags = keys %{$$self{VALUE}};
         } else {
-            foreach (keys %{$self->{VALUE}}) {
-                # only include tag if it doesn't end in a copy number
-                push @allTags, $_ unless / /;
-            }
+            # only include tag if it doesn't end in a copy number
+            @allTags = grep(!/ /, keys %{$$self{VALUE}});
         }
         $rtnTags = \@allTags;
@@ -2823 +4484 @@
                         undef $group;
                     } elsif ($group !~ /^[-\w:]*$/) {
-                        $self->Warn("Invalid group name '$group'");
+                        $self->Warn("Invalid group name '${group}'");
                         $group = 'invalid';
                     }
@@ -2842 +4503 @@
             }
             if (%exclude) {
-                my @filteredTags;
-                $exclude{$_} or push @filteredTags, $_ foreach @$rtnTags;
-                $rtnTags = \@filteredTags;      # use new filtered tag list
-                last unless @filteredTags;      # all done if nothing left
+                # remove excluded tags from return list(s)
+                RemoveTagsFromList($rtnTags, \@byValue, \@wildTags, \%exclude);
+                last unless @$rtnTags;      # all done if nothing left
             }
             last if $duplicates and not @groupOptions;
@@ -2859 +4519 @@
             $wantGroup{$family} or $wantGroup{$family} = { };
             my $groupList;
-            if (ref $options->{$groupOpt} eq 'ARRAY') {
-                $groupList = $options->{$groupOpt};
+            if (ref $$options{$groupOpt} eq 'ARRAY') {
+                $groupList = $$options{$groupOpt};
             } else {
-                $groupList = [ $options->{$groupOpt} ];
+                $groupList = [ $$options{$groupOpt} ];
             }
             foreach (@$groupList) {
@@ -2894 +4554 @@
             }
             next unless $wantTag;
-            if ($duplicates) {
-                push @tags, $tag;
-            } else {
-                my $tagName = GetTagName($tag);
-                my $bestTag = $bestTag{$tagName};
-                if (defined $bestTag) {
-                    next if $wantTag > $keepTags{$bestTag};
-                    if ($wantTag == $keepTags{$bestTag}) {
-                        # want two tags with the same name -- keep the latest one
-                        if ($tag =~ / \((\d+)\)$/) {
-                            my $tagNum = $1;
-                            next if $bestTag !~ / \((\d+)\)$/ or $1 > $tagNum;
-                        }
+            $duplicates and $keepTags{$tag} = 1, next;
+            # determine which tag we want to keep
+            my $tagName = GetTagName($tag);
+            my $bestTag = $bestTag{$tagName};
+            if (defined $bestTag) {
+                next if $wantTag > $keepTags{$bestTag};
+                if ($wantTag == $keepTags{$bestTag}) {
+                    # want two tags with the same name -- keep the latest one
+                    if ($tag =~ / \((\d+)\)$/) {
+                        my $tagNum = $1;
+                        next if $bestTag !~ / \((\d+)\)$/ or $1 > $tagNum;
                     }
-                    # this tag is better, so delete old best tag
-                    delete $keepTags{$bestTag};
-                }
-                $keepTags{$tag} = $wantTag;    # keep this tag (for now...)
-                $bestTag{$tagName} = $tag;      # this is our current best tag
-            }
-        }
-        unless ($duplicates) {
-            # construct new tag list with no duplicates, preserving order
-            foreach $tag (@$rtnTags) {
-                push @tags, $tag if $keepTags{$tag};
-            }
-        }
-        $rtnTags = \@tags;
+                }
+                # this tag is better, so delete old best tag
+                delete $keepTags{$bestTag};
+            }
+            $keepTags{$tag} = $wantTag;     # keep this tag (for now...)
+            $bestTag{$tagName} = $tag;      # this is our current best tag
+        }
+        # include only tags we want to keep in return lists
+        RemoveTagsFromList($rtnTags, \@byValue, \@wildTags, \%keepTags, 1);
         last;
     }
-    $self->{FOUND_TAGS} = $rtnTags;     # save found tags 
+    $$self{FOUND_TAGS} = $rtnTags;      # save found tags
 
     # return reference to found tag keys (and list of indices of tags to extract by value)
-    return wantarray ? ($rtnTags, \@byValue) : $rtnTags;
+    return wantarray ? ($rtnTags, \@byValue, \@wildTags) : $rtnTags;
 }
 
@@ -2934 +4587 @@
 # Inputs: 0) autoload function, 1-N) function arguments
 # Returns: result of function or dies if function not available
-# To Do: Generalize this routine so it works on systems that don't use '/'
-#        as a path name separator.
 sub DoAutoLoad(@)
 {
@@ -2946 +4597 @@
         # load Image/ExifTool/WriteMODULE.pl
         $file .= "$callInfo[2].pl";
+    } elsif ($callInfo[-1] eq 'ShiftTime') {
+        $file = 'Image/ExifTool/Shift.pl';  # load Shift.pl
     } else {
         # load Image/ExifTool/Writer.pl
@@ -2951 +4604 @@
     }
     # attempt to load the package
-    eval "require '$file'" or die "Error while attempting to call $autoload\n$@\n";
+    eval { require $file } or die "Error while attempting to call $autoload\n$@\n";
     unless (defined &$autoload) {
         my @caller = caller(0);
@@ -2971 +4624 @@
 #------------------------------------------------------------------------------
 # Add warning tag
-# Inputs: 0) ExifTool object reference, 1) warning message, 2) true if minor
+# Inputs: 0) ExifTool object reference, 1) warning message
+#         2) true if minor (2 if behaviour changes when warning is ignored,
+#            or 3 if warning shouldn't be issued when Validate option is used)
 # Returns: true if warning tag was added
 sub Warn($$;$)
@@ -2977 +4632 @@
     my ($self, $str, $ignorable) = @_;
     if ($ignorable) {
-        return 0 if $self->{OPTIONS}{IgnoreMinorErrors};
-        $str = "[minor] $str";
+        return 0 if $$self{OPTIONS}{IgnoreMinorErrors};
+        return 0 if $ignorable eq '3' and $$self{OPTIONS}{Validate};
+        $str = $ignorable eq '2' ? "[Minor] $str" : "[minor] $str";
     }
     $self->FoundTag('Warning', $str);
@@ -2991 +4647 @@
 {
     my ($self, $str, $ignorable) = @_;
-    return 0 if $ignorable and $self->{OPTIONS}{IgnoreMinorErrors};
+    return 0 if $ignorable and $$self{OPTIONS}{IgnoreMinorErrors};
     unless ($$self{WARNED_ONCE}{$str}) {
         $self->Warn($str, $ignorable);
@@ -3006 +4662 @@
 {
     my ($self, $str, $ignorable) = @_;
-    if ($ignorable) {
-        if ($self->{OPTIONS}{IgnoreMinorErrors}) {
-            $self->Warn($str);
-            return 0;
-        }
+    if ($$self{DemoteErrors}) {
+        $self->Warn($str) and ++$$self{DemoteErrors};
+        return 1;
+    } elsif ($ignorable) {
+        $$self{OPTIONS}{IgnoreMinorErrors} and $self->Warn($str), return 0;
         $str = "[minor] $str";
     }
@@ -3116 +4772 @@
 # Add hash of Composite tags to our composites
 # Inputs: 0) hash reference to table of Composite tags to add or module name,
-#         1) overwrite existing tag
+#         1) override existing tag definition
 sub AddCompositeTags($;$)
 {
     local $_;
-    my ($add, $overwrite) = @_;
-    my $module;
+    my ($add, $override) = @_;
+    my ($module, $prefix, $tagID);
     unless (ref $add) {
+        ($prefix = $add) =~ s/.*:://;
         $module = $add;
         $add .= '::Composite';
         no strict 'refs';
         $add = \%$add;
+        $prefix .= '-';
+    } else {
+        $prefix = 'UserDefined-';
     }
     my $defaultGroups = $$add{GROUPS};
+    my $compTable = GetTagTable('Image::ExifTool::Composite');
 
     # make sure default groups are defined in families 0 and 1
     if ($defaultGroups) {
-        $defaultGroups->{0} or $defaultGroups->{0} = 'Composite';
-        $defaultGroups->{1} or $defaultGroups->{1} = 'Composite';
-        $defaultGroups->{2} or $defaultGroups->{2} = 'Other';
+        $$defaultGroups{0} or $$defaultGroups{0} = 'Composite';
+        $$defaultGroups{1} or $$defaultGroups{1} = 'Composite';
+        $$defaultGroups{2} or $$defaultGroups{2} = 'Other';
     } else {
         $defaultGroups = $$add{GROUPS} = { 0 => 'Composite', 1 => 'Composite', 2 => 'Other' };
     }
-    SetupTagTable($add);    # generate tag Name, etc
-    my $tagID;
+    SetupTagTable($add);    # generate Name, TagID, etc
     foreach $tagID (sort keys %$add) {
         next if $specialTags{$tagID};   # must skip special tags
         my $tagInfo = $$add{$tagID};
-        # tagID's MUST be the exact tag name for logic in BuildCompositeTags()
-        my $tag = $$tagInfo{Name};
+        my $new = $prefix . $tagID;     # new tag ID for Composite table
         $$tagInfo{Module} = $module if $$tagInfo{Writable};
-        # allow Composite tags with the same name
-        my ($t, $n, $type);
-        while ($Image::ExifTool::Composite{$tag} and not $overwrite) {
-            $n ? $n += 1 : ($n = 2, $t = $tag);
-            $tag = "${t}_$n";
-            $$tagInfo{NewTagID} = $tag; # save new ID so we can use it in TagLookup
-        }
-        # convert scalar Require/Desire entries
-        foreach $type ('Require','Desire') {
+        $$tagInfo{Override} = 1 if $override and not defined $$tagInfo{Override};
+        $$tagInfo{IsComposite} = 1;
+        # handle Composite tags with the same name
+        if ($compositeID{$tagID}) {
+            # determine if we want to override this tag
+            # (=0 keep both, >0 override, <0 keep existing)
+            my $over = ($$tagInfo{Override} || 0) - ($$compTable{$compositeID{$tagID}[0]}{Override} || 0);
+            next if $over < 0;
+            if ($over) {
+                # remove existing tags with this ID
+                delete $$compTable{$_} foreach @{$compositeID{$tagID}};
+                delete $compositeID{$tagID};
+            }
+        }
+        # make sure new TagID is unique by adding index if necessary
+        # (could only happen for UserDefined tags now that module name is added to tag ID)
+        my $n = 0;
+        while ($$compTable{$new}) {
+            $new =~ s/-\d+$// if $n++;
+            $new .= "-$n";
+        }
+        # use new ID and save it so we can use it in TagLookup
+        $$tagInfo{NewTagID} = $new unless $tagID eq $new;
+
+        # add new ID to lookup of Composite tag ID's
+        $compositeID{$tagID} = [ ] unless $compositeID{$tagID};
+        unshift @{$compositeID{$tagID}}, $new;  # (most recent one first)
+
+        # convert scalar Require/Desire/Inhibit entries
+        my ($type, @hashes, @scalars, %used);
+        foreach $type ('Require','Desire','Inhibit') {
             my $req = $$tagInfo{$type} or next;
-            $$tagInfo{$type} = { 0 => $req } if ref($req) ne 'HASH';
+            push @{ref($req) eq 'HASH' ? \@hashes : \@scalars}, $type;
+        }
+        if (@scalars) {
+            # make lookup for indices that are used
+            foreach $type (@hashes) {
+                $used{$_} = 1 foreach keys %{$$tagInfo{$type}};
+            }
+            my $next = 0;
+            foreach $type (@scalars) {
+                ++$next while $used{$next};
+                $$tagInfo{$type} = { $next++ => $$tagInfo{$type} };
+            }
         }
         # add this Composite tag to our main Composite table
-        $$tagInfo{Table} = \%Image::ExifTool::Composite;
-        # (use the original TagID, even if we changed it)
-        # $$tagInfo{TagID} = $tag;
-        # save new tag ID so we can find entry in Composite table
-        $Image::ExifTool::Composite{$tag} = $tagInfo;
+        $$tagInfo{Table} = $compTable;
+        # (use the original TagID, even if we changed it, so don't do this:)
+        $$tagInfo{TagID} = $new;
+        # save tag under new ID in Composite table
+        $$compTable{$new} = $tagInfo;
         # set all default groups in tag
         my $groups = $$tagInfo{Groups};
@@ -3223 +4915 @@
 {
     my $tagTablePtr = shift;
+    my $avoid = $$tagTablePtr{AVOID};
     my ($tagID, $tagInfo);
     foreach $tagID (TagTableKeys($tagTablePtr)) {
@@ -3230 +4923 @@
             $$tagInfo{Table} = $tagTablePtr;
             $$tagInfo{TagID} = $tagID;
-            my $tag = $$tagInfo{Name};
-            unless (defined $tag) {
-                # generate name equal to tag ID if 'Name' doesn't exist
-                $tag = $tagID;
-                $$tagInfo{Name} = ucfirst($tag); # make first char uppercase
-            }
+            $$tagInfo{Name} or $$tagInfo{Name} = MakeTagName($tagID);
             $$tagInfo{Flags} and ExpandFlags($tagInfo);
+            $$tagInfo{Avoid} = $avoid if defined $avoid;
+            # calculate BitShift from Mask if necessary
+            if ($$tagInfo{Mask} and not defined $$tagInfo{BitShift}) {
+                my ($mask, $bitShift) = ($$tagInfo{Mask}, 0);
+                ++$bitShift until $mask & (1 << $bitShift);
+                $$tagInfo{BitShift} = $bitShift;
+            }
         }
         next unless @infoArray > 1;
@@ -3267 +4962 @@
 {
     my ($val, $sig) = @_;
-    $val == 0 and return 0;
-    my $sign = $val < 0 ? ($val=-$val, -1) : 1;
-    my $log = log($val) / log(10);
-    my $exp = int($log) - $sig + ($log > 0 ? 1 : 0);
-    return $sign * int(10 ** ($log - $exp) + 0.5) * 10 ** $exp;
+    return sprintf("%.${sig}g", $val);
 }
 
@@ -3396 +5087 @@
 sub GetDouble($$) { return DoUnpackDbl('d', @_); }
 sub Get16uRev($$) { return DoUnpackRev('S', @_); }
+sub Get32uRev($$) { return DoUnpackRev('L', @_); }
 
 # rationals may be a floating point number, 'inf' or 'undef'
+my ($ratNumer, $ratDenom);
 sub GetRational32s($$)
 {
     my ($dataPt, $pos) = @_;
-    my $numer = Get16s($dataPt,$pos);
-    my $denom = Get16s($dataPt, $pos + 2) or return $numer ? 'inf' : 'undef';
+    $ratNumer = Get16s($dataPt,$pos);
+    $ratDenom = Get16s($dataPt, $pos + 2) or return $ratNumer ? 'inf' : 'undef';
     # round off to a reasonable number of significant figures
-    return RoundFloat($numer / $denom, 7);
+    return RoundFloat($ratNumer / $ratDenom, 7);
 }
 sub GetRational32u($$)
 {
     my ($dataPt, $pos) = @_;
-    my $numer = Get16u($dataPt,$pos);
-    my $denom = Get16u($dataPt, $pos + 2) or return $numer ? 'inf' : 'undef';
-    return RoundFloat($numer / $denom, 7);
+    $ratNumer = Get16u($dataPt,$pos);
+    $ratDenom = Get16u($dataPt, $pos + 2) or return $ratNumer ? 'inf' : 'undef';
+    return RoundFloat($ratNumer / $ratDenom, 7);
 }
 sub GetRational64s($$)
 {
     my ($dataPt, $pos) = @_;
-    my $numer = Get32s($dataPt,$pos);
-    my $denom = Get32s($dataPt, $pos + 4) or return $numer ? 'inf' : 'undef';
-    return RoundFloat($numer / $denom, 10);
+    $ratNumer = Get32s($dataPt,$pos);
+    $ratDenom = Get32s($dataPt, $pos + 4) or return $ratNumer ? 'inf' : 'undef';
+    return RoundFloat($ratNumer / $ratDenom, 10);
 }
 sub GetRational64u($$)
 {
     my ($dataPt, $pos) = @_;
-    my $numer = Get32u($dataPt,$pos);
-    my $denom = Get32u($dataPt, $pos + 4) or return $numer ? 'inf' : 'undef';
-    return RoundFloat($numer / $denom, 10);
+    $ratNumer = Get32u($dataPt,$pos);
+    $ratDenom = Get32u($dataPt, $pos + 4) or return $ratNumer ? 'inf' : 'undef';
+    return RoundFloat($ratNumer / $ratDenom, 10);
 }
 sub GetFixed16s($$)
@@ -3524 +5217 @@
     int32s => 4,
     int32u => 4,
+    int32uRev => 4,
     int64s => 8,
     int64u => 8,
@@ -3534 +5228 @@
     fixed32s => 4,
     fixed32u => 4,
+    fixed64s => 8,
     float => 4,
     double => 8,
@@ -3544 +5239 @@
     ifd => 4,
     ifd64 => 8,
+    ue7 => 1,
 );
 my %readValueProc = (
@@ -3553 +5249 @@
     int32s => \&Get32s,
     int32u => \&Get32u,
+    int32uRev => \&Get32uRev,
     int64s => \&Get64s,
     int64u => \&Get64u,
@@ -3563 +5260 @@
     fixed32s => \&GetFixed32s,
     fixed32u => \&GetFixed32u,
+    fixed64s => \&GetFixed64s,
     float => \&GetFloat,
     double => \&GetDouble,
@@ -3569 +5267 @@
     ifd64 => \&Get64u,
 );
+# lookup for all rational types
+my %isRational = (
+    rational32u => 1,
+    rational32s => 1,
+    rational64u => 1,
+    rational64s => 1,
+);
 sub FormatSize($) { return $formatSize{$_[0]}; }
 
@@ -3574 +5279 @@
 # Read value from binary data (with current byte ordering)
 # Inputs: 0) data reference, 1) value offset, 2) format string,
-#         3) number of values (or undef to use all data)
-#         4) valid data length relative to offset
+#         3) number of values (or undef to use all data),
+#         4) valid data length relative to offset (or undef to use all data),
+#         5) optional pointer to returned rational
 # Returns: converted value, or undefined if data isn't there
 #          or list of values in list context
-sub ReadValue($$$$$)
-{
-    my ($dataPt, $offset, $format, $count, $size) = @_;
+sub ReadValue($$$;$$$)
+{
+    my ($dataPt, $offset, $format, $count, $size, $ratPt) = @_;
 
     my $len = $formatSize{$format};
@@ -3587 +5293 @@
         $len = 1;
     }
+    $size = length($$dataPt) - $offset unless defined $size;
     unless ($count) {
         return '' if defined $count or $size < $len;
@@ -3598 +5305 @@
     my @vals;
     my $proc = $readValueProc{$format};
-    if ($proc) {
+    if (not $proc) {
+        # handle undef/binary/string (also unsupported unicode/complex)
+        $vals[0] = substr($$dataPt, $offset, $count * $len);
+        # truncate string at null terminator if necessary
+        $vals[0] =~ s/\0.*//s if $format eq 'string';
+    } elsif ($isRational{$format} and $ratPt) {
+        # store rationals separately as string fractions
+        my @rat;
+        for (;;) {
+            push @vals, &$proc($dataPt, $offset);
+            push @rat, "$ratNumer/$ratDenom";
+            last if --$count <= 0;
+            $offset += $len;
+        }
+        $$ratPt = join(' ',@rat);
+    } else {
         for (;;) {
             push @vals, &$proc($dataPt, $offset);
@@ -3604 +5326 @@
             $offset += $len;
         }
-    } else {
-        # handle undef/binary/string (also unsupported unicode/complex)
-        $vals[0] = substr($$dataPt, $offset, $count * $len);
-        # truncate string at null terminator if necessary
-        $vals[0] =~ s/\0.*//s if $format eq 'string';
     }
     return @vals if wantarray;
@@ -3706 +5423 @@
     {
         # issue warning only if the tag was specifically requested
-        if ($self->{REQ_TAG_LOOKUP}{lc GetTagName($tag)}) {
+        if ($$self{REQ_TAG_LOOKUP}{lc GetTagName($tag)}) {
             $self->Warn("$tag is not a valid JPEG image",1);
             return undef;
@@ -3712 +5429 @@
     }
     return $imagePt;
+}
+
+#------------------------------------------------------------------------------
+# Validate a tag name argument (including group name and wildcards, etc)
+# Inputs: 0) tag name
+# Returns: true if tag name is valid
+# - a tag name may contain [-_A-Za-z0-9], but may not start with [-0-9]
+# - tag names may contain wildcards [?*], and end with a hash [#]
+# - may have group name prefixes (which may have family number prefix), separated by colons
+# - a group name may be zero or more characters
+sub ValidTagName($)
+{
+    my $tag = shift;
+    return $tag =~ /^(([-\w]*|\d*\*):)*[_a-zA-Z?*][-\w?*]*#?$/;
+}
+
+#------------------------------------------------------------------------------
+# Generate a valid tag name based on the tag ID or name
+# Inputs: 0) tag ID or name
+# Returns: valid tag name
+sub MakeTagName($)
+{
+    my $name = shift;
+    $name =~ tr/-_a-zA-Z0-9//dc;    # remove illegal characters
+    $name = ucfirst $name;          # capitalize first letter
+    $name = "Tag$name" if length($name) < 2; # must at least 2 characters long
+    return $name;
 }
 
@@ -3731 +5475 @@
     # put a space between acronyms and words
     $desc =~ s/([A-Z])([A-Z][a-z])/$1 $2/g;
-    # put spaces after numbers (if more than one character following number)
+    # put spaces after numbers (if more than one character follows the number)
     $desc =~ s/(\d)([A-Z]\S)/$1 $2/g;
     # add TagID to description
@@ -3739 +5483 @@
 
 #------------------------------------------------------------------------------
+# Get descriptions for all tags in an array
+# Inputs: 0) ExifTool ref, 1) reference to list of tag keys
+# Returns: reference to hash lookup for descriptions
+# Note: Returned descriptions are NOT escaped by ESCAPE_PROC
+sub GetDescriptions($$)
+{
+    local $_;
+    my ($self, $tags) = @_;
+    my %desc;
+    my $oldEscape = $$self{ESCAPE_PROC};
+    delete $$self{ESCAPE_PROC};
+    $desc{$_} = $self->GetDescription($_) foreach @$tags;
+    $$self{ESCAPE_PROC} = $oldEscape;
+    return \%desc;
+}
+
+#------------------------------------------------------------------------------
+# Apply filter to value(s) if necessary
+# Inputs: 0) ExifTool ref, 1) filter expression, 2) reference to value to filter
+# Returns: true unless a filter returned undef; changes value if necessary
+sub Filter($$$)
+{
+    local $_;
+    my ($self, $filter, $valPt) = @_;
+    return 1 unless defined $filter and defined $$valPt;
+    my $rtnVal;
+    if (not ref $$valPt) {
+        $_ = $$valPt;
+        #### eval Filter ($_, $self)
+        eval $filter;
+        if (defined $_) {
+            $$valPt = $_;
+            $rtnVal = 1;
+        }
+    } elsif (ref $$valPt eq 'SCALAR') {
+        my $val = $$$valPt; # make a copy to avoid filtering twice
+        $rtnVal = $self->Filter($filter, \$val);
+        $$valPt = \$val;
+    } elsif (ref $$valPt eq 'ARRAY') {
+        my @val = @{$$valPt}; # make a copy to avoid filtering twice
+        $self->Filter($filter, \$_) and $rtnVal = 1 foreach @val;
+        $$valPt = \@val;
+    } elsif (ref $$valPt eq 'HASH') {
+        my %val = %{$$valPt}; # make a copy to avoid filtering twice
+        $self->Filter($filter, \$val{$_}) and $rtnVal = 1 foreach keys %val;
+        $$valPt = \%val;
+    } else {
+        $rtnVal = 1;
+    }
+    return $rtnVal;
+}
+
+#------------------------------------------------------------------------------
 # Return printable value
 # Inputs: 0) ExifTool object reference
@@ -3748 +5545 @@
     $outStr =~ tr/\x01-\x1f\x7f-\xff/./;
     $outStr =~ s/\x00//g;
-    if (defined $maxLen) {
-        # minimum length is 20 (0 is unlimited)
-        $maxLen = 20 if $maxLen and $maxLen < 20;
+    my $verbose = $$self{OPTIONS}{Verbose};
+    if ($verbose < 4) {
+        if ($maxLen) {
+            $maxLen = 20 if $maxLen < 20;   # minimum length is 20
+        } elsif (defined $maxLen) {
+            $maxLen = length $outStr;       # 0 is unlimited
+        } else {
+            $maxLen = 60;                   # default maximum is 60
+        }
     } else {
-        $maxLen = 60;                   # default length is 60
-    }
-    # limit length only if verbose < 4
-    if ($maxLen and length($outStr) > $maxLen and $self->{OPTIONS}{Verbose} < 4) {
-        $outStr = substr($outStr,0,$maxLen-6) . '[snip]';
-    }
+        $maxLen = length $outStr;
+        # limit to 2048 characters if verbose < 5
+        $maxLen = 2048 if $maxLen > 2048 and $verbose < 5;
+    }
+
+    # limit length if necessary
+    $outStr = substr($outStr,0,$maxLen-6) . '[snip]' if length($outStr) > $maxLen;
     return $outStr;
 }
@@ -3768 +5572 @@
 {
     my ($self, $date) = @_;
-    my $dateFormat = $self->{OPTIONS}{DateFormat};
+    my $fmt = $$self{OPTIONS}{DateFormat};
+    my $shift = $$self{OPTIONS}{GlobalTimeShift};
+    if ($shift) {
+        my $dir = ($shift =~ s/^([-+])// and $1 eq '-') ? -1 : 1;
+        my $offset = $$self{GLOBAL_TIME_OFFSET};
+        $offset or $offset = $$self{GLOBAL_TIME_OFFSET} = { };
+        ShiftTime($date, $shift, $dir, $offset);
+    }
     # only convert date if a format was specified and the date is recognizable
-    if ($dateFormat) {
+    if ($fmt) {
+        # separate time zone if it exists
+        my $tz;
+        $date =~ s/([-+]\d{2}:\d{2}|Z)$// and $tz = $1;
         # a few cameras use incorrect date/time formatting:
         # - slashes instead of colons in date (RolleiD330, ImpressCam)
         # - date/time values separated by colon instead of space (Polariod, Sanyo, Sharp, Vivitar)
         # - single-digit seconds with leading space (HP scanners)
-        $date =~ s/[-+]\d{2}:\d{2}$//;  # remove timezone if it exists
-        my @a = ($date =~ /\d+/g);      # be very flexible about date/time format
-        if (@a and $a[0] > 1900 and $a[0] < 3000 and eval 'require POSIX') {
-            $date = POSIX::strftime($dateFormat, $a[5]||0, $a[4]||0, $a[3]||0,
-                                                 $a[2]||1, ($a[1]||1)-1, $a[0]-1900);
-        } elsif ($self->{OPTIONS}{StrictDate}) {
+        my @a = reverse ($date =~ /\d+/g);  # be very flexible about date/time format
+        if (@a and $a[-1] >= 1000 and $a[-1] < 3000 and eval { require POSIX }) {
+            shift @a while @a > 6;      # remove superfluous entries
+            unshift @a, 1 while @a < 3; # add month and day if necessary
+            unshift @a, 0 while @a < 6; # add h,m,s if necessary
+            $a[4] -= 1;                 # base month is 1
+            # parse %z and %s ourself (to handle time zones properly)
+            if ($fmt =~ /%[sz]/) {
+                # use system time zone unless otherwise specified
+                $tz = TimeZoneString(\@a, TimeLocal(@a)) if not $tz and eval { require Time::Local };
+                # remove colon, setting to UTC if time zone is not numeric
+                $tz = ($tz and $tz=~/^([-+]\d{2}):(\d{2})$/) ? "$1$2" : '+0000';
+                $fmt =~ s/(^|[^%])((%%)*)%z/$1$2$tz/g;      # convert '%z' format codes
+                if ($fmt =~ /%s/ and eval { require Time::Local }) {
+                    # calculate seconds since the Epoch, UTC
+                    my $s = Time::Local::timegm(@a) - 60 * ($tz - int($tz/100) * 40);
+                    $fmt =~ s/(^|[^%])((%%)*)%s/$1$2$s/g;   # convert '%s' format codes
+                }
+            }
+            $a[5] -= 1900;  # strftime year starts from 1900
+            $date = POSIX::strftime($fmt, @a);  # generate the formatted date/time
+        } elsif ($$self{OPTIONS}{StrictDate}) {
             undef $date;
         }
@@ -3832 +5662 @@
 # Inputs: 0) localtime array ref, 1) gmtime array ref
 # Returns: time zone offset in minutes
-sub GetTimeZone(;$$)
+sub GetTimeZone($$)
 {
     my ($tm, $gm) = @_;
@@ -3843 +5673 @@
         $min += ($$tm[3] - $$gm[3]) * 24 * 60;
     }
+    # MirBSD patch to round to the nearest 30 minutes because
+    # it includes leap seconds in localtime but not gmtime
+    $min = int($min / 30 + ($min > 0 ? 0.5 : -0.5)) * 30 if $^O eq 'mirbsd';
     return $min;
 }
@@ -3860 +5693 @@
     my $sign = '+';
     $min < 0 and $sign = '-', $min = -$min;
+    $min = int($min + 0.5); # round off to nearest minute
     my $h = int($min / 60);
     return sprintf('%s%.2d:%.2d', $sign, $h, $min - $h * 60);
@@ -3866 +5700 @@
 #------------------------------------------------------------------------------
 # Convert Unix time to EXIF date/time string
-# Inputs: 0) Unix time value, 1) non-zero to convert to local time
+# Inputs: 0) Unix time value, 1) non-zero to convert to local time,
+#         2) number of digits after the decimal for fractional seconds
 # Returns: EXIF date/time string (with timezone for local times)
-# Notes: fractional seconds are ignored
-sub ConvertUnixTime($;$)
-{
-    my ($time, $toLocal) = @_;
+sub ConvertUnixTime($;$$)
+{
+    my ($time, $toLocal, $dec) = @_;
     return '0000:00:00 00:00:00' if $time == 0;
     my (@tm, $tz);
+    if ($dec) {
+        my $frac = $time - int($time);
+        $time = int($time);
+        $frac < 0 and $frac += 1, $time -= 1;
+        $dec = sprintf('%.*f', $dec, $frac);
+        # remove number before decimal and increment integer time if it was rounded up
+        $dec =~ s/^(\d)// and $1 eq '1' and $time += 1;
+    } else {
+        $time = int($time + 1e-6) if $time != int($time);  # avoid round-off errors
+        $dec = '';
+    }
     if ($toLocal) {
         @tm = localtime($time);
@@ -3881 +5726 @@
         $tz = '';
     }
-    my $str = sprintf("%4d:%.2d:%.2d %.2d:%.2d:%.2d%s",
+    my $str = sprintf("%4d:%.2d:%.2d %.2d:%.2d:%.2d$dec%s",
                       $tm[5]+1900, $tm[4]+1, $tm[3], $tm[2], $tm[1], $tm[0], $tz);
     return $str;
@@ -3888 +5733 @@
 #------------------------------------------------------------------------------
 # Get Unix time from EXIF-formatted date/time string with optional timezone
-# Inputs: 0) EXIF date/time string, 1) non-zero if time is local
+# Inputs: 0) EXIF date/time string, 1) non-zero if time is local, or 2 to assume UTC
 # Returns: Unix time (seconds since 0:00 GMT Jan 1, 1970) or undefined on error
 sub GetUnixTime($;$)
@@ -3894 +5739 @@
     my ($timeStr, $isLocal) = @_;
     return 0 if $timeStr eq '0000:00:00 00:00:00';
-    my @tm = ($timeStr =~ /^(\d+):(\d+):(\d+)\s+(\d+):(\d+):(\d+)/);
-    return undef unless @tm == 6 and eval 'require Time::Local';
-    my $tzsec = 0;
+    my @tm = ($timeStr =~ /^(\d+):(\d+):(\d+)\s+(\d+):(\d+):(\d+)(.*)/);
+    return undef unless @tm == 7 and eval { require Time::Local };
+    my ($tzStr, $tzSec) = (pop(@tm), 0);
     # use specified timezone offset (if given) instead of local system time
     # if we are converting a local time value
-    if ($isLocal and $timeStr =~ /(?:Z|([-+])(\d+):(\d+))$/i) {
-        # use specified timezone if one exists
-        $tzsec = ($2 * 60 + $3) * ($1 eq '-' ? -60 : 60) if $1;
-        undef $isLocal; # convert using GMT corrected for specified timezone
-    }
-    $tm[0] -= 1900;     # convert year
+    if ($isLocal) {
+        if ($tzStr =~ /(?:Z|([-+])(\d+):(\d+))/i) {
+            # use specified timezone if one exists
+            $tzSec = ($2 * 60 + $3) * ($1 eq '-' ? -60 : 60) if $1;
+            undef $isLocal; # convert using GMT corrected for specified timezone
+        } elsif ($isLocal eq '2') {
+            undef $isLocal;
+        }
+    }
     $tm[1] -= 1;        # convert month
     @tm = reverse @tm;  # change to order required by timelocal()
-    return $isLocal ? TimeLocal(@tm) : Time::Local::timegm(@tm) - $tzsec;
+    my $val = $isLocal ? TimeLocal(@tm) : Time::Local::timegm(@tm) - $tzSec;
+    # handle fractional seconds
+    $val += $1 if $tzStr and $tzStr =~ /^(\.\d+)/;
+    return $val;
 }
 
@@ -3918 +5769 @@
     my $val = shift;
     $val < 2048 and return "$val bytes";
-    $val < 10240 and return sprintf('%.1f kB', $val / 1024);
-    $val < 2097152 and return sprintf('%.0f kB', $val / 1024);
-    $val < 10485760 and return sprintf('%.1f MB', $val / 1048576);
-    return sprintf('%.0f MB', $val / 1048576);
+    $val < 10240 and return sprintf('%.1f KiB', $val / 1024);
+    $val < 2097152 and return sprintf('%.0f KiB', $val / 1024);
+    $val < 10485760 and return sprintf('%.1f MiB', $val / 1048576);
+    $val < 2147483648 and return sprintf('%.0f MiB', $val / 1048576);
+    $val < 10737418240 and return sprintf('%.1f GiB', $val / 1073741824);
+    return sprintf('%.0f GiB', $val / 1073741824);
 }
 
@@ -3927 +5780 @@
 # Convert seconds to duration string (handles negative durations)
 # Inputs: 0) floating point seconds
-# Returns: duration string in form "S.SS s", "MM:SS" or "H:MM:SS"
+# Returns: duration string in form "S.SS s", "H:MM:SS" or "DD days HH:MM:SS"
 sub ConvertDuration($)
 {
@@ -3935 +5788 @@
     my $sign = ($time > 0 ? '' : (($time = -$time), '-'));
     return sprintf("$sign%.2f s", $time) if $time < 30;
+    $time += 0.5;   # to round off to nearest second
     my $h = int($time / 3600);
     $time -= $h * 3600;
     my $m = int($time / 60);
     $time -= $m * 60;
+    if ($h > 24) {
+        my $d = int($h / 24);
+        $h -= $d * 24;
+        $sign = "$sign$d days ";
+    }
     return sprintf("$sign%d:%.2d:%.2d", $h, $m, int($time));
 }
@@ -3961 +5820 @@
 
 #------------------------------------------------------------------------------
+# Convert file name for printing
+# Inputs: 0) ExifTool ref, 1) file name in CharsetFileName character set
+# Returns: converted file name in external character set
+sub ConvertFileName($$)
+{
+    my ($self, $val) = @_;
+    my $enc = $$self{OPTIONS}{CharsetFileName};
+    $val = $self->Decode($val, $enc) if $enc;
+    return $val;
+}
+
+#------------------------------------------------------------------------------
+# Inverse conversion for file name (encode in CharsetFileName)
+# Inputs: 0) ExifTool ref, 1) file name in external character set
+# Returns: file name in CharsetFileName character set
+sub InverseFileName($$)
+{
+    my ($self, $val) = @_;
+    my $enc = $$self{OPTIONS}{CharsetFileName};
+    $val = $self->Encode($val, $enc) if $enc;
+    $val =~ tr/\\/\//;  # make sure we are using forward slashes
+    return $val;
+}
+
+#------------------------------------------------------------------------------
 # Save information for HTML dump
 # Inputs: 0) ExifTool hash ref, 1) start offset, 2) data size
-#         3) comment string, 4) tool tip (or SAME), 5) flags
-sub HDump($$$$;$$)
+#         3) comment string, 4) tool tip (or SAME), 5) flags, 6) IFD name
+sub HDump($$$$;$$$)
 {
     my $self = shift;
-    my $pos = shift;
+    $$self{HTML_DUMP} or return;
+    my ($pos, $len, $com, $tip, $flg, $ifd) = @_;
     $pos += $$self{BASE} if $$self{BASE};
-    $$self{HTML_DUMP} and $self->{HTML_DUMP}->Add($pos, @_);
-}
-
-#------------------------------------------------------------------------------
-# JPEG constants
-my %jpegMarker = (
-    0x01 => 'TEM',
-    0xc0 => 'SOF0', # to SOF15, with a few exceptions below
-    0xc4 => 'DHT',
-    0xc8 => 'JPGA',
-    0xcc => 'DAC',
-    0xd0 => 'RST0',
-    0xd8 => 'SOI',
-    0xd9 => 'EOI',
-    0xda => 'SOS',
-    0xdb => 'DQT',
-    0xdc => 'DNL',
-    0xdd => 'DRI',
-    0xde => 'DHP',
-    0xdf => 'EXP',
-    0xe0 => 'APP0', # to APP15
-    0xf0 => 'JPG0',
-    0xfe => 'COM',
-);
-
-#------------------------------------------------------------------------------
-# Get JPEG marker name
-# Inputs: 0) Jpeg number
-# Returns: marker name
-sub JpegMarkerName($)
-{
-    my $marker = shift;
-    my $markerName = $jpegMarker{$marker};
-    unless ($markerName) {
-        $markerName = $jpegMarker{$marker & 0xf0};
-        if ($markerName and $markerName =~ /^([A-Z]+)\d+$/) {
-            $markerName = $1 . ($marker & 0x0f);
-        } else {
-            $markerName = sprintf("marker 0x%.2x", $marker);
-        }
-    }
-    return $markerName;
+    # skip structural data blocks which have been removed from the middle of this dump
+    # (SkipData list contains ordered [start,end+1] offsets to skip)
+    if ($$self{SkipData}) {
+        my $end = $pos + $len;
+        my $skip;
+        foreach $skip (@{$$self{SkipData}}) {
+            $end <= $$skip[0] and last;
+            $pos >= $$skip[1] and $pos += $$skip[1] - $$skip[0], next;
+            if ($pos != $$skip[0]) {
+                $$self{HTML_DUMP}->Add($pos, $$skip[0]-$pos, $com, $tip, $flg, $ifd);
+                $len -= $$skip[0] - $pos;
+                $tip = 'SAME';
+            }
+            $pos = $$skip[1];
+        }
+    }
+    $$self{HTML_DUMP}->Add($pos, $len, $com, $tip, $flg, $ifd);
 }
 
@@ -4041 +5901 @@
         {
             $type = 'MIE';
+        } elsif ($buff =~ /\0\0(QDIOBS|SEFT)$/) {
+            $type = 'Samsung';
+        } elsif ($buff =~ /8db42d694ccc418790edff439fe026bf$/s) {
+            $type = 'Insta360';
         }
         last;
@@ -4072 +5936 @@
 
     for (;;) { # loop through all trailers
-        require "Image/ExifTool/$dirName.pm";
-        my $proc = "Image::ExifTool::${dirName}::Process$dirName";
-        my $outBuff;
+        my ($proc, $outBuff);
+        if ($dirName eq 'Insta360') {
+            require "Image/ExifTool/QuickTimeStream.pl";
+            $proc = 'Image::ExifTool::QuickTime::ProcessInsta360';
+        } else {
+            require "Image/ExifTool/$dirName.pm";
+            $proc = "Image::ExifTool::${dirName}::Process$dirName";
+        }
         if ($outfile) {
             # write to local buffer so we can add trailer in proper order later
@@ -4089 +5958 @@
 
         # read or write this trailer
-        # (proc takes Offset as offset from end of trailer to end of file,
-        #  and returns DataPos and DirLen, and Fixup if applicable)
+        # (proc takes Offset as positive offset from end of trailer to end of file,
+        #  and returns DataPos and DirLen, and Fixup if applicable, and updates
+        #  OutFile when writing)
         no strict 'refs';
         my $result = &$proc($self, $dirInfo);
         use strict 'refs';
 
-        # restore PATH
-        pop @$path;
-        pop @$path;
+        # restore PATH (pop last 2 items)
+        splice @$path, -2;
+
         # check result
         if ($outfile) {
@@ -4108 +5978 @@
                     $outBuff = '';      # free memory
                 }
-                if ($fixup) {
-                    # add new fixup information if any
-                    $fixup->AddFixup($$dirInfo{Fixup}) if $$dirInfo{Fixup};
-                } else {
+                if ($$dirInfo{Fixup}) {
+                    if ($fixup) {
+                        # add fixup for subsequent trailers to the fixup for this trailer
+                        # (but first we must adjust for the new start position)
+                        $$fixup{Shift} += $$dirInfo{Fixup}{Start};
+                        $$fixup{Start} -= $$dirInfo{Fixup}{Start};
+                        $$dirInfo{Fixup}->AddFixup($fixup);
+                    }
                     $fixup = $$dirInfo{Fixup};  # save fixup
                 }
             } else {
-                $success = 0 if $self->Error("Error rewriting $dirName trailer", 1);
+                $success = 0 if $self->Error("Error rewriting $dirName trailer", 2);
                 last;
             }
@@ -4139 +6013 @@
 
 #------------------------------------------------------------------------------
-# Extract EXIF information from a jpg image
+# JPEG constants
+
+# JPEG marker names
+%jpegMarker = (
+    0x00 => 'NULL',
+    0x01 => 'TEM',
+    0xc0 => 'SOF0', # to SOF15, with a few exceptions below
+    0xc4 => 'DHT',
+    0xc8 => 'JPGA',
+    0xcc => 'DAC',
+    0xd0 => 'RST0', # to RST7
+    0xd8 => 'SOI',
+    0xd9 => 'EOI',
+    0xda => 'SOS',
+    0xdb => 'DQT',
+    0xdc => 'DNL',
+    0xdd => 'DRI',
+    0xde => 'DHP',
+    0xdf => 'EXP',
+    0xe0 => 'APP0', # to APP15
+    0xf0 => 'JPG0',
+    0xfe => 'COM',
+);
+
+# lookup for size of JPEG marker length word
+# (2 bytes assumed unless specified here)
+my %markerLenBytes = (
+    0x00 => 0,  0x01 => 0,
+    0xd0 => 0,  0xd1 => 0,  0xd2 => 0,  0xd3 => 0,  0xd4 => 0,  0xd5 => 0,  0xd6 => 0,  0xd7 => 0,
+    0xd8 => 0,  0xd9 => 0,  0xda => 0,
+    # J2C
+    0x30 => 0,  0x31 => 0,  0x32 => 0,  0x33 => 0,  0x34 => 0,  0x35 => 0,  0x36 => 0,  0x37 => 0,
+    0x38 => 0,  0x39 => 0,  0x3a => 0,  0x3b => 0,  0x3c => 0,  0x3d => 0,  0x3e => 0,  0x3f => 0,
+    0x4f => 0,
+    0x92 => 0,  0x93 => 0,
+    # J2C extensions
+    0x74 => 4, 0x75 => 4, 0x77 => 4,
+);
+
+#------------------------------------------------------------------------------
+# Get JPEG marker name
+# Inputs: 0) Jpeg number
+# Returns: marker name
+sub JpegMarkerName($)
+{
+    my $marker = shift;
+    my $markerName = $jpegMarker{$marker};
+    unless ($markerName) {
+        $markerName = $jpegMarker{$marker & 0xf0};
+        if ($markerName and $markerName =~ /^([A-Z]+)\d+$/) {
+            $markerName = $1 . ($marker & 0x0f);
+        } else {
+            $markerName = sprintf("marker 0x%.2x", $marker);
+        }
+    }
+    return $markerName;
+}
+
+#------------------------------------------------------------------------------
+# Adjust directory start position
+# Inputs: 0) dirInfo ref, 1) start offset
+#         2) Base for offsets (relative to DataPos, defaults to absolute Base of 0)
+sub DirStart($$;$)
+{
+    my ($dirInfo, $start, $base) = @_;
+    $$dirInfo{DirStart} = $start;
+    $$dirInfo{DirLen} -= $start;
+    if (defined $base) {
+        $$dirInfo{Base} = $$dirInfo{DataPos} + $base;
+        $$dirInfo{DataPos} = -$base;    # (relative to Base!)
+    }
+}
+
+#------------------------------------------------------------------------------
+# Extract metadata from a jpg image
 # Inputs: 0) ExifTool object reference, 1) dirInfo ref with RAF set
 # Returns: 1 on success, 0 if this wasn't a valid JPEG file
@@ -4147 +6095 @@
     my ($self, $dirInfo) = @_;
     my ($ch, $s, $length);
-    my $verbose = $self->{OPTIONS}{Verbose};
-    my $out = $self->{OPTIONS}{TextOut};
-    my $fast = $self->{OPTIONS}{FastScan};
+    my $options = $$self{OPTIONS};
+    my $verbose = $$options{Verbose};
+    my $out = $$options{TextOut};
+    my $fast = $$options{FastScan} || 0;
     my $raf = $$dirInfo{RAF};
-    my $htmlDump = $self->{HTML_DUMP};
+    my $htmlDump = $$self{HTML_DUMP};
     my %dumpParms = ( Out => $out );
-    my ($success, $icc_profile, $wantTrailer, $trailInfo, %extendedXMP);
-    my ($preview, $scalado, @dqt, $subSampling, $dumpEnd);
-
-    # check to be sure this is a valid JPG file
-    return 0 unless $raf->Read($s, 2) == 2 and $s eq "\xff\xd8";
+    my ($success, $wantTrailer, $trailInfo, $foundSOS);
+    my (@iccChunk, $iccChunkCount, $iccChunksTotal, @flirChunk, $flirCount, $flirTotal);
+    my ($preview, $scalado, @dqt, $subSampling, $dumpEnd, %extendedXMP);
+
+    # check to be sure this is a valid JPG (or J2C, or EXV) file
+    return 0 unless $raf->Read($s, 2) == 2 and $s =~ /^\xff[\xd8\x4f\x01]/;
+    if ($s eq "\xff\x01") {
+        return 0 unless $raf->Read($s, 5) == 5 and $s eq 'Exiv2';
+        $$self{FILE_TYPE} = 'EXV';
+    }
+    my $appBytes = 0;
+    my $calcImageLen = $$self{REQ_TAG_LOOKUP}{jpegimagelength};
+    if ($$options{RequestAll} and $$options{RequestAll} > 2) {
+        $calcImageLen = 1;
+    }
+    if (not $$self{VALUE}{FileType} or ($$self{DOC_NUM} and $$options{ExtractEmbedded})) {
+        $self->SetFileType();               # set FileType tag
+        return 1 if $fast == 3;             # don't process file when FastScan == 3
+        $$self{LOW_PRIORITY_DIR}{IFD1} = 1; # lower priority of IFD1 tags
+    }
+    $$raf{NoBuffer} = 1 if $self->Options('FastScan'); # disable buffering in FastScan mode
+
     $dumpParms{MaxLen} = 128 if $verbose < 4;
-    unless ($self->{VALUE}{FileType}) {
-        $self->SetFileType();               # set FileType tag
-        $$self{LOW_PRIORITY_DIR}{IFD1} = 1; # lower priority of IFD1 tags
-    }
     if ($htmlDump) {
         $dumpEnd = $raf->Tell();
-        my $pos = $dumpEnd - 2;
+        my ($n, $t, $m) = $s eq 'Exiv2' ? (7,'EXV','TEM') : (2,'JPEG','SOI');
+        my $pos = $dumpEnd - $n;
         $self->HDump(0, $pos, '[unknown header]') if $pos;
-        $self->HDump($pos, 2, 'JPEG header', 'SOI Marker');
+        $self->HDump($pos, $n, "$t header", "$m Marker");
     }
     my $path = $$self{PATH};
@@ -4175 +6138 @@
     local $/ = "\xff";
 
-    my ($nextMarker, $nextSegDataPt, $nextSegPos, $combinedSegData);
+    my ($nextMarker, $nextSegDataPt, $nextSegPos, $combinedSegData, $firstSegPos, @skipData);
 
     # read file until we reach an end of image (EOI) or start of scan (SOS)
@@ -4186 +6149 @@
         undef $nextSegDataPt;
 #
-# read ahead to the next segment unless we have reached EOI or SOS
+# read ahead to the next segment unless we have reached EOI, SOS or SOD
 #
-        unless ($marker and ($marker==0xd9 or ($marker==0xda and not $wantTrailer))) {
+        unless ($marker and ($marker==0xd9 or ($marker==0xda and not $wantTrailer) or $marker==0x93)) {
             # read up to next marker (JPEG markers begin with 0xff)
             my $buff;
@@ -4198 +6161 @@
                 last unless $nextMarker == 0xff;
             }
-            # read data for all markers except 0xd9 (EOI) and stand-alone
-            # markers 0x00, 0x01 and 0xd0-0xd7 (NULL, TEM, RST0-RST7)
-            if ($nextMarker!=0xd9 and $nextMarker!=0x00 and $nextMarker!=0x01 and
-                ($nextMarker<0xd0 or $nextMarker>0xd7))
-            {
+            # read segment data if it exists
+            if (not defined $markerLenBytes{$nextMarker}) {
                 # read record length word
                 last unless $raf->Read($s, 2) == 2;
@@ -4211 +6171 @@
                 last unless $raf->Read($buff, $len) == $len;
                 $nextSegDataPt = \$buff;    # set pointer to our next data
+            } elsif ($markerLenBytes{$nextMarker} == 4) {
+                # handle J2C extensions with 4-byte length word
+                last unless $raf->Read($s, 4) == 4;
+                my $len = unpack('N',$s);   # get data length
+                last unless defined($len) and $len >= 4;
+                $nextSegPos = $raf->Tell();
+                $len -= 4;  # subtract size of length word
+                last unless $raf->Seek($len, 1);
             }
             # read second segment too if this was the first
@@ -4227 +6195 @@
                 print $out "JPEG $markerName ($length bytes):\n";
                 HexDump($segDataPt, undef, %dumpParms, Addr=>$segPos) if $verbose>2;
+            } elsif ($htmlDump) {
+                $self->HDump($segPos-4, $length+4, "[JPEG $markerName]", undef, 0x08);
+                $dumpEnd = $segPos + $length;
             }
             next unless $length >= 6;
@@ -4232 +6203 @@
             my ($p, $h, $w, $n) = unpack('Cn2C', $$segDataPt);
             my $sof = GetTagTable('Image::ExifTool::JPEG::SOF');
-            $self->FoundTag($$sof{ImageWidth}, $w);
-            $self->FoundTag($$sof{ImageHeight}, $h);
-            $self->FoundTag($$sof{EncodingProcess}, $marker - 0xc0);
-            $self->FoundTag($$sof{BitsPerSample}, $p);
-            $self->FoundTag($$sof{ColorComponents}, $n);
+            $self->HandleTag($sof, 'ImageWidth', $w);
+            $self->HandleTag($sof, 'ImageHeight', $h);
+            $self->HandleTag($sof, 'EncodingProcess', $marker - 0xc0);
+            $self->HandleTag($sof, 'BitsPerSample', $p);
+            $self->HandleTag($sof, 'ColorComponents', $n);
             next unless $n == 3 and $length >= 15;
             my ($i, $hmin, $hmax, $vmin, $vmax);
@@ -4259 +6230 @@
             if ($hmin and $vmin) {
                 my ($hs, $vs) = ($hmax / $hmin, $vmax / $vmin);
-                $self->FoundTag($$sof{YCbCrSubSampling}, "$hs $vs");
+                $self->HandleTag($sof, 'YCbCrSubSampling', "$hs $vs");
             }
             next;
@@ -4271 +6242 @@
                 $dumpEnd = 0;
             }
-            $success = 1;
+            if ($foundSOS or $$self{FILE_TYPE} eq 'EXV') {
+                $success = 1;
+            } else {
+                $self->Warn('Missing JPEG SOS');
+            }
+            if ($$self{REQ_TAG_LOOKUP}{trailer}) {
+                # read entire trailer into memory
+                if ($raf->Seek(0,2)) {
+                    my $len = $raf->Tell() - $pos;
+                    if ($len) {
+                        my $buff;
+                        $raf->Seek($pos, 0);
+                        $self->FoundTag(Trailer => \$buff) if $raf->Read($buff,$len) == $len;
+                        $raf->Seek($pos, 0);
+                    }
+                } else {
+                    $self->Warn('Error seeking in file');
+                }
+            }
             # we are here because we are looking for trailer information
             if ($wantTrailer) {
                 my $start = $$self{PreviewImageStart};
-                if ($start) {
+                if ($start or $$options{ExtractEmbedded}) {
                     my $buff;
                     # most previews start right after the JPEG EOI, but the Olympus E-20
@@ -4282 +6271 @@
                     # (and Minolta and Sony previews can have a random first byte...)
                     my $scanLen = $$self{Make} =~ /Sony/i ? 65536 : 1024;
-                    if ($raf->Read($buff, $scanLen) and ($buff =~ /\xff\xd8\xff./g or
-                        ($self->{Make} =~ /(Minolta|Sony)/i and $buff =~ /.\xd8\xff\xdb/g)))
-                    {
-                        # adjust PreviewImageStart to this location
-                        my $actual = $pos + pos($buff) - 4;
-                        if ($start ne $actual and $verbose > 1) {
-                            print $out "(Fixed PreviewImage location: $start -> $actual)\n";
-                        }
-                        # update preview image offsets
-                        $self->{VALUE}{PreviewImageStart} = $actual if $self->{VALUE}{PreviewImageStart};
-                        $$self{PreviewImageStart} = $actual;
-                        # load preview now if we tried and failed earlier
-                        if ($$self{PreviewError} and $$self{PreviewImageLength}) {
-                            if ($raf->Seek($actual, 0) and $raf->Read($buff, $$self{PreviewImageLength})) {
-                                $self->FoundTag('PreviewImage', $buff);
-                                delete $$self{PreviewError};
+                    if ($raf->Read($buff, $scanLen)) {
+                        if ($buff =~ /^.{4}ftyp/s) {
+                            my $val;
+                            if ($raf->Seek(0,2)) {
+                                my $len = $raf->Tell() - $pos;
+                                if ($$options{Binary}) {
+                                    $val = \$buff if $raf->Seek($pos,0) and $raf->Read($buff,$len)==$len;
+                                } else {
+                                    $val = \ "Binary data $len bytes";
+                                }
+                                if ($val) {
+                                    $self->FoundTag('EmbeddedVideo', $val);
+                                } else {
+                                    $self->Warn('Error reading trailer');
+                                }
+                            } else {
+                                $self->Warn('Error seeking to end of file');
+                            }
+                        } elsif ($buff =~ /\xff\xd8\xff./g or
+                           ($$self{Make} =~ /(Minolta|Sony)/i and $buff =~ /.\xd8\xff\xdb/g))
+                        {
+                            # adjust PreviewImageStart to this location
+                            my $actual = $pos + pos($buff) - 4;
+                            if ($start and $start ne $actual and $verbose > 1) {
+                                print $out "(Fixed PreviewImage location: $start -> $actual)\n";
+                            }
+                            # update preview image offsets
+                            if ($start) {
+                                $$self{VALUE}{PreviewImageStart} = $actual if $$self{VALUE}{PreviewImageStart};
+                                $$self{PreviewImageStart} = $actual;
+                            }
+                            # load preview now if we tried and failed earlier
+                            if ($$self{PreviewError} and $$self{PreviewImageLength}) {
+                                if ($raf->Seek($actual, 0) and $raf->Read($buff, $$self{PreviewImageLength})) {
+                                    $self->FoundTag('PreviewImage', $buff);
+                                    delete $$self{PreviewError};
+                                }
                             }
                         }
@@ -4333 +6343 @@
                 }) if $endPos > $pos;
             }
+            $self->FoundTag('JPEGImageLength', $pos - $appBytes) if $calcImageLen;
             last;       # all done parsing file
         } elsif ($marker == 0xda) {         # SOS
             pop @$path;
+            $foundSOS = 1;
             # all done with meta information unless we have a trailer
             $verbose and print $out "JPEG SOS\n";
@@ -4346 +6358 @@
                     $self->ProcessTrailers($trailInfo) and undef $trailInfo;
                 }
-                if ($wantTrailer) {
+                if ($wantTrailer and $$self{PreviewImageStart}) {
                     # seek ahead and validate preview image
                     my $buff;
@@ -4363 +6375 @@
                     Image::ExifTool::Panasonic::ProcessLeicaTrailer($self);
                     $wantTrailer = 1 if $$self{LeicaTrailer};
+                } else {
+                    $wantTrailer = 1 if $$options{ExtractEmbedded};
                 }
                 next if $trailInfo or $wantTrailer or $verbose > 2 or $htmlDump;
             }
+            # must scan to EOI if Validate or JpegCompressionFactor used
+            next if $$options{Validate} or $calcImageLen or $$self{REQ_TAG_LOOKUP}{trailer};
             # nothing interesting to parse after start of scan (SOS)
             $success = 1;
             last;   # all done parsing file
-        } elsif ($marker==0x00 or $marker==0x01 or ($marker>=0xd0 and $marker<=0xd7)) {
-            # handle stand-alone markers 0x00, 0x01 and 0xd0-0xd7 (NULL, TEM, RST0-RST7)
-            $verbose and $marker and print $out "JPEG $markerName:\n";
+        } elsif ($marker == 0x93) {
+            pop @$path;
+            $verbose and print $out "JPEG SOD\n";
+            $success = 1;
+            next if $verbose > 2 or $htmlDump;
+            last;   # all done parsing file
+        } elsif (defined $markerLenBytes{$marker}) {
+            # handle other stand-alone markers and segments we skipped over
+            $verbose and $marker and print $out "JPEG $markerName\n";
             next;
         } elsif ($marker == 0xdb and length($$segDataPt) and    # DQT
             # save the DQT data only if JPEGDigest has been requested
-            $self->{REQ_TAG_LOOKUP}->{jpegdigest})
+            # (Note: since we aren't checking the RequestAll API option here, the application
+            #  must use the RequestTags option to generate these tags if they have not been
+            #  specifically requested.  The reason is that there is too much overhead involved
+            #  in the calculation of this tag to make this worth the CPU time.)
+            ($$self{REQ_TAG_LOOKUP}{jpegdigest} or $$self{REQ_TAG_LOOKUP}{jpegqualityestimate}
+            or ($$options{RequestAll} and $$options{RequestAll} > 2)))
         {
             my $num = unpack('C',$$segDataPt) & 0x0f;   # get table index
@@ -4382 +6409 @@
         # handle all other markers
         my $dumpType = '';
+        my ($desc, $tip, $xtra);
         $length = length $$segDataPt;
+        $appBytes += $length + 4 if ($marker & 0xf0) == 0xe0;  # total size of APP segments
         if ($verbose) {
             print $out "JPEG $markerName ($length bytes):\n";
@@ -4391 +6420 @@
             }
         }
+        # prepare dirInfo hash for processing this information
+        my %dirInfo = (
+            Parent   => $markerName,
+            DataPt   => $segDataPt,
+            DataPos  => $segPos,
+            DataLen  => $length,
+            DirStart => 0,
+            DirLen   => $length,
+            Base     => 0,
+        );
         if ($marker == 0xe0) {              # APP0 (JFIF, JFXX, CIFF, AVI1, Ocad)
             if ($$segDataPt =~ /^JFIF\0/) {
                 $dumpType = 'JFIF';
-                my %dirInfo = (
-                    DataPt => $segDataPt,
-                    DataPos  => $segPos,
-                    DirStart => 5,
-                    DirLen => $length - 5,
-                );
+                DirStart(\%dirInfo, 5); # start at byte 5
                 SetByteOrder('MM');
                 my $tagTablePtr = GetTagTable('Image::ExifTool::JFIF::Main');
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
-            } elsif ($$segDataPt =~ /^JFXX\0\x10/) {
+            } elsif ($$segDataPt =~ /^JFXX\0(\x10|\x11|\x13)/) {
+                my $tag = ord $1;
                 $dumpType = 'JFXX';
                 my $tagTablePtr = GetTagTable('Image::ExifTool::JFIF::Extension');
-                my $tagInfo = $self->GetTagInfo($tagTablePtr, 0x10);
+                my $tagInfo = $self->GetTagInfo($tagTablePtr, $tag);
                 $self->FoundTag($tagInfo, substr($$segDataPt, 6));
             } elsif ($$segDataPt =~ /^(II|MM).{4}HEAPJPGM/s) {
-                next if $fast and $fast > 1;    # skip processing for very fast
+                next if $fast > 1;      # skip processing for very fast
                 $dumpType = 'CIFF';
-                my %dirInfo = (
-                    RAF => new File::RandomAccess($segDataPt),
-                );
-                $self->{SET_GROUP1} = 'CIFF';
+                my %dirInfo = ( RAF => new File::RandomAccess($segDataPt) );
+                $$self{SET_GROUP1} = 'CIFF';
+                push @{$$self{PATH}}, 'CIFF';
                 require Image::ExifTool::CanonRaw;
                 Image::ExifTool::CanonRaw::ProcessCRW($self, \%dirInfo);
-                delete $self->{SET_GROUP1};
+                pop @{$$self{PATH}};
+                delete $$self{SET_GROUP1};
             } elsif ($$segDataPt =~ /^(AVI1|Ocad)/) {
                 $dumpType = $1;
                 SetByteOrder('MM');
                 my $tagTablePtr = GetTagTable("Image::ExifTool::JPEG::$dumpType");
-                my %dirInfo = (
-                    DataPt   => $segDataPt,
-                    DataPos  => $segPos,
-                    DirStart => 4,
-                    DirLen   => $length - 4,
-                );
+                DirStart(\%dirInfo, 4);
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
             }
-        } elsif ($marker == 0xe1) {         # APP1 (EXIF, XMP, QVCI)
-            if ($$segDataPt =~ /^Exif\0/) { # (some Kodak cameras don't put a second \0)
+        } elsif ($marker == 0xe1) {         # APP1 (EXIF, XMP, QVCI, PARROT)
+            # (some Kodak cameras don't put a second "\0", and I have seen an
+            #  example where there was a second 4-byte APP1 segment header)
+            if ($$segDataPt =~ /^(.{0,4})Exif\0/is) {
                 undef $dumpType;    # (will be dumped here)
                 # this is EXIF data --
                 # get the data block (into a common variable)
                 my $hdrLen = length($exifAPP1hdr);
-                my %dirInfo = (
-                    Parent => $markerName,
-                    DataPt => $segDataPt,
-                    DataPos => $segPos,
-                    DirStart => $hdrLen,
-                    Base => $segPos + $hdrLen,
-                );
+                if (length $1) {
+                    $hdrLen += length $1;
+                    $self->Warn('Unknown garbage at start of EXIF segment',1);
+                } elsif ($$segDataPt !~ /^Exif\0/) {
+                    $self->Warn('Incorrect EXIF segment identifier',1);
+                }
                 if ($htmlDump) {
                     $self->HDump($segPos-4, 4, 'APP1 header', "Data size: $length bytes");
@@ -4448 +6479 @@
                     $dumpEnd = $segPos + $length;
                 }
+                my $dataPt = $segDataPt;
+                if (defined $combinedSegData) {
+                    push @skipData, [ $segPos-4, $segPos+$hdrLen ];
+                    $combinedSegData .= substr($$segDataPt,$hdrLen);
+                    undef $$segDataPt;
+                    $dataPt = \$combinedSegData;
+                    $segPos = $firstSegPos;
+                }
+                # peek ahead to see if the next segment is extended EXIF
+                if ($nextMarker == $marker and
+                    $$nextSegDataPt =~ /^$exifAPP1hdr(?!(MM\0\x2a|II\x2a\0))/)
+                {
+                    # initialize combined data if necessary
+                    unless (defined $combinedSegData) {
+                        $combinedSegData = $$segDataPt;
+                        undef $$segDataPt;
+                        $firstSegPos = $segPos;
+                        $self->Warn('File contains multi-segment EXIF',1);
+                        $$self{ExtendedEXIF} = 1;
+                    }
+                    next;
+                }
+                $dirInfo{DataPt} = $dataPt;
+                $dirInfo{DataPos} = $segPos;
+                $dirInfo{DataLen} = $dirInfo{DirLen} = length $$dataPt;
+                DirStart(\%dirInfo, $hdrLen, $hdrLen);
+                $$self{SkipData} = \@skipData if @skipData;
                 # extract the EXIF information (it is in standard TIFF format)
-                $self->ProcessTIFF(\%dirInfo);
+                $self->ProcessTIFF(\%dirInfo) or $self->Warn('Malformed APP1 EXIF segment');
                 # avoid looking for preview unless necessary because it really slows
                 # us down -- only look for it if we found pointer, and preview is
                 # outside EXIF, and PreviewImage is specifically requested
-                my $start = $self->GetValue('PreviewImageStart');
-                my $length = $self->GetValue('PreviewImageLength');
-                if (not $start or not $length and $$self{PreviewError}) {
+                my $start = $self->GetValue('PreviewImageStart', 'ValueConv');
+                my $plen = $self->GetValue('PreviewImageLength', 'ValueConv');
+                if (not $start or not $plen and $$self{PreviewError}) {
                     $start = $$self{PreviewImageStart};
-                    $length = $$self{PreviewImageLength};
-                }
-                if ($start and $length and
-                    $start + $length > $self->{EXIF_POS} + length($self->{EXIF_DATA}) and
-                    $self->{REQ_TAG_LOOKUP}{previewimage})
+                    $plen = $$self{PreviewImageLength};
+                }
+                if ($start and $plen and IsInt($start) and IsInt($plen) and
+                    $start + $plen > $$self{EXIF_POS} + length($$self{EXIF_DATA}) and
+                    ($$self{REQ_TAG_LOOKUP}{previewimage} or
+                    # (extracted normally, so check Binary option)
+                    ($$options{Binary} and not $$self{EXCL_TAG_LOOKUP}{previewimage})))
                 {
                     $$self{PreviewImageStart} = $start;
-                    $$self{PreviewImageLength} = $length;
+                    $$self{PreviewImageLength} = $plen;
                     $wantTrailer = 1;
                 }
+                if (@skipData) {
+                    undef @skipData;
+                    delete $$self{SkipData};
+                }
+                undef $$dataPt;
+                next;
             } elsif ($$segDataPt =~ /^$xmpExtAPP1hdr/) {
                 # off len -- extended XMP header (75 bytes total):
@@ -4474 +6540 @@
                 #  71   4 bytes - offset for this XMP data portion
                 $dumpType = 'Extended XMP';
-                if (length $$segDataPt > 75) {
+                if ($length > 75) {
                     my ($size, $off) = unpack('x67N2', $$segDataPt);
                     my $guid = substr($$segDataPt, 35, 32);
-                    my $extXMP = $extendedXMP{$guid};
-                    $extXMP or $extXMP = $extendedXMP{$guid} = { };
-                    $$extXMP{Size} = $size;
-                    $$extXMP{$off} = substr($$segDataPt, 75);
-                    # process extended XMP if complete
-                    my @offsets;
-                    for ($off=0; $off<$size; ) {
-                        last unless defined $$extXMP{$off};
-                        push @offsets, $off;
-                        $off += length $$extXMP{$off};
-                    }
-                    if ($off == $size) {
-                        my $buff = '';
-                        # assemble XMP all together
-                        $buff .= $$extXMP{$_} foreach @offsets;
-                        $dumpType = 'Extended XMP';
-                        my $tagTablePtr = GetTagTable('Image::ExifTool::XMP::Main');
-                        my %dirInfo = (
-                            DataPt   => \$buff,
-                            Parent   => $markerName,
-                        );
-                        $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
-                        delete $extendedXMP{$guid};
+                    if ($guid =~ /[^A-Za-z0-9]/) { # (technically, should be uppercase)
+                        $self->WarnOnce($tip = 'Invalid extended XMP GUID');
+                    } else {
+                        my $extXMP = $extendedXMP{$guid};
+                        if (not $extXMP) {
+                            $extXMP = $extendedXMP{$guid} = { };
+                        } elsif ($size != $$extXMP{Size}) {
+                            $self->WarnOnce('Inconsistent extended XMP size');
+                        }
+                        $$extXMP{Size} = $size;
+                        $$extXMP{$off} = substr($$segDataPt, 75);
+                        $tip = "Full length: $size\nChunk offset: $off\nChunk length: " .
+                            ($length - 75) . "\nGUID: $guid";
+                        # (delay processing extended XMP until after reading all segments)
                     }
                 } else {
-                    $self->Warn('Invalid extended XMP segment');
+                    $self->WarnOnce($tip = 'Invalid extended XMP segment');
                 }
             } elsif ($$segDataPt =~ /^QVCI\0/) {
                 $dumpType = 'QVCI';
                 my $tagTablePtr = GetTagTable('Image::ExifTool::Casio::QVCI');
-                my %dirInfo = (
-                    Base     => 0,
-                    DataPt   => $segDataPt,
-                    DataPos  => $segPos,
-                    DataLen  => $length,
-                    DirStart => 0,
-                    DirLen   => $length,
-                    Parent   => $markerName,
-                );
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
+            } elsif ($$segDataPt =~ /^FLIR\0/ and $length >= 8) {
+                $dumpType = 'FLIR';
+                # must concatenate FLIR chunks (note: handle the case where
+                # some software erroneously writes zeros for the chunk counts)
+                my $chunkNum = Get8u($segDataPt, 6);
+                my $chunksTot = Get8u($segDataPt, 7) + 1; # (note the "+ 1"!)
+                $verbose and printf $out "$$self{INDENT}FLIR chunk %d of %d\n",
+                                    $chunkNum + 1, $chunksTot;
+                if (defined $flirTotal) {
+                    # abort parsing FLIR if the total chunk count is inconsistent
+                    undef $flirCount if $chunksTot != $flirTotal;
+                } else {
+                    $flirCount = 0;
+                    $flirTotal = $chunksTot;
+                }
+                if (defined $flirCount) {
+                    if (defined $flirChunk[$chunkNum]) {
+                        $self->WarnOnce('Duplicate FLIR chunk number(s)');
+                        $flirChunk[$chunkNum] .= substr($$segDataPt, 8);
+                    } else {
+                        $flirChunk[$chunkNum] = substr($$segDataPt, 8);
+                    }
+                    # process the FLIR information if we have all of the chunks
+                    if (++$flirCount >= $flirTotal) {
+                        my $flir = '';
+                        defined $_ and $flir .= $_ foreach @flirChunk;
+                        undef @flirChunk;   # free memory
+                        my $tagTablePtr = GetTagTable('Image::ExifTool::FLIR::FFF');
+                        my %dirInfo = (
+                            DataPt   => \$flir,
+                            Parent   => $markerName,
+                            DirName  => 'FLIR',
+                        );
+                        $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
+                        undef $flirCount;   # prevent reprocessing
+                    }
+                } else {
+                    $self->WarnOnce('Invalid or extraneous FLIR chunk(s)');
+                }
+            } elsif ($$segDataPt =~ /^PARROT\0(II\x2a\0|MM\0\x2a)/) {
+                # (don't know if this could span multiple segments)
+                my $tagTablePtr = GetTagTable('Image::ExifTool::JPEG::Main');
+                $self->HandleTag($tagTablePtr, 'APP1', $$segDataPt);
+                $dumpType = 'Parrot';
             } else {
                 # Hmmm.  Could be XMP, let's see
                 my $processed;
-                if ($$segDataPt =~ /^http/ or $$segDataPt =~ /<exif:/) {
+                if ($$segDataPt =~ /^(http|XMP\0)/ or $$segDataPt =~ /<(exif:|\?xpacket)/) {
                     $dumpType = 'XMP';
                     # also try to parse XMP with a non-standard header
@@ -4526 +6618 @@
                     my $start = ($$segDataPt =~ /^$xmpAPP1hdr/) ? length($xmpAPP1hdr) : 0;
                     my $tagTablePtr = GetTagTable('Image::ExifTool::XMP::Main');
-                    my %dirInfo = (
-                        Base     => 0,
-                        DataPt   => $segDataPt,
-                        DataPos  => $segPos,
-                        DataLen  => $length,
-                        DirStart => $start,
-                        DirLen   => $length - $start,
-                        Parent   => $markerName,
-                    );
+                    DirStart(\%dirInfo, $start);
+                    $dirInfo{DirName} = $start ? 'XMP' : 'XML',
                     $processed = $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
                     if ($processed and not $start) {
@@ -4541 +6626 @@
                 }
                 if ($verbose and not $processed) {
-                    $self->Warn("Ignored EXIF block length $length (bad header)");
+                    $self->Warn("Ignored APP1 segment length $length (unknown header)");
                 }
             }
         } elsif ($marker == 0xe2) {         # APP2 (ICC Profile, FPXR, MPF, PreviewImage)
-            if ($$segDataPt =~ /^ICC_PROFILE\0/) {
+            if ($$segDataPt =~ /^ICC_PROFILE\0/ and $length >= 14) {
                 $dumpType = 'ICC_Profile';
-                # must concatenate blocks of profile
-                my $block_num = Get8u($segDataPt, 12);
-                my $blocks_tot = Get8u($segDataPt, 13);
-                $icc_profile = '' if $block_num == 1;
-                if (defined $icc_profile) {
-                    $icc_profile .= substr($$segDataPt, 14);
-                    if ($block_num == $blocks_tot) {
+                # must concatenate profile chunks (note: handle the case where
+                # some software erroneously writes zeros for the chunk counts)
+                my $chunkNum = Get8u($segDataPt, 12);
+                my $chunksTot = Get8u($segDataPt, 13);
+                $verbose and print $out "$$self{INDENT}ICC_Profile chunk $chunkNum of $chunksTot\n";
+                if (defined $iccChunksTotal) {
+                    # abort parsing ICC_Profile if the total chunk count is inconsistent
+                    undef $iccChunkCount if $chunksTot != $iccChunksTotal;
+                } else {
+                    $iccChunkCount = 0;
+                    $iccChunksTotal = $chunksTot;
+                    $self->Warn('ICC_Profile chunk count is zero') if !$chunksTot;
+                }
+                if (defined $iccChunkCount) {
+                    if (defined $iccChunk[$chunkNum]) {
+                        $self->WarnOnce('Duplicate ICC_Profile chunk number(s)');
+                        $iccChunk[$chunkNum] .= substr($$segDataPt, 14);
+                    } else {
+                        $iccChunk[$chunkNum] = substr($$segDataPt, 14);
+                    }
+                    # process profile if we have all of the chunks
+                    if (++$iccChunkCount >= $iccChunksTotal) {
+                        my $icc_profile = '';
+                        defined $_ and $icc_profile .= $_ foreach @iccChunk;
+                        undef @iccChunk;   # free memory
                         my $tagTablePtr = GetTagTable('Image::ExifTool::ICC_Profile::Main');
                         my %dirInfo = (
@@ -4564 +6667 @@
                         );
                         $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
-                        undef $icc_profile;
+                        undef $iccChunkCount;     # prevent reprocessing
                     }
+                } else {
+                    $self->WarnOnce('Invalid or extraneous ICC_Profile chunk(s)');
                 }
             } elsif ($$segDataPt =~ /^FPXR\0/) {
-                next if $fast and $fast > 1;    # skip processing for very fast
+                next if $fast > 1;      # skip processing for very fast
                 $dumpType = 'FPXR';
                 my $tagTablePtr = GetTagTable('Image::ExifTool::FlashPix::Main');
-                my %dirInfo = (
-                    DataPt   => $segDataPt,
-                    DataPos  => $segPos,
-                    DataLen  => $length,
-                    DirStart => 0,
-                    DirLen   => $length,
-                    Parent   => $markerName,
-                    # set flag if this is the last FPXR segment
-                    LastFPXR => not ($nextMarker==$marker and $$nextSegDataPt=~/^FPXR\0/),
-                );
+                # set flag if this is the last FPXR segment
+                $dirInfo{LastFPXR} = not ($nextMarker==$marker and $$nextSegDataPt=~/^FPXR\0/),
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
             } elsif ($$segDataPt =~ /^MPF\0/) {
                 undef $dumpType;    # (will be dumped here)
-                my %dirInfo = (
-                    Parent => $markerName,
-                    DataPt => $segDataPt,
-                    DataPos => $segPos,
-                    DirStart => 4,
-                    Base => $segPos + 4,
-                    Multi => 1, # the MP Attribute IFD will be MPF1
-                );
+                DirStart(\%dirInfo, 4, 4);
+                $dirInfo{Multi} = 1;    # the MP Attribute IFD will be MPF1
                 if ($htmlDump) {
                     $self->HDump($segPos-4, 4, 'APP2 header', "Data size: $length bytes");
@@ -4600 +6691 @@
                 my $tagTablePtr = GetTagTable('Image::ExifTool::MPF::Main');
                 $self->ProcessTIFF(\%dirInfo, $tagTablePtr);
-            } elsif ($$segDataPt =~ /^\xff\xd8\xff\xdb/) {
-                $preview = $$segDataPt;
-                $dumpType = 'Samsung Preview';
+            } elsif ($$segDataPt =~ /^(|QVGA\0|BGTH)\xff\xd8\xff[\xdb\xe0\xe1]/) {
+                # Samsung/GE/GoPro="", BenQ DC C1220/Pentacon/Polaroid="QVGA\0",
+                # Digilife DDC-690/Rollei="BGTH"
+                $dumpType = 'Preview Image';
+                $preview = substr($$segDataPt, length($1));
             } elsif ($preview) {
+                $dumpType = 'Preview Image';
                 $preview .= $$segDataPt;
-                $dumpType = 'Samsung Preview';
             }
             if ($preview and $nextMarker ne $marker) {
@@ -4614 +6707 @@
             if ($$segDataPt =~ /^(Meta|META|Exif)\0\0/) {
                 undef $dumpType;    # (will be dumped here)
-                my %dirInfo = (
-                    Parent => $markerName,
-                    DataPt => $segDataPt,
-                    DataPos => $segPos,
-                    DirStart => 6,
-                    Base => $segPos + 6,
-                );
+                DirStart(\%dirInfo, 6, 6);
                 if ($htmlDump) {
                     $self->HDump($segPos-4, 10, 'APP3 Meta header');
@@ -4629 +6716 @@
             } elsif ($$segDataPt =~ /^Stim\0/) {
                 undef $dumpType;    # (will be dumped here)
-                my %dirInfo = (
-                    Parent => $markerName,
-                    DataPt => $segDataPt,
-                    DataPos => $segPos,
-                    DirStart => 6,
-                    Base => $segPos + 6,
-                );
+                DirStart(\%dirInfo, 6, 6);
                 if ($htmlDump) {
                     $self->HDump($segPos-4, 4, 'APP3 header', "Data size: $length bytes");
@@ -4644 +6725 @@
                 my $tagTablePtr = GetTagTable('Image::ExifTool::Stim::Main');
                 $self->ProcessTIFF(\%dirInfo, $tagTablePtr);
+            } elsif ($$self{Make} eq 'DJI') {
+                $dumpType = 'DJI ThermalData';
+                # add this data to the combined data if it exists
+                my $dataPt = $segDataPt;
+                if (defined $combinedSegData) {
+                    $combinedSegData .= $$segDataPt;
+                    $dataPt = \$combinedSegData;
+                }
+                if ($nextMarker == $marker) {
+                    $combinedSegData = $$segDataPt unless defined $combinedSegData;
+                } else {
+                    # process DJI FLIR thermal data
+                    my $tagTablePtr = GetTagTable('Image::ExifTool::JPEG::Main');
+                    $self->HandleTag($tagTablePtr, 'APP3', $$dataPt);
+                    undef $combinedSegData;
+                }
             } elsif ($$segDataPt =~ /^\xff\xd8\xff\xdb/) {
+                $dumpType = 'PreviewImage'; # (Samsung, HP, BenQ)
                 $preview = $$segDataPt;
-                $dumpType = 'Samsung/HP Preview';
-            }
-            # Samsung continues the preview in APP4
-            if ($preview and $nextMarker ne 0xe4) {
+            }
+            if ($preview and $nextMarker ne 0xe4) { # this preview continues in APP4
                 $self->FoundTag('PreviewImage', $preview);
                 undef $preview;
@@ -4668 +6764 @@
                         DataPt => \$scalado,
                     );
-                    my $tagTablePtr = GetTagTable('Image::ExifTool::JPEG::Scalado');
+                    my $tagTablePtr = GetTagTable('Image::ExifTool::Scalado::Main');
                     $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
                     undef $scalado;
                 }
             } elsif ($$segDataPt =~ /^FPXR\0/) {
-                next if $fast and $fast > 1;    # skip processing for very fast
+                next if $fast > 1;      # skip processing for very fast
                 $dumpType = 'FPXR';
                 my $tagTablePtr = GetTagTable('Image::ExifTool::FlashPix::Main');
-                my %dirInfo = (
-                    DataPt   => $segDataPt,
-                    DataPos  => $segPos,
-                    DataLen  => $length,
-                    DirStart => 0,
-                    DirLen   => $length,
-                    Parent   => $markerName,
-                    # set flag if this is the last FPXR segment
-                    LastFPXR => not ($nextMarker==$marker and $$nextSegDataPt=~/^FPXR\0/),
-                );
+                # set flag if this is the last FPXR segment
+                $dirInfo{LastFPXR} = not ($nextMarker==$marker and $$nextSegDataPt=~/^FPXR\0/),
+                $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
+            } elsif ($$self{Make} eq 'DJI' and $$segDataPt =~ /^\xaa\x55\x12\x06/) {
+                $dumpType = 'DJI ThermalParams';
+                DirStart(\%dirInfo, 0, 0);
+                my $tagTablePtr = GetTagTable('Image::ExifTool::DJI::ThermalParams');
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
             } elsif ($preview) {
                 # continued Samsung S1060 preview from APP3
+                $dumpType = 'PreviewImage';
                 $preview .= $$segDataPt;
-                # (not sure if next part would be APP5 or APP4 again, but assume APP4)
-                if ($nextMarker ne $marker) {
-                    $self->FoundTag('PreviewImage', $preview);
-                    undef $preview;
-                }
+            }
+            # (also seen "QTI Debug Metadata\0" segment in some newer Samsung images)
+            # BenQ DC E1050 continues preview in APP5
+            if ($preview and $nextMarker ne 0xe5) {
+                $self->FoundTag('PreviewImage', $preview);
+                undef $preview;
             }
         } elsif ($marker == 0xe5) {         # APP5 (Ricoh "RMETA")
             if ($$segDataPt =~ /^RMETA\0/) {
+                # (NOTE: apparently these may span multiple segments, but I haven't seen
+                # a sample like this, so multi-segment support hasn't yet been implemented)
                 $dumpType = 'Ricoh RMETA';
-                my %dirInfo = (
-                    Parent => $markerName,
-                    DataPt => $segDataPt,
-                    DataPos => $segPos,
-                    DirStart => 6,
-                    Base => $segPos + 6,
-                );
+                DirStart(\%dirInfo, 6, 6);
                 my $tagTablePtr = GetTagTable('Image::ExifTool::Ricoh::RMETA');
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
+            } elsif ($$segDataPt =~ /^ssuniqueid\0/) {
+                my $tagTablePtr = GetTagTable('Image::ExifTool::Samsung::APP5');
+                $self->HandleTag($tagTablePtr, 'ssuniqueid', substr($$segDataPt, 11));
+            } elsif ($$self{Make} eq 'DJI') {
+                $dumpType = 'DJI ThermalCal';
+                my $tagTablePtr = GetTagTable('Image::ExifTool::JPEG::Main');
+                $self->HandleTag($tagTablePtr, 'APP5', $$segDataPt);
+            } elsif ($preview) {
+                $dumpType = 'PreviewImage';
+                $preview .= $$segDataPt;
+                $self->FoundTag('PreviewImage', $preview);
+                undef $preview;
             }
         } elsif ($marker == 0xe6) {         # APP6 (Toshiba EPPIM, NITF, HP_TDHD)
             if ($$segDataPt =~ /^EPPIM\0/) {
                 undef $dumpType;    # (will be dumped here)
-                my %dirInfo = (
-                    Parent => $markerName,
-                    DataPt => $segDataPt,
-                    DataPos => $segPos,
-                    DirStart => 6,
-                    Base => $segPos + 6,
-                );
+                DirStart(\%dirInfo, 6, 6);
                 if ($htmlDump) {
                     $self->HDump($segPos-4, 10, 'APP6 EPPIM header');
@@ -4729 +6826 @@
                 SetByteOrder('MM');
                 my $tagTablePtr = GetTagTable('Image::ExifTool::JPEG::NITF');
-                my %dirInfo = (
-                    DataPt   => $segDataPt,
-                    DataPos  => $segPos,
-                    DirStart => 5,
-                    DirLen   => $length - 5,
-                );
+                DirStart(\%dirInfo, 5);
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
             } elsif ($$segDataPt =~ /^TDHD\x01\0\0\0/ and $length > 12) {
@@ -4740 +6832 @@
                 $dumpType = 'TDHD';
                 my $tagTablePtr = GetTagTable('Image::ExifTool::HP::TDHD');
-                my %dirInfo = (
-                    DataPt   => $segDataPt,
-                    DataPos  => $segPos,
-                    DirStart => 12, # (ignore first TDHD element because size includes 12-byte tag header)
-                    DirLen   => $length - 12,
-                );
+                # (ignore first TDHD element because size includes 12-byte tag header)
+                DirStart(\%dirInfo, 12);
+                $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
+            } elsif ($$segDataPt =~ /^GoPro\0/) {
+                # GoPro segment
+                $dumpType = 'GoPro';
+                my $tagTablePtr = GetTagTable('Image::ExifTool::GoPro::GPMF');
+                DirStart(\%dirInfo, 6);
+                $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
+            } elsif ($$segDataPt =~ /^DTAT\0\0.\{/s) {
+                $dumpType = 'DJI_DTAT';
+                my $tagTablePtr = GetTagTable('Image::ExifTool::JPEG::Main');
+                $self->HandleTag($tagTablePtr, 'APP6', $$segDataPt);
+            }
+        } elsif ($marker == 0xe7) {         # APP7 (Pentax, Huawei, Qualcomm)
+            if ($$segDataPt =~ /^PENTAX \0(II|MM)/) {
+                # found in K-3 images (is this multi-segment??)
+                SetByteOrder($1);
+                undef $dumpType; # (dump this ourself)
+                my $hdrLen = 10;
+                my $tagTablePtr = GetTagTable('Image::ExifTool::Pentax::Main');
+                DirStart(\%dirInfo, $hdrLen, 0);
+                $dirInfo{DirName} = 'Pentax APP7';
+                if ($htmlDump) {
+                    $self->HDump($segPos-4, 4, 'APP7 header', "Data size: $length bytes");
+                    $self->HDump($segPos, $hdrLen, 'Pentax header', 'APP7 data type: Pentax');
+                    $dumpEnd = $segPos + $length;
+                }
+                $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
+            } elsif ($$segDataPt =~ /^HUAWEI\0\0(II|MM)/) {
+                SetByteOrder($1);
+                undef $dumpType; # (dump this ourself)
+                my $hdrLen = 16;
+                my $tagTablePtr = GetTagTable('Image::ExifTool::Unknown::Main');
+                DirStart(\%dirInfo, $hdrLen, 8);
+                $dirInfo{DirName} = 'Huawei APP7';
+                if ($htmlDump) {
+                    $self->HDump($segPos-4, 4, 'APP7 header', "Data size: $length bytes");
+                    $self->HDump($segPos, $hdrLen, 'Huawei header', 'APP7 data type: Huawei');
+                    $dumpEnd = $segPos + $length;
+                }
+                $$self{SET_GROUP0} = 'APP7';
+                $$self{SET_GROUP1} = 'Huawei';
+                $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
+                delete $$self{SET_GROUP0};
+                delete $$self{SET_GROUP1};
+            } elsif ($$segDataPt =~ /^\x1aQualcomm Camera Attributes/) {
+                # found in HP iPAQ_VoiceMessenger
+                $dumpType = 'Qualcomm';
+                my $tagTablePtr = GetTagTable('Image::ExifTool::Qualcomm::Main');
+                DirStart(\%dirInfo, 27);
+                $dirInfo{DirName} = 'Qualcomm';
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
             }
@@ -4752 +6890 @@
             if ($$segDataPt =~ /^SPIFF\0/ and $length == 32) {
                 $dumpType = 'SPIFF';
-                my %dirInfo = (
-                    DataPt => $segDataPt,
-                    DataPos  => $segPos,
-                    DirStart => 6,
-                    DirLen => $length - 6,
-                );
+                DirStart(\%dirInfo, 6);
                 my $tagTablePtr = GetTagTable('Image::ExifTool::JPEG::SPIFF');
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
+            }
+        } elsif ($marker == 0xe9) {         # APP9 (Media Jukebox)
+            if ($$segDataPt =~ /^Media Jukebox\0/ and $length > 22) {
+                $dumpType = 'MediaJukebox';
+                # (start parsing after the "<MJMD>")
+                DirStart(\%dirInfo, 22);
+                $dirInfo{DirName} = 'MediaJukebox';
+                require Image::ExifTool::XMP;
+                my $tagTablePtr = GetTagTable('Image::ExifTool::JPEG::MediaJukebox');
+                $self->ProcessDirectory(\%dirInfo, $tagTablePtr, \&Image::ExifTool::XMP::ProcessXMP);
             }
         } elsif ($marker == 0xea) {         # APP10 (PhotoStudio Unicode comments)
@@ -4766 +6909 @@
                 my $comment = $self->Decode(substr($$segDataPt,8), 'UCS2', 'MM');
                 $self->FoundTag('Comment', $comment);
+            } elsif ($$segDataPt =~ /^AROT\0/ and $length > 10) {
+                # iPhone "AROT" segment containing integrated intensity per 16 scan lines
+                # (with number of elements N = ImageHeight / 16 - 1, ref PH/NealKrawetz)
+                $xtra = 'segment (N=' . unpack('x6N', $$segDataPt) . ')';
+            }
+        } elsif ($marker == 0xeb) {         # APP11 (JPEG-HDR)
+            if ($$segDataPt =~ /^HDR_RI /) {
+                $dumpType = 'JPEG-HDR';
+                my $dataPt = $segDataPt;
+                if (defined $combinedSegData) {
+                    if ($$segDataPt =~ /~\0/g) {
+                        $combinedSegData .= substr($$segDataPt,pos($$segDataPt));
+                    } else {
+                        $self->Warn('Invalid format for JPEG-HDR extended segment');
+                    }
+                    $dataPt = \$combinedSegData;
+                }
+                if ($nextMarker == $marker and $$nextSegDataPt =~ /^HDR_RI /) {
+                    $combinedSegData = $$segDataPt unless defined $combinedSegData;
+                } else {
+                    my $tagTablePtr = GetTagTable('Image::ExifTool::JPEG::HDR');
+                    my %dirInfo = ( DataPt => $dataPt );
+                    $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
+                    undef $combinedSegData;
+                }
             }
         } elsif ($marker == 0xec) {         # APP12 (Ducky, Picture Info)
             if ($$segDataPt =~ /^Ducky/) {
                 $dumpType = 'Ducky';
-                my %dirInfo = (
-                    DataPt => $segDataPt,
-                    DataPos => $segPos,
-                    DirStart => 5,
-                    DirLen => $length - 5,
-                );
+                DirStart(\%dirInfo, 5);
                 my $tagTablePtr = GetTagTable('Image::ExifTool::APP12::Ducky');
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
             } else {
-                my %dirInfo = ( DataPt => $segDataPt );
                 my $tagTablePtr = GetTagTable('Image::ExifTool::APP12::PictureInfo');
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr) and $dumpType = 'Picture Info';
@@ -4799 +6961 @@
                     # (will handle the Photoshop data the next time around)
                 } else {
-                    my $hdrlen = $isOld ? 27 : 14;
+                    my $hdrLen = $isOld ? 27 : 14;
                     # process APP13 Photoshop record
                     my $tagTablePtr = GetTagTable('Image::ExifTool::Photoshop::Main');
@@ -4806 +6968 @@
                         DataPos  => $segPos,
                         DataLen  => length $$dataPt,
-                        DirStart => $hdrlen,    # directory starts after identifier
-                        DirLen   => length($$dataPt) - $hdrlen,
+                        DirStart => $hdrLen,    # directory starts after identifier
+                        DirLen   => length($$dataPt) - $hdrLen,
                         Parent   => $markerName,
                     );
@@ -4817 +6979 @@
                 SetByteOrder('MM');
                 my $tagTablePtr = GetTagTable('Image::ExifTool::JPEG::AdobeCM');
-                my %dirInfo = (
-                    DataPt   => $segDataPt,
-                    DataPos  => $segPos,
-                    DirStart => 8,
-                    DirLen   => $length - 8,
-                );
+                DirStart(\%dirInfo, 8);
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
             }
         } elsif ($marker == 0xee) {         # APP14 (Adobe)
             if ($$segDataPt =~ /^Adobe/) {
+                # extract as a block if requested, or if copying tags from file
+                if ($$self{REQ_TAG_LOOKUP}{adobe} or
+                    # (not extracted normally, so check TAGS_FROM_FILE)
+                    ($$self{TAGS_FROM_FILE} and not $$self{EXCL_TAG_LOOKUP}{adobe}))
+                {
+                    $self->FoundTag('Adobe', $$segDataPt);
+                }
                 $dumpType = 'Adobe';
                 SetByteOrder('MM');
                 my $tagTablePtr = GetTagTable('Image::ExifTool::JPEG::Adobe');
-                my %dirInfo = (
-                    DataPt   => $segDataPt,
-                    DataPos  => $segPos,
-                    DirStart => 5,
-                    DirLen   => $length - 5,
-                );
+                DirStart(\%dirInfo, 5);
                 $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
             }
@@ -4848 +7007 @@
             $$segDataPt =~ s/\0+$//;    # some dumb softwares add null terminators
             $self->FoundTag('Comment', $$segDataPt);
+        } elsif ($marker == 0x64) {         # CME (J2C comment and extension)
+            $dumpType = 'Comment';
+            if ($length > 2) {
+                my $reg = unpack('n', $$segDataPt); # get registration value
+                my $val = substr($$segDataPt, 2);
+                $val = $self->Decode($val, 'Latin') if $reg == 1;
+                # (actually an extension for $reg==65535, but store as binary comment)
+                $self->FoundTag('Comment', ($reg==0 or $reg==65535) ? \$val : $val);
+            }
+        } elsif ($marker == 0x51) {         # SIZ (J2C)
+            my ($w, $h) = unpack('x2N2', $$segDataPt);
+            $self->FoundTag('ImageWidth', $w);
+            $self->FoundTag('ImageHeight', $h);
         } elsif (($marker & 0xf0) != 0xe0) {
-            undef $dumpType;    # only dump unknown APP segments
+            $dumpType = "$markerName segment";
+            $desc = "[JPEG $markerName]";   # (other known JPEG segments)
         }
         if (defined $dumpType) {
-            if (not $dumpType and $self->{OPTIONS}{Unknown}) {
-                $self->Warn("Unknown $markerName segment", 1);
+            if (not $dumpType and ($$options{Unknown} or $$options{Validate})) {
+                my $str = ($$segDataPt =~ /^([\x20-\x7e]{1,20})\0/) ? " '${1}'" : '';
+                $xtra = 'segment' unless $xtra;
+                $self->Warn("Unknown $markerName$str $xtra", 1);
             }
             if ($htmlDump) {
-                my $desc = $markerName . ($dumpType ? " $dumpType" : '') . ' segment';
-                $self->HDump($segPos-4, $length+4, $desc, undef, 0x08);
+                $desc or $desc = $markerName . ($dumpType ? " $dumpType" : '') . ' segment';
+                $self->HDump($segPos-4, $length+4, $desc, $tip, 0x08);
                 $dumpEnd = $segPos + $length;
             }
@@ -4863 +7038 @@
         undef $$segDataPt;
     }
+    # process extended XMP now if it existed
+    if (%extendedXMP) {
+        my $guid;
+        # GUID indicated by the last main XMP segment
+        my $goodGuid = $$self{VALUE}{HasExtendedXMP} || '';
+        # GUID of the extended XMP that we will process ('2' for all)
+        my $readGuid = $$options{ExtendedXMP} || 0;
+        $readGuid = $goodGuid if $readGuid eq '1';
+        foreach $guid (sort keys %extendedXMP) {
+            next unless length $guid == 32;     # ignore other (internal) keys
+            my $extXMP = $extendedXMP{$guid};
+            my ($off, @offsets, $warn);
+            # make sure we have all chunks, and create a list of sorted offsets
+            for ($off=0; $off<$$extXMP{Size}; ) {
+                last unless defined $$extXMP{$off};
+                push @offsets, $off;
+                $off += length $$extXMP{$off};
+            }
+            unless ($off == $$extXMP{Size}) {
+                $self->Warn("Incomplete extended XMP (GUID $guid)");
+                next;
+            }
+            if ($guid eq $readGuid or $readGuid eq '2') {
+                $warn = 'Reading non-' if $guid ne $goodGuid;
+                my $buff = '';
+                # assemble XMP all together
+                $buff .= $$extXMP{$_} foreach @offsets;
+                my $tagTablePtr = GetTagTable('Image::ExifTool::XMP::Main');
+                my %dirInfo = (
+                    DataPt      => \$buff,
+                    Parent      => 'APP1',
+                    IsExtended  => 1,
+                );
+                $$path[$pn] = 'APP1';
+                $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
+                pop @$path;
+            } else {
+                $warn = 'Ignored ';
+                $warn .= 'non-' if $guid ne $goodGuid;
+            }
+            $self->Warn("${warn}standard extended XMP (GUID $guid)") if $warn;
+            delete $extendedXMP{$guid};
+        }
+    }
     # calculate JPEGDigest if requested
-    if (@dqt and $subSampling) {
+    if (@dqt) {
         require Image::ExifTool::JPEGDigest;
         Image::ExifTool::JPEGDigest::Calculate($self, \@dqt, $subSampling);
     }
+    # issue necessary warnings
+    $self->Warn('Incomplete ICC_Profile record', 1) if defined $iccChunkCount;
+    $self->Warn('Incomplete FLIR record', 1) if defined $flirCount;
     $self->Warn('Error reading PreviewImage', 1) if $$self{PreviewError};
-    $self->Warn('Invalid extended XMP') if %extendedXMP;
     $success or $self->Warn('JPEG format error');
     pop @$path if @$path > $pn;
     return 1;
+}
+
+#------------------------------------------------------------------------------
+# Extract metadata from an Exiv2 EXV file
+# Inputs: 0) ExifTool object reference, 1) dirInfo ref with RAF set
+# Returns: 1 on success, 0 if this wasn't a valid JPEG file
+sub ProcessEXV($$)
+{
+    my ($self, $dirInfo) = @_;
+    return $self->ProcessJPEG($dirInfo);
 }
 
@@ -4914 +7145 @@
     my $base = $$dirInfo{Base} || 0;
     my $outfile = $$dirInfo{OutFile};
-    my ($err, $canonSig, $otherSig);
+    my ($err, $sig, $canonSig, $otherSig);
 
     # attempt to read TIFF header
-    $self->{EXIF_DATA} = '';
+    $$self{EXIF_DATA} = '';
     if ($raf) {
         if ($outfile) {
@@ -4930 +7161 @@
         # extract full EXIF block (for block copy) from EXIF file
         my $amount = $fileType eq 'EXIF' ? 65536 * 8 : 8;
-        my $n = $raf->Read($self->{EXIF_DATA}, $amount);
+        my $n = $raf->Read($$self{EXIF_DATA}, $amount);
         if ($n < 8) {
             return 0 if $n or not $outfile or $fileType ne 'EXIF';
             # create EXIF file from scratch
-            delete $self->{EXIF_DATA};
+            delete $$self{EXIF_DATA};
             undef $raf;
         }
@@ -4940 +7171 @@
             $raf->Seek(8, 0);
             if ($n == $amount) {
-                $self->{EXIF_DATA} = substr($self->{EXIF_DATA}, 0, 8);
+                $$self{EXIF_DATA} = substr($$self{EXIF_DATA}, 0, 8);
                 $self->Warn('EXIF too large to extract as a block'); #(shouldn't happen)
             }
@@ -4948 +7179 @@
         my $dirStart = $$dirInfo{DirStart} || 0;
         my $dirLen = $$dirInfo{DirLen} || (length($$dataPt) - $dirStart);
-        $self->{EXIF_DATA} = substr($$dataPt, $dirStart, $dirLen);
-        $self->VerboseDir('TIFF') if $self->{OPTIONS}{Verbose} and length($$self{INDENT}) > 2;
+        $$self{EXIF_DATA} = substr($$dataPt, $dirStart, $dirLen);
+        $self->VerboseDir('TIFF') if $$self{OPTIONS}{Verbose} and length($$self{INDENT}) > 2;
     } elsif ($outfile) {
-        delete $self->{EXIF_DATA};  # create from scratch
+        delete $$self{EXIF_DATA};  # create from scratch
     } else {
-        $self->{EXIF_DATA} = '';
-    }
-    unless (defined $self->{EXIF_DATA}) {
+        $$self{EXIF_DATA} = '';
+    }
+    unless (defined $$self{EXIF_DATA}) {
+        # set default byte order for creating new GPS in CR3 images
+        my $defaultByteOrder;
+        if ($$dirInfo{DirName} and $$dirInfo{DirName} eq 'GPS') {
+            $defaultByteOrder = $$self{SaveExifByteOrder};
+        }
         # create TIFF information from scratch
-        if ($self->SetPreferredByteOrder() eq 'MM') {
-            $self->{EXIF_DATA} = "MM\0\x2a\0\0\0\x08";
+        if ($self->SetPreferredByteOrder($defaultByteOrder) eq 'MM') {
+            $$self{EXIF_DATA} = "MM\0\x2a\0\0\0\x08";
         } else {
-            $self->{EXIF_DATA} = "II\x2a\0\x08\0\0\0";
-        }
-    }
-    $$self{FIRST_EXIF_POS} = $base + $$self{BASE} unless defined $$self{FIRST_EXIF_POS};
+            $$self{EXIF_DATA} = "II\x2a\0\x08\0\0\0";
+        }
+    }
     $$self{EXIF_POS} = $base + $$self{BASE};
-    $dataPt = \$self->{EXIF_DATA};
+    $$self{FIRST_EXIF_POS} = $$self{EXIF_POS} unless defined $$self{FIRST_EXIF_POS};
+    $dataPt = \$$self{EXIF_DATA};
 
     # set byte ordering
@@ -4977 +7213 @@
   # (TIFF=0x2a, RW2/RWL=0x55, HDP=0xbc, BTF=0x2b, ORF=0x4f52/0x5352/0x????)
   #  return 0 unless $identifier == 0x2a;
+    $self->Warn('Invalid magic number in EXIF TIFF header') if $fileType eq 'APP1' and $identifier != 0x2a;
 
     # get offset to IFD0
+    return 0 if length $$dataPt < 8;
     my $offset = Get32u($dataPt, 4);
     $offset >= 8 or return 0;
 
     if ($raf) {
-        # Canon CR2 images usually have an offset of 16, but it may be
-        # greater if edited by PhotoMechanic, so check the 4-byte signature
+        # check for canon or EXIF signature
+        # (Canon CR2 images should have an offset of 16, but it may be
+        #  greater if edited by PhotoMechanic)
         if ($identifier == 0x2a and $offset >= 16) {
-            $raf->Read($canonSig, 8) == 8 or return 0;
-            $$dataPt .= $canonSig;
-            if ($canonSig =~ /^(CR\x02\0|\xba\xb0\xac\xbb)/) {
-                $fileType = $canonSig =~ /^CR/ ? 'CR2' : 'Canon 1D RAW';
-                $self->HDump($base+8, 8, "[$fileType header]") if $self->{HTML_DUMP};
-            } else {
-                undef $canonSig;
+            $raf->Read($sig, 8) == 8 or return 0;
+            $$dataPt .= $sig;
+            if ($sig =~ /^(CR\x02\0|\xba\xb0\xac\xbb|ExifMeta)/) {
+                if ($sig eq 'ExifMeta') {
+                    $self->SetFileType($fileType = 'EXIF');
+                    $otherSig = $sig;
+                } else {
+                    $fileType = $sig =~ /^CR/ ? 'CR2' : 'Canon 1D RAW';
+                    $canonSig = $sig;
+                }
+                $self->HDump($base+8, 8, "[$fileType header]") if $$self{HTML_DUMP};
             }
         } elsif ($identifier == 0x55 and $fileType =~ /^(RAW|RW2|RWL|TIFF)$/) {
@@ -5008 +7251 @@
             }
             $tagTablePtr = GetTagTable('Image::ExifTool::PanasonicRaw::Main');
-        } elsif ($identifier == 0x2b and $fileType eq 'TIFF') {
-            # this looks like a BigTIFF image
-            $raf->Seek(0);
-            require Image::ExifTool::BigTIFF;
-            return 1 if Image::ExifTool::BigTIFF::ProcessBTF($self, $dirInfo);
-        } elsif (Get8u($dataPt, 2) == 0xbc and $byteOrder eq 'II' and $fileType eq 'TIFF') {
-            $fileType = 'HDP';  # Windows HD Photo file
-            # check version number
-            my $ver = Get8u($dataPt, 3);
-            if ($ver > 1) {
-                $self->Error("Windows HD Photo version $ver files not yet supported");
-                return 1;
-            }
-        } elsif ($identifier == 0x4352 and $fileType eq 'TIFF') {
-            $fileType = 'DCP';
+        } elsif ($fileType eq 'TIFF') {
+            if ($identifier == 0x2b) {
+                # this looks like a BigTIFF image
+                $raf->Seek(0);
+                require Image::ExifTool::BigTIFF;
+                return 1 if Image::ExifTool::BigTIFF::ProcessBTF($self, $dirInfo);
+            } elsif ($identifier == 0x4f52 or $identifier == 0x5352) {
+                # Olympus ORF image (set FileType now because base type is 'ORF')
+                $self->SetFileType($fileType = 'ORF');
+            } elsif ($identifier == 0x4352) {
+                $fileType = 'DCP';
+            } elsif ($byteOrder eq 'II' and ($identifier & 0xff) == 0xbc) {
+                $fileType = 'HDP';  # Windows HD Photo file
+                # check version number
+                my $ver = Get8u($dataPt, 3);
+                if ($ver > 1) {
+                    $self->Error("Windows HD Photo version $ver files not yet supported");
+                    return 1;
+                }
+            }
         }
         # we have a valid TIFF (or whatever) file
-        if ($fileType and not $self->{VALUE}{FileType}) {
+        if ($fileType and not $$self{VALUE}{FileType}) {
             my $lookup = $fileTypeLookup{$fileType};
             $lookup = $fileTypeLookup{$lookup} unless ref $lookup or not $lookup;
             # use file extension to pre-determine type if extension is TIFF-based or type is RAW
-            my $t = (($lookup and $$lookup[0] eq 'TIFF') or $fileType =~ /RAW/) ? $fileType : undef;
+            my $baseType = $lookup ? (ref $$lookup[0] ? $$lookup[0][0] : $$lookup[0]) : '';
+            my $t = ($baseType eq 'TIFF' or $fileType =~ /RAW/) ? $fileType : undef;
             $self->SetFileType($t);
         }
-    }
-    my $ifdName = 'IFD0';
+        # don't process file if FastScan == 3
+        return 1 if not $outfile and $$self{OPTIONS}{FastScan} and $$self{OPTIONS}{FastScan} == 3;
+    }
+    # (accommodate CR3 images which have a TIFF directory with ExifIFD at the top level)
+    my $ifdName = ($$dirInfo{DirName} and $$dirInfo{DirName} =~ /^(ExifIFD|GPS)$/) ? $1 : 'IFD0';
     if (not $tagTablePtr or $$tagTablePtr{GROUPS}{0} eq 'EXIF') {
-        $self->FoundTag('ExifByteOrder', $byteOrder);
+        $self->FoundTag('ExifByteOrder', $byteOrder) unless $outfile;
+    } elsif ($$tagTablePtr{GROUPS}{0} eq 'MakerNotes') { # (for writing CR3 maker notes)
+        $ifdName = $$tagTablePtr{GROUPS}{0};
     } else {
         $ifdName = $$tagTablePtr{GROUPS}{1};
     }
-    if ($self->{HTML_DUMP}) {
+    if ($$self{HTML_DUMP}) {
         my $tip = sprintf("Byte order: %s endian\nIdentifier: 0x%.4x\n$ifdName offset: 0x%.4x",
                           ($byteOrder eq 'II') ? 'Little' : 'Big', $identifier, $offset);
@@ -5045 +7299 @@
     }
     # remember where we found the TIFF data (APP1, APP3, TIFF, NEF, etc...)
-    $self->{TIFF_TYPE} = $fileType;
+    $$self{TIFF_TYPE} = $fileType;
 
     # get reference to the main EXIF table
@@ -5070 +7324 @@
         $self->ProcessDirectory(\%dirInfo, $tagTablePtr);
         # process GeoTiff information if available
-        if ($self->{VALUE}{GeoTiffDirectory}) {
+        if ($$self{VALUE}{GeoTiffDirectory}) {
             require Image::ExifTool::GeoTiff;
             Image::ExifTool::GeoTiff::ProcessGeoTiff($self);
@@ -5081 +7335 @@
                 $self->ProcessTrailers($trailInfo);
             }
-            # dump any other known trailer (ie. A100 RAW Data)
+            # dump any other known trailer (eg. A100 RAW Data)
             if ($$self{HTML_DUMP} and $$self{KnownTrailer}) {
                 my $known = $$self{KnownTrailer};
@@ -5091 +7345 @@
         }
         # update FileType if necessary now that we know more about the file
-        if ($$self{DNGVersion} and $self->{VALUE}{FileType} ne 'DNG') {
+        if ($$self{DNGVersion} and $$self{VALUE}{FileType} !~ /^(DNG|GPR)$/) {
             # override whatever FileType we set since we now know it is DNG
-            $self->OverrideFileType('DNG');
+            $self->OverrideFileType($$self{TIFF_TYPE} = 'DNG');
         }
         return 1;
@@ -5101 +7355 @@
 #
     if ($$dirInfo{NoTiffEnd}) {
-        delete $self->{TIFF_END};
+        delete $$self{TIFF_END};
     } else {
         # initialize TIFF_END so it will be updated by WriteExif()
-        $self->{TIFF_END} = 0;
+        $$self{TIFF_END} = 0;
     }
     if ($canonSig) {
@@ -5114 +7368 @@
     } else {
         # write TIFF header (8 bytes [plus optional signature] followed by IFD)
-        $otherSig = '' unless defined $otherSig;
+        if ($fileType eq 'EXIF') {
+            $otherSig = 'ExifMeta'; # force this signature for all EXIF files
+        } elsif (not defined $otherSig) {
+            $otherSig = '';
+        }
         my $offset = 8 + length($otherSig);
         # construct tiff header
@@ -5138 +7396 @@
                 # write any required ARW trailer and patch other ARW quirks
                 require Image::ExifTool::Sony;
-                my $errStr = Image::ExifTool::Sony::FinishARW($self, $dirInfo, \$newData, 
+                my $errStr = Image::ExifTool::Sony::FinishARW($self, $dirInfo, \$newData,
                                                               $dirInfo{ImageData});
                 $errStr and $self->Error($errStr);
@@ -5154 +7412 @@
     }
     # make local copy of TIFF_END now (it may be reset when processing trailers)
-    my $tiffEnd = $self->{TIFF_END};
-    delete $self->{TIFF_END};
+    my $tiffEnd = $$self{TIFF_END};
+    delete $$self{TIFF_END};
 
     # rewrite trailers if they exist
@@ -5189 +7447 @@
         if ($extra > 0) {
             my $known = $$self{KnownTrailer};
-            if ($self->{DEL_GROUP}{Trailer} and not $known) {
+            if ($$self{DEL_GROUP}{Trailer} and not $known) {
                 $self->VPrint(0, "  Deleting unknown trailer ($extra bytes)\n");
-                ++$self->{CHANGED};
+                ++$$self{CHANGED};
             } elsif ($known) {
                 $self->VPrint(0, "  Copying $$known{Name} ($extra bytes)\n");
@@ -5228 +7486 @@
     if ($$self{DNGVersion}) {
         my $ver = $$self{DNGVersion};
-        # currently support up to DNG version 1.2
-        unless ($ver =~ /^(\d+) (\d+)/ and "$1.$2" <= 1.3) {
+        # currently support up to DNG version 1.5
+        unless ($ver =~ /^(\d+) (\d+)/ and "$1.$2" <= 1.5) {
             $ver =~ tr/ /./;
-            $self->Error("DNG Version $ver not yet supported", 1);
+            $self->Error("DNG Version $ver not yet tested", 1);
         }
     }
@@ -5276 +7534 @@
                 my $module = $1;
                 if (eval "require $module") {
-                    # load additional XMP modules if required
-                    if (not %$tableName and $module eq 'Image::ExifTool::XMP') {
-                        require 'Image/ExifTool/XMP2.pl';
+                    # load additional modules if required
+                    if (not %$tableName) {
+                        if ($module eq 'Image::ExifTool::XMP') {
+                            require 'Image/ExifTool/XMP2.pl';
+                        } elsif ($tableName eq 'Image::ExifTool::QuickTime::Stream') {
+                            require 'Image/ExifTool/QuickTimeStream.pl';
+                        }
                     }
                 } else {
@@ -5292 +7554 @@
         $table = \%$tableName;
         use strict 'refs';
+        &{$$table{INIT_TABLE}}($table) if $$table{INIT_TABLE};
         $$table{TABLE_NAME} = $tableName;   # set table name
         ($$table{SHORT_NAME} = $tableName) =~ s/^Image::ExifTool:://;
@@ -5317 +7580 @@
         }
         # generate a tag prefix for unknown tags if necessary
-        unless ($$table{TAG_PREFIX}) {
+        unless (defined $$table{TAG_PREFIX}) {
             my $tagPrefix;
             if ($tableName =~ /Image::.*?::(.*)::Main/ || $tableName =~ /Image::.*?::(.*)/) {
@@ -5328 +7591 @@
         # set up the new table
         SetupTagTable($table);
-        # add any user-defined tags
-        if (%UserDefined and $UserDefined{$tableName}) {
+        # add any user-defined tags (except Composite tags, which are handled specially)
+        if (%UserDefined and $UserDefined{$tableName} and $table ne \%Image::ExifTool::Composite) {
             my $tagID;
             foreach $tagID (TagTableKeys($UserDefined{$tableName})) {
-                my $tagInfo = $UserDefined{$tableName}{$tagID};
-                if (ref $tagInfo eq 'HASH') {
-                    $$tagInfo{Name} or $$tagInfo{Name} = ucfirst($tagID);
-                } else {
-                    $tagInfo = { Name => $tagInfo };
-                }
-                if ($$table{WRITABLE} and not defined $$tagInfo{Writable} and
-                    not $$tagInfo{SubDirectory})
-                {
-                    $$tagInfo{Writable} = $$table{WRITABLE};
-                }
+                next if $specialTags{$tagID};
                 delete $$table{$tagID}; # replace any existing entry
-                AddTagToTable($table, $tagID, $tagInfo);
+                AddTagToTable($table, $tagID, $UserDefined{$tableName}{$tagID}, 1);
             }
         }
@@ -5351 +7604 @@
         # insert newly loaded table into list
         $allTables{$tableName} = $table;
+    }
+    # must check each time to add UserDefined Composite tags because the Composite table
+    # may be loaded before the UserDefined tags are available
+    if ($table eq \%Image::ExifTool::Composite and not $$table{VARS}{LOADED_USERDEFINED} and
+        %UserDefined and $UserDefined{$tableName})
+    {
+        my $userComp = $UserDefined{$tableName};
+        delete $UserDefined{$tableName};        # (must delete first to avoid infinite recursion)
+        AddCompositeTags($userComp, 1);
+        $UserDefined{$tableName} = $userComp;   # (add back again for adding writable tags later)
+        $$table{VARS}{LOADED_USERDEFINED} = 1;  # set flag to avoid doing this again
     }
     return $table;
@@ -5368 +7632 @@
     $proc or $proc = $$tagTablePtr{PROCESS_PROC} || \&Image::ExifTool::Exif::ProcessExif;
     # set directory name from default group0 name if not done already
-    $$dirInfo{DirName} or $$dirInfo{DirName} = $tagTablePtr->{GROUPS}{0};
+    my $dirName = $$dirInfo{DirName};
+    unless ($dirName) {
+        $dirName = $$tagTablePtr{GROUPS}{0};
+        $dirName = $$tagTablePtr{GROUPS}{1} if $dirName =~ /^APP\d+$/; # (use specific APP name)
+        $$dirInfo{DirName} = $dirName;
+    }
+
     # guard against cyclical recursion into the same directory
-    if (defined $$dirInfo{DirStart} and defined $$dirInfo{DataPos}) {
-        my $addr = $$dirInfo{DirStart} + $$dirInfo{DataPos} + ($$dirInfo{Base}||0);
-        if ($self->{PROCESSED}{$addr}) {
-            $self->Warn("$$dirInfo{DirName} pointer references previous $self->{PROCESSED}{$addr} directory");
-            return 0;
-        }
-        $self->{PROCESSED}{$addr} = $$dirInfo{DirName};
+    if (defined $$dirInfo{DirStart} and defined $$dirInfo{DataPos} and
+        # directories don't overlap if the length is zero
+        ($$dirInfo{DirLen} or not defined $$dirInfo{DirLen}))
+    {
+        my $addr = $$dirInfo{DirStart} + $$dirInfo{DataPos} + ($$dirInfo{Base}||0) + $$self{BASE};
+        if ($$self{PROCESSED}{$addr}) {
+            $self->Warn("$dirName pointer references previous $$self{PROCESSED}{$addr} directory");
+            # patch for bug in Windows phone 7.5 O/S that writes incorrect InteropIFD pointer
+            return 0 unless $dirName eq 'GPS' and $$self{PROCESSED}{$addr} eq 'InteropIFD';
+        }
+        $$self{PROCESSED}{$addr} = $dirName;
     }
     my $oldOrder = GetByteOrder();
-    my $oldIndent = $self->{INDENT};
-    my $oldDir = $self->{DIR_NAME};
-    $self->{LIST_TAGS} = { };  # don't build lists across different directories
-    $self->{INDENT} .= '| ';
-    $self->{DIR_NAME} = $$dirInfo{DirName};
-    push @{$self->{PATH}}, $$dirInfo{DirName};
+    my @save = @$self{'INDENT','DIR_NAME','Compression','SubfileType'};
+    $$self{LIST_TAGS} = { };    # don't build lists across different directories
+    $$self{INDENT} .= '| ';
+    $$self{DIR_NAME} = $dirName;
+    push @{$$self{PATH}}, $dirName;
+    $$self{FOUND_DIR}{$dirName} = 1;
 
     # process the directory
+    no strict 'refs';
     my $rtnVal = &$proc($self, $dirInfo, $tagTablePtr);
-
-    pop @{$self->{PATH}};
-    $self->{INDENT} = $oldIndent;
-    $self->{DIR_NAME} = $oldDir;
+    use strict 'refs';
+
+    pop @{$$self{PATH}};
+    @$self{'INDENT','DIR_NAME','Compression','SubfileType'} = @save;
     SetByteOrder($oldOrder);
     return $rtnVal;
@@ -5398 +7673 @@
 #------------------------------------------------------------------------------
 # Get Metadata path
-# Inputs: 0) Exiftool object ref
+# Inputs: 0) ExifTool object ref
 # Return: Metadata path string
 sub MetadataPath($)
@@ -5414 +7689 @@
     my $filename = shift;
     my $fileExt;
-    if ($filename and $filename =~ /.*\.(.+)$/) {
+    if ($filename and $filename =~ /^.*\.([^.]+)$/s) {
         $fileExt = uc($1);   # change extension to upper case
         # convert TIF extension to TIFF because we use the
@@ -5433 +7708 @@
     my $tagInfo = $$tagTablePtr{$tagID};
 
-    if (ref $tagInfo eq 'HASH') {
+    if ($specialTags{$tagID}) {
+        # (hopefully this won't happen)
+        warn "Tag $tagID conflicts with internal ExifTool variable in $$tagTablePtr{TABLE_NAME}\n";
+    } elsif (ref $tagInfo eq 'HASH') {
         return ($tagInfo);
     } elsif (ref $tagInfo eq 'ARRAY') {
@@ -5477 +7755 @@
             }
         }
-        if ($$tagInfo{Unknown} and not $$self{OPTIONS}{Unknown} and not $$self{OPTIONS}{Verbose}) {
+        if ($$tagInfo{Unknown} and not $$self{OPTIONS}{Unknown} and
+            not $$self{OPTIONS}{Verbose} and not $$self{HTML_DUMP})
+        {
             # don't return Unknown tags unless that option is set
             return undef;
@@ -5515 +7795 @@
 # Add new tag to table (must use this routine to add new tags to a table)
 # Inputs: 0) reference to tag table, 1) tag ID
-#         2) [optional] reference to tag information hash
-# Notes: - will not overwrite existing entry in table
+#         2) [optional] tag name or reference to tag information hash
+#         3) [optional] flag to avoid adding prefix when generating tag name
+# Returns: tagInfo ref
+# Notes: - will not override existing entry in table
 # - info need contain no entries when this routine is called
-sub AddTagToTable($$;$)
-{
-    my ($tagTablePtr, $tagID, $tagInfo) = @_;
-    $tagInfo or $tagInfo = { };
+# - tag name is cleaned if necessary
+sub AddTagToTable($$;$$)
+{
+    my ($tagTablePtr, $tagID, $tagInfo, $noPrefix) = @_;
+
+    # generate tag info hash if necessary
+    $tagInfo = $tagInfo ? { Name => $tagInfo } : { } unless ref $tagInfo eq 'HASH';
 
     # define necessary entries in information hash
@@ -5527 +7812 @@
         # fill in default groups from table GROUPS
         foreach (keys %{$$tagTablePtr{GROUPS}}) {
-            next if $tagInfo->{Groups}{$_};
-            $tagInfo->{Groups}{$_} = $tagTablePtr->{GROUPS}{$_};
+            next if $$tagInfo{Groups}{$_};
+            $$tagInfo{Groups}{$_} = $$tagTablePtr{GROUPS}{$_};
         }
     } else {
@@ -5537 +7822 @@
     $$tagInfo{Table} = $tagTablePtr;
     $$tagInfo{TagID} = $tagID;
+    if (defined $$tagTablePtr{AVOID} and not defined $$tagInfo{Avoid}) {
+        $$tagInfo{Avoid} = $$tagTablePtr{AVOID};
+    }
 
     my $name = $$tagInfo{Name};
-    if (defined $name) {
-        $name =~ tr/-_a-zA-Z0-9//dc;    # remove illegal characters
-    } else {
-        # construct a name from the tag ID
-        $name = $tagID;
-        $name =~ tr/-_a-zA-Z0-9//dc;    # remove illegal characters
-        $name = ucfirst $name;          # start with uppercase
-        # make sure name is a reasonable length
-        my $prefix = $$tagTablePtr{TAG_PREFIX};
-        if ($prefix) {
-            # make description to prevent tagID from getting mangled by MakeDescription()
-            $$tagInfo{Description} = MakeDescription($prefix, $name);
-            $name = "${prefix}_$name";
-        }
-    }
-    # tag names must be at least 2 characters long and begin with a letter
-    $name = "Tag$name" if length($name) <= 1 or $name !~ /^[A-Z]/i;
+    $name = $tagID unless defined $name;
+    $name =~ tr/-_a-zA-Z0-9//dc;    # remove illegal characters
+    $name = ucfirst $name;          # capitalize first letter
+    # add tag-name prefix if specified and tag name not provided
+    unless (defined $$tagInfo{Name} or $noPrefix or not $$tagTablePtr{TAG_PREFIX}) {
+        # make description to prevent tagID from getting mangled by MakeDescription()
+        $$tagInfo{Description} = MakeDescription($$tagTablePtr{TAG_PREFIX}, $name);
+        $name = "$$tagTablePtr{TAG_PREFIX}_$name";
+    }
+    # tag names must be at least 2 characters long and prefer them to start with a letter
+    $name = "Tag$name" if length($name) < 2 or $name !~ /^[A-Z]/i;
     $$tagInfo{Name} = $name;
-    # add tag to table, but never overwrite existing entries (could potentially happen
+    # add tag to table, but never override existing entries (could potentially happen
     # if someone thinks there isn't any tagInfo because a condition wasn't satisfied)
-    $$tagTablePtr{$tagID} = $tagInfo unless defined $$tagTablePtr{$tagID};
+    unless (defined $$tagTablePtr{$tagID} or $specialTags{$tagID}) {
+        $$tagTablePtr{$tagID} = $tagInfo;
+    }
+    return $tagInfo;
 }
 
@@ -5565 +7850 @@
 # Handle simple extraction of new tag information
 # Inputs: 0) ExifTool object ref, 1) tag table reference, 2) tagID, 3) value,
-#         4-N) parameters hash: Index, DataPt, DataPos, Start, Size, Parent,
-#              TagInfo, ProcessProc, RAF
+#         4-N) parameters hash: Index, DataPt, DataPos, Base, Start, Size, Parent,
+#              TagInfo, ProcessProc, RAF, Format, Count
 # Returns: tag key or undef if tag not found
 # Notes: if value is not defined, it is extracted from DataPt using TagInfo
@@ -5573 +7858 @@
 {
     my ($self, $tagTablePtr, $tag, $val, %parms) = @_;
-    my $verbose = $self->{OPTIONS}{Verbose};
-    my $tagInfo = $parms{TagInfo} || $self->GetTagInfo($tagTablePtr, $tag, \$val);
+    my $verbose = $$self{OPTIONS}{Verbose};
+    my $pfmt = $parms{Format};
+    my $tagInfo = $parms{TagInfo} || $self->GetTagInfo($tagTablePtr, $tag, \$val, $pfmt, $parms{Count});
     my $dataPt = $parms{DataPt};
-    my ($subdir, $format, $count, $size, $noTagInfo);
+    my ($subdir, $format, $noTagInfo, $rational);
 
     if ($tagInfo) {
-        $subdir = $$tagInfo{SubDirectory}
+        $subdir = $$tagInfo{SubDirectory};
     } else {
         return undef unless $verbose;
@@ -5586 +7872 @@
     }
     # read value if not done already (not necessary for subdir)
-    unless (defined $val or ($subdir and not $$tagInfo{Writable})) {
+    unless (defined $val or ($subdir and not $$tagInfo{Writable} and not $$tagInfo{RawConv})) {
         my $start = $parms{Start} || 0;
-        my $size = $parms{Size} || 0;
+        my $dLen = $dataPt ? length($$dataPt) : -1;
+        my $size = $parms{Size};
+        $size = $dLen unless defined $size;
         # read from data in memory if possible
-        if ($dataPt and $start >= 0 and $start + $size <= length($$dataPt)) {
+        if ($start >= 0 and $start + $size <= $dLen) {
             $format = $$tagInfo{Format} || $$tagTablePtr{FORMAT};
+            $format = $pfmt if not $format and $pfmt and $formatSize{$pfmt};
             if ($format) {
-                $val = ReadValue($dataPt, $start, $format, $$tagInfo{Count}, $size);
+                $val = ReadValue($dataPt, $start, $format, $$tagInfo{Count}, $size, \$rational);
             } else {
                 $val = substr($$dataPt, $start, $size);
@@ -5606 +7895 @@
         undef $tagInfo if $noTagInfo;
         $parms{Value} = $val;
+        $parms{Value} .= " ($rational)" if defined $rational;
         $parms{Table} = $tagTablePtr;
         if ($format) {
-            $count or $count = int(($parms{Size} || 0) / ($formatSize{$format} || 1));
+            my $count = int(($parms{Size} || 0) / ($formatSize{$format} || 1));
             $parms{Format} = $format . "[$count]";
         }
@@ -5617 +7907 @@
             my $subdirStart = $parms{Start};
             my $subdirLen = $parms{Size};
+            if ($$tagInfo{RawConv} and not $$tagInfo{Writable}) {
+                my $conv = $$tagInfo{RawConv};
+                local $SIG{'__WARN__'} = \&SetWarning;
+                undef $evalWarning;
+                if (ref $conv eq 'CODE') {
+                    $val = &$conv($val, $self);
+                } else {
+                    my ($priority, @grps);
+                    # NOTE: RawConv is evaluated in Writer.pl and twice in ExifTool.pm
+                    #### eval RawConv ($self, $val, $tag, $tagInfo, $priority, @grps)
+                    $val = eval $conv;
+                    $@ and $evalWarning = $@;
+                }
+                $self->Warn("RawConv $tag: " . CleanWarning()) if $evalWarning;
+                return undef unless defined $val;
+                $val = $$val if ref $val eq 'SCALAR';
+                $dataPt = \$val;
+                $subdirStart = 0;
+                $subdirLen = length $val;
+            }
             if ($$subdir{Start}) {
                 my $valuePtr = 0;
                 #### eval Start ($valuePtr)
-                my $off = eval $$subdir{Start};                
+                my $off = eval $$subdir{Start};
                 $subdirStart += $off;
                 $subdirLen -= $off;
@@ -5640 +7950 @@
             );
             my $oldOrder = GetByteOrder();
-            SetByteOrder($$subdir{ByteOrder}) if $$subdir{ByteOrder};
+            if ($$subdir{ByteOrder}) {
+                if ($$subdir{ByteOrder} eq 'Unknown') {
+                    if ($subdirStart + 2 <= $subdirLen) {
+                        # attempt to determine the byte ordering of an IFD-style subdirectory
+                        my $num = Get16u($dataPt, $subdirStart);
+                        ToggleByteOrder if $num & 0xff00 and ($num>>8) > ($num&0xff);
+                    }
+                } else {
+                    SetByteOrder($$subdir{ByteOrder});
+                }
+            }
             my $subTablePtr = GetTagTable($$subdir{TagTable}) || $tagTablePtr;
             $self->ProcessDirectory(\%dirInfo, $subTablePtr, $$subdir{ProcessProc} || $parms{ProcessProc});
@@ -5647 +7967 @@
             return undef unless $$tagInfo{Writable};
         }
-        return $self->FoundTag($tagInfo, $val);
+        my $key = $self->FoundTag($tagInfo, $val);
+        # save original components of rational numbers
+        $$self{RATIONAL}{$key} = $rational if defined $rational and defined $key;
+        return $key;
     }
     return undef;
@@ -5657 +7980 @@
 #         1) reference to tagInfo hash or tag name
 #         2) data value (or reference to require hash if Composite)
+#         3) optional family 0 group, 4) optional family 1 group
 # Returns: tag key or undef if no value
-sub FoundTag($$$)
+sub FoundTag($$$;@)
 {
     local $_;
-    my ($self, $tagInfo, $value) = @_;
-    my $tag;
+    my ($self, $tagInfo, $value, @grps) = @_;
+    my ($tag, $noListDel);
+    my $options = $$self{OPTIONS};
 
     if (ref $tagInfo eq 'HASH') {
@@ -5673 +7998 @@
         # (not advised to do this since the tag won't show in list)
         $tagInfo or $tagInfo = { Name => $tag, Groups => \%allGroupsExifTool };
-        $self->{OPTIONS}{Verbose} and $self->VerboseInfo(undef, $tagInfo, Value => $value);
-    }
-    my $rawValueHash = $self->{VALUE};
+        $$options{Verbose} and $self->VerboseInfo(undef, $tagInfo, Value => $value);
+    }
+    # get tag priority
+    my $priority = $$tagInfo{Priority};
+    unless (defined $priority) {
+        $priority = $$tagInfo{Table}{PRIORITY};
+        $priority = 0 if not defined $priority and $$tagInfo{Avoid};
+    }
+    $grps[0] or $grps[0] = $$self{SET_GROUP0};
+    $grps[1] or $grps[1] = $$self{SET_GROUP1};
+    my $valueHash = $$self{VALUE};
+
     if ($$tagInfo{RawConv}) {
         # initialize @val for use in Composite RawConv expressions
         my @val;
-        if (ref $value eq 'HASH') {
-            foreach (keys %$value) { $val[$_] = $$rawValueHash{$$value{$_}}; }
+        if (ref $value eq 'HASH' and $$tagInfo{IsComposite}) {
+            foreach (keys %$value) { $val[$_] = $$valueHash{$$value{$_}}; }
         }
         my $conv = $$tagInfo{RawConv};
@@ -5687 +8021 @@
         if (ref $conv eq 'CODE') {
             $value = &$conv($value, $self);
+            $$self{grps} and @grps = @{$$self{grps}}, delete $$self{grps};
         } else {
-            my $val = $value;   # must do this in case eval references $val
+            my $val = $value;   # do this so eval can use $val
             # NOTE: RawConv is also evaluated in Writer.pl
-            #### eval RawConv ($self, $val, $tag, $tagInfo)
+            #### eval RawConv ($self, $val, $tag, $tagInfo, $priority, @grps)
             $value = eval $conv;
             $@ and $evalWarning = $@;
@@ -5697 +8032 @@
         return undef unless defined $value;
     }
-    # get tag priority
-    my $priority = $$tagInfo{Priority};
-    defined $priority or $priority = $tagInfo->{Table}{PRIORITY};
     # handle duplicate tag names
-    if (defined $$rawValueHash{$tag}) {
+    if (defined $$valueHash{$tag}) {
         # add to list if there is an active list for this tag
-        if ($self->{LIST_TAGS}{$tagInfo}) {
-            $tag = $self->{LIST_TAGS}{$tagInfo};  # use key from previous list tag
-            if (ref $$rawValueHash{$tag} ne 'ARRAY') {
-                $$rawValueHash{$tag} = [ $$rawValueHash{$tag} ];
-            }
-            push @{$$rawValueHash{$tag}}, $value;
-            return $tag;    # return without creating a new entry
+        if ($$self{LIST_TAGS}{$tagInfo}) {
+            $tag = $$self{LIST_TAGS}{$tagInfo}; # use key from previous list tag
+            if (defined $$self{NO_LIST}) {
+                # accumulate list in TAG_EXTRA "NoList" element
+                if (defined $$self{TAG_EXTRA}{$tag}{NoList}) {
+                    push @{$$self{TAG_EXTRA}{$tag}{NoList}}, $value;
+                } else {
+                    $$self{TAG_EXTRA}{$tag}{NoList} = [ $$valueHash{$tag}, $value ];
+                }
+                $noListDel = 1; # set flag to delete this tag if re-listed
+            } else {
+                if (ref $$valueHash{$tag} ne 'ARRAY') {
+                    $$valueHash{$tag} = [ $$valueHash{$tag} ];
+                }
+                push @{$$valueHash{$tag}}, $value;
+                return $tag;    # return without creating a new entry
+            }
         }
         # get next available tag key
-        my $nextInd = $self->{DUPL_TAG}{$tag} = ($self->{DUPL_TAG}{$tag} || 0) + 1;
+        my $nextInd = $$self{DUPL_TAG}{$tag} = ($$self{DUPL_TAG}{$tag} || 0) + 1;
         my $nextTag = "$tag ($nextInd)";
 #
@@ -5718 +8060 @@
 #
         # promote existing 0-priority tag so it takes precedence over a new 0-tag
-        # (unless old tag was a sub-document and new tag isn't)
-        my $oldPriority = $self->{PRIORITY}{$tag};
+        # (unless old tag was a sub-document and new tag isn't.  Also, never override
+        #  a Warning tag because they may be added by ValueConv, which could be confusing)
+        my $oldPriority = $$self{PRIORITY}{$tag};
         unless ($oldPriority) {
-            if ($self->{DOC_NUM} or not $self->{TAG_EXTRA}{$tag} or
-                                    not $self->{TAG_EXTRA}{$tag}{G3})
+            if ($$self{DOC_NUM} or not $$self{TAG_EXTRA}{$tag} or $tag eq 'Warning' or
+                                   not $$self{TAG_EXTRA}{$tag}{G3})
             {
                 $oldPriority = 1;
@@ -5734 +8077 @@
             $priority = 1 if not $priority and $$self{DIR_NAME} and
                              $$self{DIR_NAME} eq $$self{PRIORITY_DIR};
-        } elsif ($$self{DIR_NAME} and $$self{LOW_PRIORITY_DIR}{$$self{DIR_NAME}}) {
+        } elsif ($$self{LOW_PRIORITY_DIR}{'*'} or
+            ($$self{DIR_NAME} and $$self{LOW_PRIORITY_DIR}{$$self{DIR_NAME}}))
+        {
             $priority = 0;  # default is 0 for a LOW_PRIORITY_DIR
         } else {
             $priority = 1;  # the normal default
         }
-        if ($priority >= $oldPriority and not $self->{DOC_NUM}) {
+        if ($priority >= $oldPriority and (not $$self{DOC_NUM} or
+            ($$self{TAG_EXTRA}{$tag} and $$self{TAG_EXTRA}{$tag}{G3} and
+             $$self{DOC_NUM} eq $$self{TAG_EXTRA}{$tag}{G3})) and not $noListDel)
+        {
             # move existing tag out of the way since this tag is higher priority
-            $self->{MOVED_KEY} = $nextTag;  # used in BuildCompositeTags()
-            $self->{PRIORITY}{$nextTag} = $self->{PRIORITY}{$tag};
-            $$rawValueHash{$nextTag} = $$rawValueHash{$tag};
-            $self->{FILE_ORDER}{$nextTag} = $self->{FILE_ORDER}{$tag};
-            my $oldInfo = $self->{TAG_INFO}{$nextTag} = $self->{TAG_INFO}{$tag};
-            if ($self->{TAG_EXTRA}{$tag}) {
-                $self->{TAG_EXTRA}{$nextTag} = $self->{TAG_EXTRA}{$tag};
-                delete $self->{TAG_EXTRA}{$tag};
-            }
+            # (NOTE: any new members added here must also be added to DeleteTag())
+            $$self{PRIORITY}{$nextTag} = $$self{PRIORITY}{$tag};
+            $$valueHash{$nextTag} = $$valueHash{$tag};
+            $$self{FILE_ORDER}{$nextTag} = $$self{FILE_ORDER}{$tag};
+            my $oldInfo = $$self{TAG_INFO}{$nextTag} = $$self{TAG_INFO}{$tag};
+            foreach ('TAG_EXTRA','RATIONAL') {
+                if ($$self{$_}{$tag}) {
+                    $$self{$_}{$nextTag} = $$self{$_}{$tag};
+                    delete $$self{$_}{$tag};
+                }
+            }
+            delete $$self{BOTH}{$tag};
             # update tag key for list if necessary
-            $self->{LIST_TAGS}{$oldInfo} = $nextTag if $self->{LIST_TAGS}{$oldInfo};
+            $$self{LIST_TAGS}{$oldInfo} = $nextTag if $$self{LIST_TAGS}{$oldInfo};
+            # update this key if used in a Composite tag
+            if ($$self{COMP_KEYS}{$tag}) {
+                $$_[0]{$$_[1]} = $nextTag foreach @{$$self{COMP_KEYS}{$tag}};
+                $$self{COMP_KEYS}{$nextTag} = $$self{COMP_KEYS}{$tag};
+                delete $$self{COMP_KEYS}{$tag};
+            }
         } else {
             $tag = $nextTag;        # don't override the existing tag
         }
-        $self->{PRIORITY}{$tag} = $priority;
+        $$self{PRIORITY}{$tag} = $priority;
+        $$self{TAG_EXTRA}{$tag}{NoListDel} = 1 if $noListDel;
     } elsif ($priority) {
-        # set tag priority (only if exists and non-zero)
-        $self->{PRIORITY}{$tag} = $priority;
+        # set tag priority (only if exists and is non-zero)
+        $$self{PRIORITY}{$tag} = $priority;
     }
 
     # save the raw value, file order, tagInfo ref, group1 name,
     # and tag key for lists if necessary
-    $$rawValueHash{$tag} = $value;
-    $self->{FILE_ORDER}{$tag} = ++$self->{NUM_FOUND};
-    $self->{TAG_INFO}{$tag} = $tagInfo;
-    # set dynamic groups 1 and 3 if necessary
-    $self->{TAG_EXTRA}{$tag}{G1} = $self->{SET_GROUP1} if $self->{SET_GROUP1};
-    if ($self->{DOC_NUM}) {
-        $self->{TAG_EXTRA}{$tag}{G3} = $self->{DOC_NUM};
-        if ($self->{DOC_NUM} =~ /^(\d+)/) {
+    $$valueHash{$tag} = $value;
+    $$self{FILE_ORDER}{$tag} = ++$$self{NUM_FOUND};
+    $$self{TAG_INFO}{$tag} = $tagInfo;
+    # set dynamic groups 0, 1 and 3 if necessary
+    $$self{TAG_EXTRA}{$tag}{G0} = $grps[0] if $grps[0];
+    $$self{TAG_EXTRA}{$tag}{G1} = $grps[1] if $grps[1];
+    if ($$self{DOC_NUM}) {
+        $$self{TAG_EXTRA}{$tag}{G3} = $$self{DOC_NUM};
+        if ($$self{DOC_NUM} =~ /^(\d+)/) {
             # keep track of maximum 1st-level sub-document number
-            $self->{DOC_COUNT} = $1 unless $self->{DOC_COUNT} >= $1;
+            $$self{DOC_COUNT} = $1 unless $$self{DOC_COUNT} >= $1;
         }
     }
     # save path if requested
-    $self->{TAG_EXTRA}{$tag}{G5} = $self->MetadataPath() if $self->{OPTIONS}{SavePath};
+    $$self{TAG_EXTRA}{$tag}{G5} = $self->MetadataPath() if $$options{SavePath};
 
     # remember this tagInfo if we will be accumulating values in a list
-    $self->{LIST_TAGS}{$tagInfo} = $tag if $$tagInfo{List} and not $$self{NO_LIST};
+    # (but don't override earlier list if this may be deleted by NoListDel flag)
+    if ($$tagInfo{List} and not $$self{NO_LIST} and not $noListDel) {
+        $$self{LIST_TAGS}{$tagInfo} = $tag;
+    }
+
+    # validate tag if requested (but only for simple values -- could result
+    # in infinite recursion if called for a Composite tag (HASH ref value)
+    # because FoundTag is called in the middle of building Composite tags
+    if ($$options{Validate} and not ref $value) {
+        Image::ExifTool::Validate::ValidateRaw($self, $tag, $value);
+    }
 
     return $tag;
@@ -5790 +8159 @@
 {
     my $self = shift;
-    $self->{PRIORITY_DIR} = $self->{DIR_NAME} unless $self->{PRIORITY_DIR};
+    $$self{PRIORITY_DIR} = $$self{DIR_NAME} unless $$self{PRIORITY_DIR};
 }
 
@@ -5799 +8168 @@
 {
     my ($self, $tagKey, $extra, $fam) = @_;
-    $self->{TAG_EXTRA}{$tagKey}{defined $fam ? "G$fam" : 'G1'} = $extra;
+    $$self{TAG_EXTRA}{$tagKey}{defined $fam ? "G$fam" : 'G1'} = $extra;
 }
 
@@ -5808 +8177 @@
 {
     my ($self, $tag) = @_;
-    delete $self->{VALUE}{$tag};
-    delete $self->{FILE_ORDER}{$tag};
-    delete $self->{TAG_INFO}{$tag};
-    delete $self->{TAG_EXTRA}{$tag};
+    delete $$self{VALUE}{$tag};
+    delete $$self{FILE_ORDER}{$tag};
+    delete $$self{TAG_INFO}{$tag};
+    delete $$self{TAG_EXTRA}{$tag};
+    delete $$self{PRIORITY}{$tag};
+    delete $$self{RATIONAL}{$tag};
+    delete $$self{BOTH}{$tag};
 }
 
@@ -5838 +8210 @@
 #         1) Optional file type (uses FILE_TYPE if not specified)
 #         2) Optional MIME type (uses our lookup if not specified)
+#         3) Optional recommended extension (converted to lower case; uses FileType if undef)
 # Notes:  Will NOT set file type twice (subsequent calls ignored)
-sub SetFileType($;$$)
-{
-    my ($self, $fileType, $mimeType) = @_;
-    unless ($self->{VALUE}{FileType}) {
-        my $baseType = $self->{FILE_TYPE};
+sub SetFileType($;$$$)
+{
+    my ($self, $fileType, $mimeType, $normExt) = @_;
+    unless ($$self{VALUE}{FileType} and not $$self{DOC_NUM}) {
+        my $baseType = $$self{FILE_TYPE};
+        my $ext = $$self{FILE_EXT};
         $fileType or $fileType = $baseType;
+        # handle sub-types which are identified by extension
+        if (defined $ext and $ext ne $fileType and not $$self{DOC_NUM}) {
+            my ($f,$e) = @fileTypeLookup{$fileType,$ext};
+            if (ref $f eq 'ARRAY' and ref $e eq 'ARRAY' and $$f[0] eq $$e[0]) {
+                # make sure $fileType was a root type and not another sub-type
+                $fileType = $ext if $$f[0] eq $fileType or not $fileTypeLookup{$$f[0]};
+            }
+        }
         $mimeType or $mimeType = $mimeType{$fileType};
         # use base file type if necessary (except if 'TIFF', which is a special case)
         $mimeType = $mimeType{$baseType} unless $mimeType or $baseType eq 'TIFF';
+        unless (defined $normExt) {
+            $normExt = $fileTypeExt{$fileType};
+            $normExt = $fileType unless defined $normExt;
+        }
+        $$self{FileType} = $fileType;
         $self->FoundTag('FileType', $fileType);
+        $self->FoundTag('FileTypeExtension', uc $normExt);
         $self->FoundTag('MIMEType', $mimeType || 'application/unknown');
     }
@@ -5855 +8243 @@
 #------------------------------------------------------------------------------
 # Override the FileType and MIMEType tags
-# Inputs: 0) ExifTool object ref, 1) file type
+# Inputs: 0) ExifTool object ref, 1) file type, 2) MIME type, 3) normal extension
 # Notes:  does nothing if FileType was not previously defined (ie. when writing)
-sub OverrideFileType($$)
-{
-    my ($self, $fileType) = @_;
+sub OverrideFileType($$;$$)
+{
+    my ($self, $fileType, $mimeType, $normExt) = @_;
     if (defined $$self{VALUE}{FileType} and $fileType ne $$self{VALUE}{FileType}) {
+        $$self{FileType} = $fileType;
         $$self{VALUE}{FileType} = $fileType;
-        $$self{VALUE}{MIMEType} = $mimeType{$fileType} || 'application/unknown';
+        unless (defined $normExt) {
+            $normExt = $fileTypeExt{$fileType};
+            $normExt = $fileType unless defined $normExt;
+        }
+        $$self{VALUE}{FileTypeExtension} = uc $normExt;
+        $mimeType or $mimeType = $mimeType{$fileType};
+        $$self{VALUE}{MIMEType} = $mimeType if $mimeType;
         if ($$self{OPTIONS}{Verbose}) {
             $self->VPrint(0,"$$self{INDENT}FileType [override] = $fileType\n");
-            $self->VPrint(0,"$$self{INDENT}MIMEType [override] = $$self{VALUE}{MIMEType}\n");
+            $self->VPrint(0,"$$self{INDENT}FileTypeExtension [override] = $$self{VALUE}{FileTypeExtension}\n");
+            $self->VPrint(0,"$$self{INDENT}MIMEType [override] = $mimeType\n") if $mimeType;
         }
     }
@@ -5878 +8274 @@
     my ($self, $mime) = @_;
     $mime =~ m{/} or $mime = $mimeType{$mime} or return;
-    my $old = $self->{VALUE}{MIMEType};
+    my $old = $$self{VALUE}{MIMEType};
     if (defined $old) {
         my ($a, $b) = split '/', $old;
         my ($c, $d) = split '/', $mime;
         $d =~ s/^x-//;
-        $self->{VALUE}{MIMEType} = "$c/$b-$d";
+        $$self{VALUE}{MIMEType} = "$c/$b-$d";
         $self->VPrint(0, "  Modified MIMEType = $c/$b-$d\n");
     } else {
@@ -5897 +8293 @@
     my $self = shift;
     my $level = shift;
-    if ($self->{OPTIONS}{Verbose} and $self->{OPTIONS}{Verbose} > $level) {
-        my $out = $self->{OPTIONS}{TextOut};
+    if ($$self{OPTIONS}{Verbose} and $$self{OPTIONS}{Verbose} > $level) {
+        my $out = $$self{OPTIONS}{TextOut};
         print $out @_;
-    }
+        print $out "\n" unless $_[-1] =~ /\n$/;
+    }
+}
+
+#------------------------------------------------------------------------------
+# Print verbose directory information
+# Inputs: 0) ExifTool object reference, 1) directory name or dirInfo ref
+#         2) number of entries in directory (or 0 if unknown)
+#         3) optional size of directory in bytes
+sub VerboseDir($$;$$)
+{
+    my ($self, $name, $entries, $size) = @_;
+    return unless $$self{OPTIONS}{Verbose};
+    if (ref $name eq 'HASH') {
+        $size = $$name{DirLen} unless $size;
+        $name = $$name{Name} || $$name{DirName};
+    }
+    my $indent = substr($$self{INDENT}, 0, -2);
+    my $out = $$self{OPTIONS}{TextOut};
+    my $str = ($entries or defined $entries and not $size) ? " with $entries entries" : '';
+    $str .= ", $size bytes" if $size;
+    print $out "$indent+ [$name directory$str]\n";
 }
 
@@ -5910 +8327 @@
     my $self = shift;
     my $dataPt = shift;
-    if ($self->{OPTIONS}{Verbose} and $self->{OPTIONS}{Verbose} > 2) {
+    my $verbose = $$self{OPTIONS}{Verbose};
+    if ($verbose and $verbose > 2) {
         my %parms = (
-            Prefix => $self->{INDENT},
-            Out    => $self->{OPTIONS}{TextOut},
-            MaxLen => $self->{OPTIONS}{Verbose} < 4 ? 96 : undef,
+            Prefix => $$self{INDENT},
+            Out    => $$self{OPTIONS}{TextOut},
+            MaxLen => $verbose < 4 ? 96 : $verbose < 5 ? 2048 : undef,
         );
         HexDump($dataPt, undef, %parms, @_);
     }
+}
+
+#------------------------------------------------------------------------------
+# Print data in hex
+# Inputs: 0) data
+# Returns: hex string
+# (this is a convenience function for use in debugging PrintConv statements)
+sub PrintHex($)
+{
+    my $val = shift;
+    return join(' ', unpack('H2' x length($val), $val));
 }
 
@@ -5931 +8360 @@
     my ($isPreview, $buff);
 
-    if ($tag and $tag eq 'PreviewImage') {
-        # save PreviewImage start/length in case we want to dump trailer
-        $$self{PreviewImageStart} = $offset;
-        $$self{PreviewImageLength} = $length;
-        $isPreview = 1;
-    }
-    if ($tag and not $self->{OPTIONS}{Binary} and not $self->{OPTIONS}{Verbose} and
-        not $self->{REQ_TAG_LOOKUP}{lc($tag)})
-    {
-        return "Binary data $length bytes";
-    }
-    unless ($self->{RAF}->Seek($offset,0)
-        and $self->{RAF}->Read($buff, $length) == $length)
+    if ($tag) {
+        if ($tag eq 'PreviewImage') {
+            # save PreviewImage start/length in case we want to dump trailer
+            $$self{PreviewImageStart} = $offset;
+            $$self{PreviewImageLength} = $length;
+            $isPreview = 1;
+        }
+        my $lcTag = lc $tag;
+        if ((not $$self{OPTIONS}{Binary} or $$self{EXCL_TAG_LOOKUP}{$lcTag}) and
+             not $$self{OPTIONS}{Verbose} and not $$self{REQ_TAG_LOOKUP}{$lcTag})
+        {
+            return "Binary data $length bytes";
+        }
+    }
+    unless ($$self{RAF}->Seek($offset,0)
+        and $$self{RAF}->Read($buff, $length) == $length)
     {
         $tag or $tag = 'binary data';
@@ -5969 +8401 @@
     my $size = $$dirInfo{DirLen} || (length($$dataPt) - $offset);
     my $base = $$dirInfo{Base} || 0;
-    my $verbose = $self->{OPTIONS}{Verbose};
-    my $unknown = $self->{OPTIONS}{Unknown};
+    my $verbose = $$self{OPTIONS}{Verbose};
+    my $unknown = $$self{OPTIONS}{Unknown};
     my $dataPos = $$dirInfo{DataPos} || 0;
 
@@ -5982 +8414 @@
     }
     # prepare list of tag numbers to extract
-    my @tags;
+    my (@tags, $topIndex);
     if ($unknown > 1 and defined $$tagTablePtr{FIRST_ENTRY}) {
+        # don't create a stupid number of tags if data is huge
+        my $sizeLimit = $size < 65536 ? $size : 65536;
         # scan through entire binary table
-        @tags = ($$tagTablePtr{FIRST_ENTRY}..(int($size/$increment) - 1));
+        $topIndex = int($sizeLimit/$increment);
+        @tags = ($$tagTablePtr{FIRST_ENTRY}..($topIndex - 1));
         # add in floating point tag ID's if they exist
         my @ftags = grep /\./, TagTableKeys($tagTablePtr);
@@ -5992 +8427 @@
         @tags = @{$$dirInfo{DataMember}};
         $verbose = 0;   # no verbose output of extracted values when writing
+    } elsif ($$dirInfo{MixedTags}) {
+        # process sorted integer-ID tags only
+        @tags = sort { $a <=> $b } grep /^\d+$/, TagTableKeys($tagTablePtr);
     } else {
         # extract known tags in numerical order
-        @tags = sort { $a <=> $b } TagTableKeys($tagTablePtr);
+        @tags = sort { ($a < 0 ? $a + 1e9 : $a) <=> ($b < 0 ? $b + 1e9 : $b) } TagTableKeys($tagTablePtr);
     }
     $self->VerboseDir('BinaryData', undef, $size) if $verbose;
@@ -6003 +8441 @@
     my $varSize = 0;
     foreach $index (@tags) {
-        my ($tagInfo, $val, $saveNextIndex, $len, $mask, $wasVar);
+        my ($tagInfo, $val, $saveNextIndex, $len, $mask, $wasVar, $rational);
         if ($$tagTablePtr{$index}) {
             $tagInfo = $self->GetTagInfo($tagTablePtr, $index);
@@ -6009 +8447 @@
                 next unless defined $tagInfo;
                 my $entry = int($index) * $increment + $varSize;
+                if ($entry < 0) {
+                    $entry += $size;
+                    next if $entry < 0;
+                }
                 next if $entry >= $size;
                 my $more = $size - $entry;
@@ -6018 +8460 @@
             next if $$tagInfo{Unknown} and
                    ($$tagInfo{Unknown} > $unknown or $index < $nextIndex);
+        } elsif ($topIndex and $$tagTablePtr{$index - $topIndex}) {
+            $tagInfo = $self->GetTagInfo($tagTablePtr, $index - $topIndex) or next;
         } else {
             # don't generate unknown tags in binary tables unless Unknown > 1
@@ -6027 +8471 @@
         # get relative offset of this entry
         my $entry = int($index) * $increment + $varSize;
+        # allow negative indices to represent bytes from end
+        if ($entry < 0) {
+            $entry += $size;
+            next if $entry < 0;
+        }
         my $more = $size - $entry;
         last if $more <= 0;     # all done if we have reached the end of data
@@ -6050 +8499 @@
                 $@ and warn("Format $$tagInfo{Name}: $@"), next;
                 next if $count < 0;
-                # allow a variable-length of any format type (with base $count = 1)
+                # allow a variable-length value of any format
+                # (note: the next incremental index points to data immediately after
+                #  this value, regardless of the size of this value, even if it is zero)
                 if ($format =~ s/^var_//) {
-                    $varSize += ($count - 1) * ($formatSize{$format} || 1);
+                    $varSize += $count * ($formatSize{$format} || 1) - $increment;
+                    $wasVar = 1;
                     # save variable size data if required for writing
                     if ($$dirInfo{VarFormatData}) {
-                        push @{$$dirInfo{VarFormatData}}, $index, $varSize;
+                        push @{$$dirInfo{VarFormatData}}, [ $index, $varSize, $format ];
                     }
+                    # don't extract value if large and we wanted it just to get
+                    # the variable-format information when writing
+                    next if $$tagInfo{LargeTag} and $$dirInfo{VarFormatData};
                 }
             } elsif ($format =~ /^var_/) {
@@ -6069 +8524 @@
                     $count = Get8u($dataPt, ($entry++)+$offset);
                     --$more;
-                } elsif ($format eq 'pstr32') {
+                } elsif ($format eq 'pstr32' or $format eq 'ustr32') {
                     last if $more < 4;
                     $count = Get32u($dataPt, $entry + $offset);
+                    $count *= 2 if $format eq 'ustr32';
                     $entry += 4;
                     $more -= 4;
+                    $nextIndex += 4 / $increment;   # (increment next index for int32u)
                 } elsif ($format eq 'int16u') {
                     # int16u size of binary data to follow
@@ -6080 +8537 @@
                     $varSize -= 2;  # ($count includes size word)
                     $format = 'undef';
+                } elsif ($format eq 'ue7') {
+                    require Image::ExifTool::BPG;
+                    ($val, $count) = Image::ExifTool::BPG::Get_ue7($dataPt, $entry + $offset);
+                    last unless defined $val;
+                    --$varSize;     # ($count includes base size of 1 byte)
                 } elsif ($$dataPt =~ /\0/g) {
                     $count = pos($$dataPt) - ($entry+$offset);
@@ -6086 +8548 @@
                 $count = $more if not defined $count or $count > $more;
                 $varSize += $count; # shift subsequent indices
-                $val = substr($$dataPt, $entry+$offset, $count);
-                $val = $self->Decode($val, 'UCS2') if $format eq 'ustring';
-                $val =~ s/\0.*//s unless $format eq 'undef';  # truncate at null
+                unless (defined $val) {
+                    $val = substr($$dataPt, $entry+$offset, $count);
+                    $val = $self->Decode($val, 'UCS2') if $format eq 'ustring' or $format eq 'ustr32';
+                    $val =~ s/\0.*//s unless $format eq 'undef';  # truncate at null
+                }
                 $wasVar = 1;
                 # save variable size data if required for writing
                 if ($$dirInfo{VarFormatData}) {
-                    push @{$$dirInfo{VarFormatData}}, $index, $varSize;
+                    push @{$$dirInfo{VarFormatData}}, [ $index, $varSize, $format ];
                 }
             }
@@ -6098 +8562 @@
         # hook to allow format, etc to be set dynamically
         if (defined $$tagInfo{Hook}) {
-            #### eval Hook ($format, $varSize)
+            my $oldVarSize = $varSize;
+            my $pos = $entry + $offset;
+            #### eval Hook ($format, $varSize, $size, $dataPt, $pos)
             eval $$tagInfo{Hook};
             # save variable size data if required for writing (in case changed by Hook)
             if ($$dirInfo{VarFormatData}) {
-                $#{$$dirInfo{VarFormatData}} -= 2 if $wasVar; # remove previous entries for this tag
-                push @{$$dirInfo{VarFormatData}}, $index, $varSize;
+                $#{$$dirInfo{VarFormatData}} -= 1 if $wasVar; # remove previous entry for this tag
+                push @{$$dirInfo{VarFormatData}}, [ $index, $varSize, $format ];
+            } elsif ($varSize != $oldVarSize and $verbose > 2) {
+                my ($tmp, $sign) = ($varSize, '+');
+                $tmp < 0 and $tmp = -$tmp, $sign = '-';
+                $self->VPrint(2, sprintf("$$self{INDENT}\[offsets adjusted by ${sign}0x%.4x after 0x%.4x $$tagInfo{Name}]\n", $tmp, $index));
             }
         }
@@ -6113 +8583 @@
             $nextIndex = $ni unless $nextIndex > $ni;
         }
+        # allow large tags to be excluded from extraction
+        # (provides a work-around for some tight memory situations)
+        next if $$tagInfo{LargeTag} and $$self{EXCL_TAG_LOOKUP}{lc $$tagInfo{Name}};
         # read value now if necessary
         unless (defined $val and not $$tagInfo{SubDirectory}) {
-            $val = ReadValue($dataPt, $entry+$offset, $format, $count, $more);
+            $val = ReadValue($dataPt, $entry+$offset, $format, $count, $more, \$rational);
             $mask = $$tagInfo{Mask};
-            $val &= $mask if $mask;
+            $val = ($val & $mask) >> $$tagInfo{BitShift} if $mask;
         }
         if ($verbose and not $$tagInfo{Hidden}) {
@@ -6140 +8613 @@
         # parse nested BinaryData directories
         if ($$tagInfo{SubDirectory}) {
-            my $subTablePtr = GetTagTable($tagInfo->{SubDirectory}{TagTable});
+            my $subdir = $$tagInfo{SubDirectory};
+            my $subTablePtr = GetTagTable($$subdir{TagTable});
             # use specified subdirectory length if given
             if ($$tagInfo{Format} and $formatSize{$format}) {
@@ -6146 +8620 @@
                 $len = $more if $len > $more;
             } else {
-                $len = $more;
+                $len = $more;   # directory size is all of remaining data
                 if ($$subTablePtr{PROCESS_PROC} and
                     $$subTablePtr{PROCESS_PROC} eq \&ProcessBinaryData)
@@ -6154 +8628 @@
                 }
             }
+            my $subdirBase = $base;
+            if (defined $$subdir{Base}) {
+                #### eval Base ($start,$base)
+                my $start = $entry + $offset + $dataPos;
+                $subdirBase = eval($$subdir{Base}) + $base;
+            }
+            my $start = $$subdir{Start} || 0;
             my %subdirInfo = (
                 DataPt   => $dataPt,
                 DataPos  => $dataPos,
-                DirStart => $entry + $offset,
-                DirLen   => $len,
-                Base     => $base,
+                DataLen  => length $$dataPt,
+                DirStart => $entry + $offset + $start,
+                DirLen   => $len - $start,
+                Base     => $subdirBase,
             );
-            $self->ProcessDirectory(\%subdirInfo, $subTablePtr);
+            delete $$self{NO_UNKNOWN};
+            $self->ProcessDirectory(\%subdirInfo, $subTablePtr, $$subdir{ProcessProc});
+            $$self{NO_UNKNOWN} = 1 if $unknown < 2;
             next;
         }
         if ($$tagInfo{IsOffset} and $$tagInfo{IsOffset} ne '3') {
-            my $exifTool = $self;
-            #### eval IsOffset ($val, $exifTool)
+            my $et = $self;
+            #### eval IsOffset ($val, $et)
             $val += $base + $$self{BASE} if eval $$tagInfo{IsOffset};
         }
         $val{$index} = $val;
-        unless ($self->FoundTag($tagInfo,$val)) {
+        my $oldBase;
+        if ($$tagInfo{SetBase}) {
+            $oldBase = $$self{BASE};
+            $$self{BASE} += $base;
+        }
+        my $key = $self->FoundTag($tagInfo,$val);
+        $$self{BASE} = $oldBase if defined $oldBase;
+        if ($key) {
+            $$self{RATIONAL}{$key} = $rational if defined $rational;
+        } else {
             # don't increment nextIndex if we didn't extract a tag
             $nextIndex = $saveNextIndex if defined $saveNextIndex;
@@ -6183 +8676 @@
 # (use of noConfig is now deprecated, use configFile = '' instead)
 until ($Image::ExifTool::noConfig) {
-    my $file = $Image::ExifTool::configFile;
-    if (not defined $file) {
-        my $config = '.ExifTool_config';
+    my $config = $Image::ExifTool::configFile;
+    my $file;
+    if (not defined $config) {
+        $config = '.ExifTool_config';
         # get our home directory (HOMEDRIVE and HOMEPATH are used in Windows cmd shell)
         my $home = $ENV{EXIFTOOL_HOME} || $ENV{HOME} ||
@@ -6191 +8685 @@
         # look for the config file in 1) the home directory, 2) the program dir
         $file = "$home/$config";
-        -r $file or $file = ($0 =~ /(.*[\\\/])/ ? $1 : './') . $config;
-        -r $file or last;
     } else {
-        length $file or last;   # filename of "" disables configuration
-        -r $file or warn("Config file not found\n"), last;
-    }
-    eval "require '$file'"; # load the config file
+        length $config or last; # filename of "" disables configuration
+        $file = $config;
+    }
+    # also check executable directory unless path is absolute
+    $exePath = $0 unless defined $exePath; # (undocumented $exePath setting)
+    -r $file or $config =~ /^\// or $file = ($exePath =~ /(.*[\\\/])/ ? $1 : './') . $config;
+    unless (-r $file) {
+        warn("Config file not found\n") if defined $Image::ExifTool::configFile;
+        last;
+    }
+    unshift @INC, '.';      # look in current directory first
+    eval { require $file }; # load the config file
+    shift @INC;
     # print warning (minus "Compilation failed" part)
     $@ and $_=$@, s/Compilation failed.*//s, warn $_;
-    if (@Image::ExifTool::UserDefined::Lenses) {
-        foreach (@Image::ExifTool::UserDefined::Lenses) {
-            $Image::ExifTool::userLens{$_} = 1;
-        }
-    }
     last;
 }
+# read user-defined lenses (may have been defined by script instead of config file)
+if (@Image::ExifTool::UserDefined::Lenses) {
+    foreach (@Image::ExifTool::UserDefined::Lenses) {
+        $Image::ExifTool::userLens{$_} = 1;
+    }
+}
+# add user-defined file types
+if (%Image::ExifTool::UserDefined::FileTypes) {
+    foreach (sort keys %Image::ExifTool::UserDefined::FileTypes) {
+        my $fileInfo = $Image::ExifTool::UserDefined::FileTypes{$_};
+        my $type = uc $_;
+        ref $fileInfo eq 'HASH' or $fileTypeLookup{$type} = $fileInfo, next;
+        my $baseType = $$fileInfo{BaseType};
+        if ($baseType) {
+            if ($$fileInfo{Description}) {
+                $fileTypeLookup{$type} = [ $baseType, $$fileInfo{Description} ];
+            } else {
+                $fileTypeLookup{$type} = $baseType;
+            }
+            if (defined $$fileInfo{Writable} and not $$fileInfo{Writable}) {
+                # first make sure we are using an actual base type and not a derived type
+                $baseType = $fileTypeLookup{$baseType} while $baseType and not ref $fileTypeLookup{$baseType};
+                # mark this type as not writable
+                $noWriteFile{$baseType} or $noWriteFile{$baseType} = [ ];
+                push @{$noWriteFile{$baseType}}, $type;
+            }
+        } else {
+            $fileTypeLookup{$type} = [ $type, $$fileInfo{Description} || $type ];
+            $moduleName{$type} = 0; # not supported
+            if ($$fileInfo{Magic}) {
+                $magicNumber{$type} = $$fileInfo{Magic};
+                push @fileTypes, $type unless grep /^$type$/, @fileTypes;
+            }
+        }
+        $mimeType{$type} = $$fileInfo{MIMEType} if defined $$fileInfo{MIMEType};
+    }
+}
 
 #------------------------------------------------------------------------------