diff --git a/scraper-tests/EmbeddedMetadata/config.py b/scraper-tests/EmbeddedMetadata/config.py new file mode 100644 index 000000000..cdac14e88 --- /dev/null +++ b/scraper-tests/EmbeddedMetadata/config.py @@ -0,0 +1,7 @@ +# +# gets imported when running scraper from test instead of optional config.py in scraper directory +# + +skip_ensure_requirements = True + +FORCE_METHOD = None \ No newline at end of file diff --git a/scraper-tests/EmbeddedMetadata/test-data/EXIF_IPTC_XMP_keywords_title.jpg b/scraper-tests/EmbeddedMetadata/test-data/EXIF_IPTC_XMP_keywords_title.jpg new file mode 100755 index 000000000..eca3c3dd6 Binary files /dev/null and b/scraper-tests/EmbeddedMetadata/test-data/EXIF_IPTC_XMP_keywords_title.jpg differ diff --git a/scraper-tests/EmbeddedMetadata/test-data/EXIF_camera.jpg b/scraper-tests/EmbeddedMetadata/test-data/EXIF_camera.jpg new file mode 100755 index 000000000..cf503ef31 Binary files /dev/null and b/scraper-tests/EmbeddedMetadata/test-data/EXIF_camera.jpg differ diff --git a/scraper-tests/EmbeddedMetadata/test_embedded_metadata.py b/scraper-tests/EmbeddedMetadata/test_embedded_metadata.py new file mode 100644 index 000000000..9d1836ba0 --- /dev/null +++ b/scraper-tests/EmbeddedMetadata/test_embedded_metadata.py @@ -0,0 +1,72 @@ +import os +import sys +import unittest + +# add scraper directory to loadable module path +sys.path.insert(0, os.path.join(os.path.dirname(__file__),'..','..','scrapers', 'EmbeddedMetadata')) +# add py_common path to module path +sys.path.insert(1, os.path.join(os.path.dirname(__file__),'..','..','scrapers')) +import embedded_metadata + +import config + + +class TestEmbeddedMetadataExiftool(unittest.TestCase): + + @classmethod + def setUpClass(cls): + config.FORCE_METHOD = "exiftool" + + def test_EXIF_IPTC_XMP_keywords_title(self): + data = embedded_metadata.process_image(os.path.join(os.path.dirname(__file__), + 'test-data/EXIF_IPTC_XMP_keywords_title.jpg')) + + self.assertSetEqual({'Title', 'Date', 'Tags', 'Details'}, set(data.keys())) + self.assertEqual("Blue Square Test File - .jpg", data['Title']) + self.assertEqual("2005-09-07", data['Date']) + self.assertListEqual( + [{"Name": "XMP"}, {"Name": "Blue Square"}, {"Name": "test file"}, {"Name": "Photoshop"}, {"Name": ".jpg"}, + {'Name': 'XMP keyword 1'},{'Name': 'XMP keyword 2'}], + data['Tags'] + ) + self.assertEqual(('XMPFiles BlueSquare test file, created in Photoshop CS2, saved as .psd, ' + '.jpg, and .tif.\n' + '\n'), data['Details']) + + def test_EXIF_camera(self): + data = embedded_metadata.process_image(os.path.join(os.path.dirname(__file__), 'test-data/EXIF_camera.jpg')) + + self.assertSetEqual(set(data.keys()), {'Date'}) + self.assertEqual("2008-11-01", data['Date']) # not the same as pyexiv2, exiftool does not return exif.DateTime + + +class TestEmbeddedMetadataPyexiv2tool(unittest.TestCase): + + @classmethod + def setUpClass(cls): + config.FORCE_METHOD = "pyexiv2" + + def test_EXIF_IPTC_XMP_keywords_title(self): + data = embedded_metadata.process_image(os.path.join(os.path.dirname(__file__), + 'test-data/EXIF_IPTC_XMP_keywords_title.jpg')) + + self.assertSetEqual({'Title', 'Date', 'Tags', 'Details'}, set(data.keys())) + self.assertEqual("Blue Square Test File - .jpg", data['Title']) + self.assertEqual("2005-09-07", data['Date']) + self.assertListEqual( + [{"Name": "XMP"}, {"Name": "Blue Square"}, {"Name": "test file"}, {"Name": "Photoshop"}, {"Name": ".jpg"}, + {'Name': 'XMP keyword 1'},{'Name': 'XMP keyword 2'} + ], + data['Tags'] + ) + self.assertEqual(('XMPFiles BlueSquare test file, created in Photoshop CS2, saved as .psd, ' + '.jpg, and .tif.\n' + 'XMPFiles BlueSquare test file, created in Photoshop CS2, saved as .psd, ' + '.jpg, and .tif.\n' + '\n'), data['Details']) + + def test_EXIF_camera(self): + data = embedded_metadata.process_image(os.path.join(os.path.dirname(__file__), 'test-data/EXIF_camera.jpg')) + + self.assertSetEqual(set(data.keys()), {'Date'}) + self.assertEqual("2008-10-22", data['Date']) diff --git a/scrapers/EmbeddedMetadata/EmbeddedMetadata.yml b/scrapers/EmbeddedMetadata/EmbeddedMetadata.yml index 8fa1d171b..c5c82d364 100755 --- a/scrapers/EmbeddedMetadata/EmbeddedMetadata.yml +++ b/scrapers/EmbeddedMetadata/EmbeddedMetadata.yml @@ -4,5 +4,5 @@ imageByFragment: action: script script: - python3 - - embedded-metadata.py + - embedded_metadata.py - image-by-fragment \ No newline at end of file diff --git a/scrapers/EmbeddedMetadata/embedded-metadata.py b/scrapers/EmbeddedMetadata/embedded_metadata.py similarity index 88% rename from scrapers/EmbeddedMetadata/embedded-metadata.py rename to scrapers/EmbeddedMetadata/embedded_metadata.py index 5d8a211e3..6f2c488d4 100755 --- a/scrapers/EmbeddedMetadata/embedded-metadata.py +++ b/scrapers/EmbeddedMetadata/embedded_metadata.py @@ -1,7 +1,5 @@ from datetime import datetime -import hashlib import json -import os import sys from py_common import graphql, log @@ -10,9 +8,9 @@ # try importing config try: - import config + import config except: - config = object() + config = object() skip_ensure_requirements = config.skip_ensure_requirements if hasattr(config, 'skip_ensure_requirements') else False @@ -20,15 +18,20 @@ ensure_requirements("pyexiv2", "pyexiftool") try: - import pyexiv2 -# might fail due to old GLIBC, fall back to exiftool + import pyexiv2 +# might fail due to old GLIBC except: - try: - import exiftool - except: - log.error("You need to install the pyexiv2 or exiftool module.") - log.error("If you have pip (normally installed with python), run this command in a terminal (cmd): pip install pyexiv2 exiftool") - sys.exit() + pass + +try: + import exiftool +except: + pass + +if "pyexiv2" not in locals() and "exiftool" not in locals(): + log.error("You need to install the pyexiv2 or exiftool module.") + log.error("If you have pip (normally installed with python), run this command in a terminal (cmd): pip install pyexiv2 exiftool") + sys.exit() details_date_fields = config.details_date_fields if hasattr(config, 'details_date_fields') else False details_title_fields = config.details_title_fields if hasattr(config, 'details_title_fields') else False @@ -40,10 +43,10 @@ details_upprocessed_fields_unignored = config.details_upprocessed_fields_unignored if hasattr(config, 'details_upprocessed_fields_unignored') else [] details_ignored_labels = { - 'ExifTag', 'Orientation', 'PhotometricInterpretation', 'ResolutionUnit', 'Contrast', 'CustomRendered', 'DigitalZoomRatio', 'ExposureBiasValue', 'ExposureMode', 'ExposureProgram', 'ExposureTime', 'ExposureCompensation', + 'ExifTag', 'Orientation', 'PhotometricInterpretation', 'ResolutionUnit', 'Contrast', 'CustomRendered', 'DigitalZoomRatio', 'ExposureBiasValue', 'ExposureMode', 'ExposureProgram', 'ExposureTime', 'ExposureCompensation', 'ColorSpace', 'ComponentsConfiguration', 'CompressedBitsPerPixel', 'ExifVersion', 'FlashpixVersion', 'YCbCrPositioning', 'JPEGInterchangeFormat', 'JPEGInterchangeFormatLength', 'BaselineExposureOffset', 'FNumber', 'FileSource', 'Flash', 'FocalLength', 'FocalLengthIn35mmFilm', 'GainControl', 'ISOSpeedRatings', 'LightSource', 'MaxApertureValue', 'MeteringMode', 'Saturation', 'SceneCaptureType', 'SensingMethod', 'BaselineExposure', - 'Sharpness', 'WhiteBalance', 'ShutterSpeedValue', 'ApertureValue', 'FocalLength', 'FocalLengthIn35mmFilm', 'FocalLengthIn35mmFormat', 'ExposureTime', 'ExposureProgram', 'ExposureBiasValue', 'MaxApertureValue', 'ShutterSpeedValue', + 'Sharpness', 'WhiteBalance', 'ShutterSpeedValue', 'ApertureValue', 'FocalLength', 'FocalLengthIn35mmFilm', 'FocalLengthIn35mmFormat', 'ExposureTime', 'ExposureProgram', 'ExposureBiasValue', 'MaxApertureValue', 'ShutterSpeedValue', 'ExposureIndex', 'FocalPlaneResolutionUnit', 'FocalPlaneXResolution', 'FocalPlaneYResolution', 'AFAreaMode', 'AFPoint', 'AFPointsInFocus', 'Focus', 'FlashSetting', 'DigitalZoom', 'ISOSelection', 'ISOSpeed', 'ToneComp', 'Quality', 'RetouchHistory', 'Saturation2', 'ScanIFD', 'SceneAssist', 'SceneMode', 'Sharpening', 'ShotInfo', 'ToneComp', 'PixelXDimension', 'PixelYDimension', 'XResolution', 'YResolution', 'ExifImageWidth', 'ExifImageHeight', 'ImageHeight', 'ImageWidth', 'RelatedImageHeight', 'RelatedImageWidth', 'ImageLength', 'ByteOrder', 'Offset', @@ -54,20 +57,36 @@ details_ignored_labels.update(details_upprocessed_fields_ignored) details_ignored_labels.difference_update(details_upprocessed_fields_unignored) +def _read_data_pyexiv2(image_path: str): + data: dict[str, str] = {} + + with pyexiv2.Image(image_path) as img: + data.update(img.read_exif()) + data.update(img.read_iptc()) + data.update(img.read_xmp()) -def process_image(image_path: str): + return data + +def _read_data_exiftool(image_path: str): data: dict[str, str] = {} - - try: - with pyexiv2.Image(image_path) as img: - data.update(img.read_exif()) - data.update(img.read_iptc()) - data.update(img.read_xmp()) - except: - with exiftool.ExifToolHelper() as et: - metadata = et.get_metadata(image_path) - log.debug(f"exiftool metadata {metadata}") - data.update(metadata[0]) + + with exiftool.ExifToolHelper() as et: + metadata = et.get_metadata(image_path) + log.debug(f"exiftool metadata {metadata}") + data.update(metadata[0]) + + return data + +def process_image(image_path: str): + if hasattr(config, 'FORCE_METHOD') and config.FORCE_METHOD == 'pyexiv2': + data = _read_data_pyexiv2(image_path) + elif hasattr(config, 'FORCE_METHOD') and config.FORCE_METHOD == 'exiftool': + data = _read_data_exiftool(image_path) + else: + try: + data = _read_data_pyexiv2(image_path) + except: + data = _read_data_exiftool(image_path) ret = {} @@ -113,7 +132,7 @@ def process_image(image_path: str): date_details = '' for field in ['EXIF:CreateDate', 'Iptc.Application2.DateCreated', 'IPTC:DateCreated', 'Exif.Image.DateTime', 'Exif.Photo.DateTime', 'EXIF:DateTime', 'Iptc.Application2.DigitizationDate', 'IPTC:DigitizationDate', 'Exif.Photo.DateTimeDigitized', 'EXIF:DateTimeDigitized', - 'Exif.Image.DateTimeOriginal', 'Exif.Photo.DateTimeOriginal', 'EXIF:DateTimeOriginal']: + 'Exif.Image.DateTimeOriginal', 'Exif.Photo.DateTimeOriginal', 'EXIF:DateTimeOriginal', 'EXIF:ModifyDate']: if field not in data: continue value = data[field] @@ -148,7 +167,9 @@ def process_image(image_path: str): # title # title_details = '' - for field in ['Exif.Image.DocumentName', 'EXIF:DocumentName', 'Exif.Image.XPTitle', 'EXIF:XPTitle', 'Exif.Photo.ImageTitle', 'EXIF:ImageTitle', 'Iptc.Application2.Headline', 'IPTC:Headline','Xmp.dc.title', 'XMP:Title', 'Xmp.dc.headline', 'XMP:Headline']: + for field in ['Exif.Image.DocumentName', 'EXIF:DocumentName', 'Exif.Image.XPTitle', 'EXIF:XPTitle', 'Exif.Photo.ImageTitle', 'EXIF:ImageTitle', + 'Iptc.Application2.Headline', 'IPTC:Headline', 'Xmp.dc.title', 'XMP:Title', 'Xmp.dc.headline', 'XMP:Headline', + 'Iptc.Application2.ObjectName', 'IPTC:ObjectName']: if field not in data: continue value = data[field] @@ -209,7 +230,7 @@ def process_image(image_path: str): # # tags # - for field in ['Exif.Image.XPKeywords', 'EXIF:XPKeywords', 'Iptc.Application2.Keywords', 'IPTC:Keywords']: + for field in ['Exif.Image.XPKeywords', 'EXIF:XPKeywords', 'Iptc.Application2.Keywords', 'IPTC:Keywords', 'Xmp.pdf.Keywords', 'XMP:Keywords']: if field not in data: continue value = data[field] @@ -225,6 +246,7 @@ def process_image(image_path: str): for list_value in value: value_parts.extend(list_value.split(',')) else: + value_parts = [] value_parts.extend(value.split(',')) for value_part in value_parts: @@ -405,6 +427,6 @@ def get_imape_paths(image_id): case _: log.error(f"Not Implemented: Operation: {op}, arguments: {json.dumps(args)}") sys.exit(1) - -print(json.dumps({})) -sys.exit(1) \ No newline at end of file + + print(json.dumps({})) + sys.exit(1)