Stamp distance samples past the last GPS timestamp by the clock

README.md

@@ -240,7 +240,9 @@ description-level processing error are included in `processed_file_count`.
 Unsupported entries include their filename, category, stable reason code, and
 normalized extension. Hidden/system files and common GPS or metadata sidecars
 are ignored during directory discovery, but an explicitly supplied unsupported
-file is always reported. The report is written only to the requested local path
+file is always reported. Such a file also fails processing, with a
+`MapillaryUnsupportedFileError` entry in the image description file, whether
+or not the report is requested. The report is written only to the requested local path
 and contains absolute source filenames, so treat it as local capture metadata.
 
 ## Upload

mapillary_tools/exceptions.py

@@ -58,6 +58,13 @@ class MapillaryGeoTaggingError(MapillaryDescriptionError):
     pass
 
 
+class MapillaryUnsupportedFileError(MapillaryDescriptionError):
+    """
+    A file named for processing is neither an image nor a video that can be
+    processed
+    """
+
+
 class MapillaryVideoGPSUnusableError(MapillaryDescriptionError):
     """
     The video has no GPS track that can be used: none at all, an empty one, or

mapillary_tools/geotag/factory.py

@@ -47,6 +47,12 @@
     SourceType.CAMM: types.FileType.CAMM,
 }
 
+_VIDEO_TYPE_NAMES = {
+    types.FileType.GOPRO: "GoPro",
+    types.FileType.BLACKVUE: "BlackVue",
+    types.FileType.CAMM: "CAMM",
+}
+
 # The XML that the exiftool_xml source reads for a file, without a source path
 # or a pattern: the output of `exiftool -X` for it, named after it
 _EXIFTOOL_XML_DEFAULT_PATTERN = "%g.xml"
@@ -141,24 +147,60 @@ def parse_source_options(
         parsed_options.extend(parse_source_option(s))
 
     if geotag_source_path is not None:
-        for parsed_option in parsed_options:
-            if parsed_option.source_path is None:
-                parsed_option.source_path = SourcePathOption(
-                    source_path=Path(geotag_source_path)
-                )
-            else:
-                source_path_option = parsed_option.source_path
-                if source_path_option.source_path is None:
-                    source_path_option.source_path = Path(geotag_source_path)
-                else:
-                    LOG.warning(
-                        "The option --geotag_source_path is ignored for source %s",
-                        parsed_option,
-                    )
+        _give_source_path(parsed_options, geotag_source_path)
 
     return parsed_options
 
 
+def _give_source_path(options: T.Sequence[SourceOption], source_path: Path) -> None:
+    """
+    Give **source_path** to the options without a source path of their own,
+    keeping their patterns
+    """
+    for option in options:
+        if option.source_path is None:
+            option.source_path = SourcePathOption(source_path=Path(source_path))
+        else:
+            source_path_option = option.source_path
+            if source_path_option.source_path is None:
+                source_path_option.source_path = Path(source_path)
+            else:
+                LOG.warning(
+                    "The option --geotag_source_path is ignored for source %s",
+                    option,
+                )
+
+
+def give_video_import_path(
+    options: T.Sequence[SourceOption], video_import_path: Path
+) -> None:
+    """
+    Give the video import path of video_process to the options without a source
+    path of their own, as --geotag_source_path would. An exiftool_xml source
+    without a source path or a pattern keeps its default pattern, as in process
+    (see exiftool_xml_default_source_path)
+    """
+    for option in options:
+        if option.source is SourceType.EXIFTOOL_XML and option.source_path is None:
+            option.source_path = exiftool_xml_default_source_path(video_import_path)
+    _give_source_path(options, video_import_path)
+
+
+def exiftool_xml_default_source_path(video_import_path: Path) -> SourcePathOption:
+    """
+    Where an exiftool_xml source without a source path or a pattern finds the
+    XML of a video in video_process and sample_video: in the XML named after it
+    beside it, as in process, or else in the XMLs in the video import path,
+    matched by the paths in them
+
+    >>> exiftool_xml_default_source_path(Path("videos"))
+    SourcePathOption(pattern='%g.xml', source_path=PosixPath('videos'))
+    """
+    return SourcePathOption(
+        pattern=_EXIFTOOL_XML_DEFAULT_PATTERN, source_path=video_import_path
+    )
+
+
 def _as_video_source(option: SourceOption) -> SourceOption:
     """
     Limit a --video_geotag_source option to videos. The legacy gopro, blackvue
@@ -178,6 +220,7 @@ def _as_video_source(option: SourceOption) -> SourceOption:
         option.filetypes = types.combine_filetype_filters(
             option.filetypes, {video_type}
         )
+        option.named_video_type = video_type
     option.filetypes = types.combine_filetype_filters(
         option.filetypes, {types.FileType.VIDEO}
     )
@@ -283,6 +326,11 @@ def process(
             video_metadata_or_errors = (
                 video_geotag.to_description(video_paths) if video_geotag else []
             )
+            if option.named_video_type is not None:
+                video_metadata_or_errors = [
+                    _say_of_another_type(metadata, option.named_video_type)
+                    for metadata in video_metadata_or_errors
+                ]
         else:
             video_metadata_or_errors = []
 
@@ -356,20 +404,34 @@ def process(
         sorted(reprocessable_paths - handed_paths), filetypes
     )
 
-    if video_types is not None and (unread_paths or sourceless_videos):
+    # Files that sources were let through, but that none of them described,
+    # nor said why, which they should not do: so that every file given gets
+    # one entry, but those that --filetypes leaves out
+    described_paths = {metadata.filename for metadata in final_metadatas}
+    unhandled_images, unhandled_videos = _filter_images_and_videos(
+        sorted(
+            (reprocessable_paths & handed_paths) - described_paths - set(unread_paths)
+        ),
+        filetypes,
+    )
+
+    if video_types is not None and (
+        unread_paths or sourceless_videos or unhandled_videos
+    ):
         # Limited to some types of video, a video that no source read may not
         # be of any of them. Those of other types are left out, as the videos
         # found to be of other types
         kept_paths = set(
             _videos_not_of_other_types(
-                unread_paths + sourceless_videos,
+                unread_paths + sourceless_videos + unhandled_videos,
                 video_types,
                 # The same for every option (see process_geotag_properties)
                 num_processes=next(iter(options)).num_processes,
             )
         )
         unread_paths = [path for path in unread_paths if path in kept_paths]
         sourceless_videos = [path for path in sourceless_videos if path in kept_paths]
+        unhandled_videos = [path for path in unhandled_videos if path in kept_paths]
 
     final_metadatas.extend(
         _describe_unread_video(path, unread_videos[path]) for path in unread_paths
@@ -378,6 +440,10 @@ def process(
         _describe_sourceless_file(path, options, filetypes)
         for path in sourceless_images + sourceless_videos
     )
+    final_metadatas.extend(
+        _describe_unhandled_file(path, options)
+        for path in unhandled_images + unhandled_videos
+    )
 
     return final_metadatas
 
@@ -461,6 +527,84 @@ def _describe_sourceless_file(
     )
 
 
+def _describe_unhandled_file(
+    path: Path, options: T.Iterable[SourceOption]
+) -> types.ErrorMetadata:
+    """
+    Describe a file that the sources of **options** were let through, but that
+    none of them described, rather than leave it out of the results as if it
+    had not been there
+
+    >>> options = [SourceOption(SourceType.NATIVE), SourceOption(SourceType.EXIFTOOL_RUNTIME)]
+    >>> _describe_unhandled_file(Path("i.jpg"), options).error
+    MapillaryGeoTaggingError('The geotag sources native, exiftool_runtime gave no result for this image')
+    >>> _describe_unhandled_file(Path("v.mp4"), options[:1]).error
+    MapillaryVideoGPSNotFoundError('The geotag source native gave no result for this video')
+    """
+    names = list(dict.fromkeys(option.source.value for option in options))
+    sources = (
+        f"The geotag sources {', '.join(names)}"
+        if len(names) > 1
+        else f"The geotag source {names[0]}"
+    )
+    if utils.is_image_file(path):
+        return types.describe_error_metadata(
+            exceptions.MapillaryGeoTaggingError(
+                f"{sources} gave no result for this image"
+            ),
+            path,
+            filetype=types.FileType.IMAGE,
+        )
+    return types.describe_error_metadata(
+        exceptions.MapillaryVideoGPSNotFoundError(
+            f"{sources} gave no result for this video"
+        ),
+        path,
+        filetype=types.FileType.VIDEO,
+    )
+
+
+def _say_of_another_type(
+    metadata: types.VideoMetadataOrError, video_type: types.FileType
+) -> types.VideoMetadataOrError:
+    """
+    Say that a video in which the native reader, limited to the videos of
+    **video_type** by the name of the source, finds no GPS is of another
+    type, rather than that it has no GPS. The error stays the one the native
+    reader reports, which tells process() what to do with the video
+
+    >>> not_found = types.describe_error_metadata(
+    ...     exceptions.MapillaryVideoGPSNotFoundError("No GPS data found from the video"),
+    ...     Path("v.mp4"),
+    ...     filetype=types.FileType.VIDEO,
+    ... )
+    >>> said = _say_of_another_type(not_found, types.FileType.GOPRO)
+    >>> type(said.error).__name__, str(said.error)
+    ('MapillaryVideoGPSNotFoundError', 'Not a GoPro video: the geotag source gopro reads GoPro videos only')
+    >>> noisy = types.describe_error_metadata(
+    ...     exceptions.MapillaryGPSNoiseError("GPS is too noisy"),
+    ...     Path("v.mp4"),
+    ...     filetype=types.FileType.VIDEO,
+    ... )
+    >>> _say_of_another_type(noisy, types.FileType.GOPRO) is noisy
+    True
+    """
+    if not (
+        isinstance(metadata, types.ErrorMetadata)
+        and type(metadata.error) is exceptions.MapillaryVideoGPSNotFoundError
+    ):
+        return metadata
+
+    name = _VIDEO_TYPE_NAMES[video_type]
+    return types.describe_error_metadata(
+        exceptions.MapillaryVideoGPSNotFoundError(
+            f"Not a {name} video: the geotag source {video_type.value} reads {name} videos only"
+        ),
+        metadata.filename,
+        filetype=metadata.filetype,
+    )
+
+
 def _describe_unread_video(
     video_path: Path, sources: T.Sequence[SourceType]
 ) -> types.ErrorMetadata:

mapillary_tools/geotag/geotag_images_from_gpx.py

@@ -28,6 +28,75 @@
 LOG = logging.getLogger(__name__)
 
 
+def check_within_track(
+    time: float,
+    sorted_points: T.Sequence[geo.Point],
+    gaps: T.Sequence[tuple[geo.Point, geo.Point]] = (),
+) -> None:
+    """
+    Raise MapillaryOutsideGPXTrackError if **time** falls outside the track of
+    **sorted_points**, with a tolerance of 1 ms: before its start, after its
+    end, or in one of its **gaps** (see track_gaps())
+
+    >>> track = [geo.Point(time=t, lat=0, lon=0, alt=None, angle=None) for t in (10, 20, 9000)]
+    >>> check_within_track(15.0, track)
+    >>> check_within_track(9000.0005, track)
+    >>> check_within_track(9.5, track)
+    Traceback (most recent call last):
+    ...
+    mapillary_tools.exceptions.MapillaryOutsideGPXTrackError: The image date time is 0.5 seconds behind the GPX start point
+    >>> check_within_track(4000.0, track, track_gaps(track, 3600))
+    Traceback (most recent call last):
+    ...
+    mapillary_tools.exceptions.MapillaryOutsideGPXTrackError: The GPX track has a gap of 2.5 hours, from 1970-01-01T00:00:20+00:00 at (0, 0) to 1970-01-01T02:30:00+00:00 at (0, 0), which the image time 1970-01-01T01:06:40+00:00 falls in
+    """
+    assert sorted_points, "must have at least one point"
+
+    if time < sorted_points[0].time:
+        delta = sorted_points[0].time - time
+        gpx_start_time = build_capture_time(sorted_points[0].time)
+        gpx_end_time = build_capture_time(sorted_points[-1].time)
+        # with the tolerance of 1ms
+        if 0.001 < delta:
+            raise exceptions.MapillaryOutsideGPXTrackError(
+                f"The image date time is {round(delta, 3)} seconds behind the GPX start point",
+                image_time=build_capture_time(time),
+                gpx_start_time=gpx_start_time,
+                gpx_end_time=gpx_end_time,
+            )
+
+    if sorted_points[-1].time < time:
+        delta = time - sorted_points[-1].time
+        gpx_start_time = build_capture_time(sorted_points[0].time)
+        gpx_end_time = build_capture_time(sorted_points[-1].time)
+        # with the tolerance of 1ms
+        if 0.001 < delta:
+            raise exceptions.MapillaryOutsideGPXTrackError(
+                f"The image time is {round(delta, 3)} seconds beyond the GPX end point",
+                image_time=build_capture_time(time),
+                gpx_start_time=gpx_start_time,
+                gpx_end_time=gpx_end_time,
+            )
+
+    # The last gap that starts at or before the image
+    idx = bisect.bisect_right([before.time for before, _ in gaps], time)
+    if idx:
+        before, after = gaps[idx - 1]
+        delta = min(time - before.time, after.time - time)
+        # with the tolerance of 1ms, as at the ends of the track
+        if 0.001 < delta:
+            captured_at = datetime.datetime.fromtimestamp(
+                time, tz=datetime.timezone.utc
+            ).isoformat()
+            track_gap = describe_gap(before, after, before.time, after.time)
+            raise exceptions.MapillaryOutsideGPXTrackError(
+                f"The GPX track has {track_gap}, which the image time {captured_at} falls in",
+                image_time=build_capture_time(time),
+                gpx_start_time=build_capture_time(sorted_points[0].time),
+                gpx_end_time=build_capture_time(sorted_points[-1].time),
+            )
+
+
 class GeotagImagesFromGPX(GeotagImagesFromGeneric):
     def __init__(
         self,
@@ -55,55 +124,7 @@ def _interpolate_image_metadata_along(
         sorted_points: T.Sequence[geo.Point],
         gaps: T.Sequence[tuple[geo.Point, geo.Point]] = (),
     ) -> types.ImageMetadata:
-        assert sorted_points, "must have at least one point"
-
-        if image_metadata.time < sorted_points[0].time:
-            delta = sorted_points[0].time - image_metadata.time
-            gpx_start_time = build_capture_time(sorted_points[0].time)
-            gpx_end_time = build_capture_time(sorted_points[-1].time)
-            # with the tolerance of 1ms
-            if 0.001 < delta:
-                raise exceptions.MapillaryOutsideGPXTrackError(
-                    f"The image date time is {round(delta, 3)} seconds behind the GPX start point",
-                    image_time=build_capture_time(image_metadata.time),
-                    gpx_start_time=gpx_start_time,
-                    gpx_end_time=gpx_end_time,
-                )
-
-        if sorted_points[-1].time < image_metadata.time:
-            delta = image_metadata.time - sorted_points[-1].time
-            gpx_start_time = build_capture_time(sorted_points[0].time)
-            gpx_end_time = build_capture_time(sorted_points[-1].time)
-            # with the tolerance of 1ms
-            if 0.001 < delta:
-                raise exceptions.MapillaryOutsideGPXTrackError(
-                    f"The image time is {round(delta, 3)} seconds beyond the GPX end point",
-                    image_time=build_capture_time(image_metadata.time),
-                    gpx_start_time=gpx_start_time,
-                    gpx_end_time=gpx_end_time,
-                )
-
-        # The last gap that starts at or before the image
-        idx = bisect.bisect_right(
-            [before.time for before, _ in gaps], image_metadata.time
-        )
-        if idx:
-            before, after = gaps[idx - 1]
-            delta = min(
-                image_metadata.time - before.time, after.time - image_metadata.time
-            )
-            # with the tolerance of 1ms, as at the ends of the track
-            if 0.001 < delta:
-                captured_at = datetime.datetime.fromtimestamp(
-                    image_metadata.time, tz=datetime.timezone.utc
-                ).isoformat()
-                track_gap = describe_gap(before, after, before.time, after.time)
-                raise exceptions.MapillaryOutsideGPXTrackError(
-                    f"The GPX track has {track_gap}, which the image time {captured_at} falls in",
-                    image_time=build_capture_time(image_metadata.time),
-                    gpx_start_time=build_capture_time(sorted_points[0].time),
-                    gpx_end_time=build_capture_time(sorted_points[-1].time),
-                )
+        check_within_track(image_metadata.time, sorted_points, gaps)
 
         interpolated = geo.interpolate(sorted_points, image_metadata.time)