diff --git a/README.md b/README.md index d791e4690..dc1534441 100644 --- a/README.md +++ b/README.md @@ -327,7 +327,10 @@ mapillary_tools upload MY_SAMPLES --desc_path=/tmp/mapillary_description.json If you use external GPS devices for mapping, you will need to geotag your captures with the external GPS tracks. To geotag images with a GPX file, the capture time (extracted from EXIF tag "Date/Time Original" or "GPS Date/Time") is minimally required. -It is used to locate the images along the GPS tracks. +It is used to locate the images along the GPS tracks. Images captured before the track starts, after it ends, or in a +gap of more than an hour between two of its points (such as between two trips in one GPX file) fall outside the track, +and are not geotagged from it. The same applies to videos. To allow longer gaps, set the environment variable +`MAPILLARY_TOOLS_MAX_GPX_GAP_SECONDS` (3600 by default). ```sh mapillary_tools process MY_IMAGE_DIR --geotag_source "gpx" --geotag_source_path MY_EXTERNAL_GPS.gpx @@ -462,6 +465,8 @@ with a GPX file called `foo.gpx`. ```sh mapillary_tools process --video_geotag_source gpx VIDEO_DIR/ ``` +NMEA files work the same way with `--video_geotag_source nmea`, which associates `foo.mp4` with `foo.nmea`. What +follows about GPX tracks applies to NMEA tracks too. #### Insta360 stitched videos diff --git a/mapillary_tools/commands/video_process.py b/mapillary_tools/commands/video_process.py index d127b9b98..c6896cd43 100644 --- a/mapillary_tools/commands/video_process.py +++ b/mapillary_tools/commands/video_process.py @@ -6,8 +6,10 @@ from __future__ import annotations import logging +from pathlib import Path -from .. import exceptions +from .. import constants, exceptions +from ..sample_video import find_sample_dirs from ..types import FileType from .process import Command as ProcessCommand from .sample_video import Command as SampleCommand @@ -43,7 +45,39 @@ def run(self, args: dict): ) args[option] = {FileType.IMAGE} + if args["skip_subfolders"]: + _add_sample_dirs(args) + ProcessCommand().run(args) if sample_error is not None: raise sample_error + + +def _add_sample_dirs(args: dict) -> None: + """ + Add the folders that the videos were sampled into to the import paths. + --skip_subfolders chose the videos to sample, and hides the subfolders of + the import path from processing (and from uploading in + video_process_and_upload), which the samples of each video are in + """ + import_path: Path = args["import_path"] + sample_dirs = [ + sample_dir + for sample_dir in find_sample_dirs( + args["video_import_path"], import_path, skip_subfolders=True + ) + # Not a video that failed to sample + if sample_dir.is_dir() + ] + if not sample_dirs: + return + + if args.get("desc_path") is None: + # Where process writes it for the import path alone, rather than to + # STDOUT, as for several import paths + args["desc_path"] = str( + import_path.joinpath(constants.IMAGE_DESCRIPTION_FILENAME) + ) + + args["import_path"] = [import_path, *sample_dirs] diff --git a/mapillary_tools/constants.py b/mapillary_tools/constants.py index cab9e9b1a..d7dfdf89f 100644 --- a/mapillary_tools/constants.py +++ b/mapillary_tools/constants.py @@ -82,6 +82,11 @@ def _parse_scaled_integers( ################### USER_DATA_DIR = appdirs.user_data_dir(appname="mapillary_tools", appauthor="Mapillary") PROMPT_DISABLED: bool = _yes_or_no(os.getenv(_ENV_PREFIX + "PROMPT_DISABLED", "NO")) +# In seconds. Consecutive points of a GPX or NMEA track further apart than +# this leave a gap that no position is interpolated across, such as a GPS +# logger switched off between trips: the time inside it is outside the track, +# like the time before its start or after its end +MAX_GPX_GAP_SECONDS = float(os.getenv(_ENV_PREFIX + "MAX_GPX_GAP_SECONDS", 3600)) ############################ diff --git a/mapillary_tools/exif_read.py b/mapillary_tools/exif_read.py index 8a5dd3b8c..65189bd17 100644 --- a/mapillary_tools/exif_read.py +++ b/mapillary_tools/exif_read.py @@ -45,6 +45,18 @@ def eval_frac(value: Ratio) -> float: def gps_to_decimal(values: tuple[Ratio, Ratio, Ratio]) -> float | None: + """ + Convert degrees, minutes and seconds to decimal degrees, or None if they + are not a coordinate: a GPS IFD that a camera wrote at bad offsets reads + as minutes or seconds of more than 60 (60 itself may be 59.996 rounded) + + >>> gps_to_decimal((Ratio(10, 1), Ratio(22, 1), Ratio(30, 1))) + 10.375 + >>> round(gps_to_decimal((Ratio(10, 1), Ratio(22, 1), Ratio(60, 1))), 6) + 10.383333 + >>> gps_to_decimal((Ratio(0, 1), Ratio(1, 4), Ratio(381, 5))) is None + True + """ try: deg, min, sec, *_ = values except (TypeError, ValueError): @@ -61,6 +73,8 @@ def gps_to_decimal(values: tuple[Ratio, Ratio, Ratio]) -> float | None: seconds = eval_frac(sec) except ZeroDivisionError: return None + if not (0 <= minutes <= 60 and 0 <= seconds <= 60): + return None return degrees + minutes / 60 + seconds / 3600 diff --git a/mapillary_tools/exiftool_read_video.py b/mapillary_tools/exiftool_read_video.py index f666944ae..178849b7d 100644 --- a/mapillary_tools/exiftool_read_video.py +++ b/mapillary_tools/exiftool_read_video.py @@ -30,6 +30,7 @@ "XMP-aux": "http://ns.exiftool.org/XMP/XMP-aux/1.0/", "DJI": "http://ns.exiftool.org/MakerNotes/DJI/1.0/", "XMP-drone-dji": "http://ns.exiftool.org/XMP/XMP-drone-dji/1.0/", + "LIGO": "http://ns.exiftool.org/QuickTime/LIGO/1.0/", **{ f"Track{track_id}": f"http://ns.exiftool.org/QuickTime/Track{track_id}/1.0/" for track_id in range(1, MAX_TRACK_ID + 1) @@ -310,6 +311,121 @@ def _aggregate_gps_track( return track +def _group_gps_records( + elements: T.Iterable[ET.Element], + tags: T.Collection[str], + record_start_tag: str | None = None, + first_tag: str | None = None, +) -> T.Generator[dict[str, str], None, None]: + """ + Group the tags that ExifTool lists for one GPS record after another, one + dict per record from tag (expanded) to text. A record starts at + **record_start_tag**, at **first_tag**, which ExifTool lists first in a + record that has it, or at a tag that the record so far already has. + + A tag that a record does not have is left out of its dict, rather than + taken from a later record: a BlackVue writes no GPSTrack while stationary. + + >>> elements = [] + >>> for tag, text in [ + ... ("SampleTime", "0"), ("GPSLatitude", "1"), ("GPSTrack", "90"), + ... ("SampleTime", "1"), ("GPSLatitude", "2"), + ... ("GPSLatitude", "3"), ("GPSTrack", "180"), + ... ]: + ... elements.append(ET.Element(expand_tag(f"QuickTime:{tag}"))) + ... elements[-1].text = text + >>> records = _group_gps_records( + ... elements, + ... ["QuickTime:GPSLatitude", "QuickTime:GPSTrack"], + ... record_start_tag="QuickTime:SampleTime", + ... ) + >>> [sorted(record.values()) for record in records] + [['1', '90'], ['2'], ['180', '3']] + """ + expanded_tags = {expand_tag(tag) for tag in tags} + expanded_record_start_tag = ( + expand_tag(record_start_tag) if record_start_tag is not None else None + ) + expanded_first_tag = expand_tag(first_tag) if first_tag is not None else None + + record: dict[str, str] = {} + for element in elements: + if element.tag == expanded_record_start_tag: + if record: + yield record + record = {} + elif element.tag in expanded_tags and element.text is not None: + if record and (element.tag in record or element.tag == expanded_first_tag): + yield record + record = {} + record[element.tag] = element.text + if record: + yield record + + +def _aggregate_gps_track_by_record( + elements: T.Iterable[ET.Element], + time_tag: str, + lon_tag: str, + lat_tag: str, + alt_tag: str, + direction_tag: str, + record_start_tag: str | None = None, +) -> list[GPSPoint]: + """ + Aggregate the GPS records that ExifTool lists one after another, each with + its GPS time in **time_tag**, into a track whose times start at 0. Every + point takes its altitude and direction from its own record. + """ + expanded_time_tag = expand_tag(time_tag) + expanded_lon_tag = expand_tag(lon_tag) + expanded_lat_tag = expand_tag(lat_tag) + expanded_alt_tag = expand_tag(alt_tag) + expanded_direction_tag = expand_tag(direction_tag) + + track: list[GPSPoint] = [] + for record in _group_gps_records( + elements, + [time_tag, lon_tag, lat_tag, alt_tag, direction_tag], + record_start_tag=record_start_tag, + first_tag=time_tag, + ): + time_text = record.get(expanded_time_tag) + dt = exif_read.parse_gps_datetime(time_text) if time_text is not None else None + timestamp = geo.as_unix_time(dt) if dt is not None else None + lon = _maybe_float(record.get(expanded_lon_tag)) + lat = _maybe_float(record.get(expanded_lat_tag)) + if timestamp is None or lon is None or lat is None: + continue + + track.append( + GPSPoint( + time=timestamp, + lon=lon, + lat=lat, + alt=_maybe_float(record.get(expanded_alt_tag)), + angle=_maybe_float(record.get(expanded_direction_tag)), + epoch_time=timestamp, + fix=None, + precision=None, + ground_speed=None, + ) + ) + + track.sort(key=lambda point: point.time) + + # A BlackVue video holds a copy of its GPS log in the low resolution + # recording embedded in it, which ExifTool lists as well + track = _deduplicate_gps_points(track, same_gps_point=_same_gps_point) + + if track: + first_time = track[0].time + for point in track: + point.time = point.time - first_time + + return track + + def _aggregate_samples( elements: T.Iterable[ET.Element], sample_time_tag: str, @@ -436,6 +552,12 @@ def extract_gps_track(self) -> list[geo.Point]: if track_with_fix: return T.cast(T.List[geo.Point], track_with_fix) + # dashcams that write LIGOGPSINFO (e.g. GKU, Abask), which ExifTool + # decrypts. Its GPS times are in UTC, unlike the clocks of these cameras + track_with_fix = self._extract_gps_track_from_quicktime(namespace="LIGO") + if track_with_fix: + return T.cast(T.List[geo.Point], track_with_fix) + return [] def _extract_make_and_model(self) -> tuple[str | None, str | None]: @@ -585,12 +707,16 @@ def _extract_gps_track_from_quicktime( ): return [] - return _aggregate_gps_track( - self._texts_by_tag, + root = self.etree.getroot() + if root is None: + raise ValueError("ElementTree root is None") + + return _aggregate_gps_track_by_record( + root, time_tag=f"{namespace}:GPSDateTime", lon_tag=f"{namespace}:GPSLongitude", lat_tag=f"{namespace}:GPSLatitude", alt_tag=f"{namespace}:GPSAltitude", - gps_time_tag=f"{namespace}:GPSDateTime", direction_tag=f"{namespace}:GPSTrack", + record_start_tag=f"{namespace}:SampleTime", ) diff --git a/mapillary_tools/ffmpeg.py b/mapillary_tools/ffmpeg.py index c75e2abf6..30b5fa544 100644 --- a/mapillary_tools/ffmpeg.py +++ b/mapillary_tools/ffmpeg.py @@ -54,6 +54,7 @@ class Stream(T.TypedDict): r_frame_rate: str avg_frame_rate: str nb_frames: str + sample_aspect_ratio: str class Format(T.TypedDict, total=False): @@ -199,6 +200,7 @@ def extract_frames_by_interval( sample_dir: Path, sample_interval: float, stream_specifier: int | str = "v", + sample_aspect_ratio: str | None = None, ) -> None: """ Extract frames from video at regular time intervals using fps filter. @@ -210,6 +212,8 @@ def extract_frames_by_interval( stream_specifier: Stream specifier to target specific stream(s). Can be an integer (stream index) or "v" (all video streams) See https://ffmpeg.org/ffmpeg.html#Stream-specifiers-1 + sample_aspect_ratio: The sample aspect ratio of the stream, as ffprobe + reports it (e.g. "4:3"). See _scale_to_square_pixels Raises: FFmpegNotFoundError: If ffmpeg binary is not found @@ -229,7 +233,15 @@ def extract_frames_by_interval( # Select stream *stream_selector, # Filter videos - *["-vf", f"fps=1/{sample_interval}"], + *[ + "-vf", + ",".join( + [ + f"fps=1/{sample_interval}", + *self._scale_to_square_pixels(sample_aspect_ratio), + ] + ), + ], # Video quality level (or the alias -q:v) # -q:v=1 is the best quality but larger image sizes # see https://stackoverflow.com/a/10234065 @@ -286,6 +298,7 @@ def extract_specified_frames( sample_dir: Path, frame_indices: set[int], stream_specifier: int | str = "v", + sample_aspect_ratio: str | None = None, ) -> None: """ Extract specific frames from video by frame number using select filter. @@ -300,6 +313,8 @@ def extract_specified_frames( stream_specifier: Stream specifier to target specific stream(s). Can be an integer (stream index) or "v" (all video streams) See https://ffmpeg.org/ffmpeg.html#Stream-specifiers-1 + sample_aspect_ratio: The sample aspect ratio of the stream, as ffprobe + reports it (e.g. "4:3"). See _scale_to_square_pixels Raises: FFmpegNotFoundError: If ffmpeg binary is not found @@ -333,7 +348,14 @@ def extract_specified_frames( # https://devblogs.microsoft.com/oldnewthing/20031210-00/?p=41553 with tempfile.NamedTemporaryFile(mode="w+", delete=delete) as select_file: try: - select_file.write(f"select={eqs}") + select_file.write( + ",".join( + [ + f"select={eqs}", + *self._scale_to_square_pixels(sample_aspect_ratio), + ] + ) + ) select_file.flush() # If not close, error "The process cannot access the file because it is being used by another process" if not delete: @@ -463,6 +485,48 @@ def iterate_samples( stream_specifier, frame_idx = result yield (stream_specifier, frame_idx, sample_path) + @classmethod + def _scale_to_square_pixels(cls, sample_aspect_ratio: str | None) -> list[str]: + """ + The filters that scale the frames of a stream with non-square pixels + to square ones, so that they are written at the aspect ratio the + video is displayed at. A JPEG keeps only a density hint for the + pixel shape, which most viewers ignore. The frames are stretched + along one side, never shrunk along the other, so no detail is lost. + + Frames of a stream with square pixels, or whose pixel shape is + unknown ("0:1" or not reported), are written as they are decoded. + + Examples: + >>> FFMPEG._scale_to_square_pixels("4:3") + ['scale=w=if(gt(sar\\\\,1)\\\\,round(iw*sar/2)*2\\\\,iw):h=if(lt(sar\\\\,1)\\\\,round(ih/sar/2)*2\\\\,ih)', 'setsar=1'] + >>> FFMPEG._scale_to_square_pixels("1:1") + [] + >>> FFMPEG._scale_to_square_pixels("0:1") + [] + >>> FFMPEG._scale_to_square_pixels(None) + [] + """ + if not sample_aspect_ratio: + return [] + + try: + num, den = (int(x) for x in sample_aspect_ratio.split(":")) + except ValueError: + return [] + + if num <= 0 or den <= 0 or num == den: + return [] + + # sar is that of the frames as decoded, which ffmpeg may have rotated + # since the stream was probed. The stretched side is rounded to an + # even size, as 4:2:0 chroma subsampling halves it + return [ + "scale=w=if(gt(sar\\,1)\\,round(iw*sar/2)*2\\,iw)" + ":h=if(lt(sar\\,1)\\,round(ih/sar/2)*2\\,ih)", + "setsar=1", + ] + def _read_filter_from_file_args(self, filter_path: str) -> list[str]: """ Arguments that apply a video filter graph read from a file. diff --git a/mapillary_tools/geotag/factory.py b/mapillary_tools/geotag/factory.py index cb6ede822..bb7d20b3b 100644 --- a/mapillary_tools/geotag/factory.py +++ b/mapillary_tools/geotag/factory.py @@ -22,6 +22,7 @@ geotag_images_from_video, geotag_videos_from_exiftool, geotag_videos_from_gpx, + geotag_videos_from_nmea, geotag_videos_from_video, ) from .options import ( @@ -38,8 +39,7 @@ _ALL_VIDEO_TYPES = {types.FileType.VIDEO, *types.NATIVE_VIDEO_FILETYPES} # Sources that read a GPS track from outside the video file, as opposed to -# re-reading the telemetry embedded in it. (NMEA cannot geotag videos yet: it -# skips them, and they keep the error they had) +# re-reading the telemetry embedded in it EXTERNAL_GPS_SOURCES = frozenset({SourceType.GPX, SourceType.NMEA}) @@ -146,6 +146,8 @@ def can_geotag_videos(option: SourceOption) -> bool: Examples: >>> can_geotag_videos(SourceOption(SourceType.GPX)) True + >>> can_geotag_videos(SourceOption(SourceType.NMEA)) + True >>> can_geotag_videos(SourceOption(SourceType.EXIF)) False >>> can_geotag_videos( @@ -163,10 +165,14 @@ def process( # Collection: ABC for sized iterable container classes paths: T.Iterable[Path], options: T.Collection[SourceOption], + # --filetypes, which the filetypes of every option are limited to already + filetypes: set[types.FileType] | None = None, ) -> list[types.MetadataOrError]: if not options: raise ValueError("No geotag options provided") + video_types = _video_types_asked_for(filetypes) + final_metadatas: list[types.MetadataOrError] = [] # Paths (image path or video path) that will be sent to the next geotag process @@ -234,7 +240,20 @@ def process( ] = [] for metadata in image_metadata_or_errors + video_metadata_or_errors: - if isinstance(metadata, types.ErrorMetadata) and _is_reprocessable( + if video_types is not None and _is_of_other_type( + metadata, option, video_types + ): + # Its type, known only once it is read, is not one asked for: + # leave it out, as the files of other kinds + LOG.debug( + "Skipping %s, which is not of the file types %s", + metadata.filename, + filetypes, + ) + reprocessable_paths.remove(metadata.filename) + pending_errors.pop(metadata.filename, None) + gpx_rejections.pop(metadata.filename, None) + elif isinstance(metadata, types.ErrorMetadata) and _is_reprocessable( metadata ): # Leave what it is for the next geotag process @@ -272,11 +291,14 @@ def process( break final_metadatas.extend(pending_errors.values()) - final_metadatas.extend( - _describe_unread_video(path, sources) - for path, sources in unread_videos.items() - if path in reprocessable_paths and path not in pending_errors - ) + # Limited to some types of video, a video that no source read may not be + # of any of them, and is left out as the videos found to be of other types + if video_types is None: + final_metadatas.extend( + _describe_unread_video(path, sources) + for path, sources in unread_videos.items() + if path in reprocessable_paths and path not in pending_errors + ) return final_metadatas @@ -342,6 +364,81 @@ def _warn_geotagged_despite_gpx( LOG.debug("Images outside %s: %s", gpx_track, [p.name for p in image_paths]) +def _video_types_asked_for( + filetypes: set[types.FileType] | None, +) -> set[types.FileType] | None: + """ + The types of video that --filetypes limits processing to, or None if it + lets videos of every type through + + >>> _video_types_asked_for({types.FileType.IMAGE, types.FileType.GOPRO}) + {} + >>> _video_types_asked_for({types.FileType.GOPRO, types.FileType.VIDEO}) is None + True + >>> _video_types_asked_for(None) is None + True + """ + if filetypes is None or types.FileType.VIDEO in filetypes: + return None + return filetypes & types.NATIVE_VIDEO_FILETYPES + + +def _is_of_other_type( + metadata: types.MetadataOrError, + option: SourceOption, + video_types: set[types.FileType], +) -> bool: + """ + Whether the source **option** found a video to be of none of the types + **video_types** asked for, which _filter_images_and_videos() cannot tell by + its name: it read the video as another type, or it is the native reader, + which tries only the readers of the types it is given, and finds no GPS + data only in a video that none of them recognizes. A generic video is of a + type that is not known, and is kept. + + >>> gopro = {types.FileType.GOPRO} + >>> video = types.VideoMetadata(Path("v.mp4"), types.FileType.VIDEO, points=[]) + >>> _is_of_other_type(video, SourceOption(SourceType.GPX), gopro) + False + >>> blackvue = types.VideoMetadata(Path("v.mp4"), types.FileType.BLACKVUE, points=[]) + >>> _is_of_other_type(blackvue, SourceOption(SourceType.EXIFTOOL_RUNTIME), gopro) + True + >>> not_found = types.describe_error_metadata( + ... exceptions.MapillaryVideoGPSNotFoundError("No GPS data found from the video"), + ... Path("v.mp4"), + ... filetype=types.FileType.VIDEO, + ... ) + >>> _is_of_other_type(not_found, SourceOption(SourceType.NATIVE, filetypes=gopro), gopro) + True + + Limited further by its own filetypes, a source found only that the video is + of none of those, and leaves it to the next source: + + >>> _is_of_other_type( + ... not_found, + ... SourceOption(SourceType.NATIVE, filetypes=gopro), + ... {types.FileType.GOPRO, types.FileType.BLACKVUE}, + ... ) + False + """ + if isinstance(metadata, types.VideoMetadata): + return ( + metadata.filetype in types.NATIVE_VIDEO_FILETYPES + and metadata.filetype not in video_types + ) + + if isinstance(metadata, types.ErrorMetadata): + return ( + option.source is SourceType.NATIVE + and isinstance(metadata.error, exceptions.MapillaryVideoGPSNotFoundError) + and option.filetypes is not None + and types.FileType.VIDEO not in option.filetypes + and video_types <= option.filetypes + ) + + return False + + def _is_reprocessable(metadata: types.MetadataOrError) -> bool: if isinstance(metadata, types.ErrorMetadata): if isinstance( @@ -478,8 +575,9 @@ def _build_video_geotag(option: SourceOption) -> base.GeotagVideosFromGeneric | 'GeotagVideosFromGPX' >>> opt = SourceOption(SourceType.NMEA, source_path=Path("/tmp/test.nmea")) - >>> _build_video_geotag(opt) is None - True + >>> geotagger = _build_video_geotag(opt) + >>> geotagger.__class__.__name__ + 'GeotagVideosFromNMEA' >>> opt = SourceOption(SourceType.EXIF) >>> _build_video_geotag(opt) is None @@ -523,8 +621,12 @@ def _build_video_geotag(option: SourceOption) -> base.GeotagVideosFromGeneric | ) elif option.source is SourceType.NMEA: - # TODO: geotag videos from NMEA - return None + return geotag_videos_from_nmea.GeotagVideosFromNMEA( + source_path=option.source_path, + num_processes=option.num_processes, + video_sync=option.video_sync, + interpolation=option.interpolation, + ) elif option.source is SourceType.EXIF: # Legacy image-specific geotag types diff --git a/mapillary_tools/geotag/geotag_images_from_gpx.py b/mapillary_tools/geotag/geotag_images_from_gpx.py index f06c1d1f7..ef9fdca73 100644 --- a/mapillary_tools/geotag/geotag_images_from_gpx.py +++ b/mapillary_tools/geotag/geotag_images_from_gpx.py @@ -5,7 +5,9 @@ from __future__ import annotations +import bisect import dataclasses +import datetime import logging import sys import typing as T @@ -16,10 +18,11 @@ else: from typing_extensions import override -from .. import exceptions, geo, types +from .. import constants, exceptions, geo, types from ..serializer.description import build_capture_time from .base import GeotagImagesFromGeneric from .geotag_images_from_exif import ImageEXIFExtractor +from .video_extractors.gpx import describe_gap, track_gaps LOG = logging.getLogger(__name__) @@ -33,12 +36,16 @@ def __init__( use_image_start_time: bool = False, offset_time: float = 0.0, num_processes: int | None = None, + # Whether an image in a gap of the track longer than + # constants.MAX_GPX_GAP_SECONDS is outside it + check_gaps: bool = True, ): super().__init__(num_processes=num_processes) self.points = points self.use_gpx_start_time = use_gpx_start_time self.use_image_start_time = use_image_start_time self.offset_time = offset_time + self.check_gaps = check_gaps # Why the track could not be read, reported for every image instead self.track_error: Exception | None = None @@ -46,6 +53,7 @@ def _interpolate_image_metadata_along( self, image_metadata: types.ImageMetadata, sorted_points: T.Sequence[geo.Point], + gaps: T.Sequence[tuple[geo.Point, geo.Point]] = (), ) -> types.ImageMetadata: assert sorted_points, "must have at least one point" @@ -75,6 +83,28 @@ def _interpolate_image_metadata_along( gpx_end_time=gpx_end_time, ) + # The last gap that starts at or before the image + idx = bisect.bisect_right( + [before.time for before, _ in gaps], image_metadata.time + ) + if idx: + before, after = gaps[idx - 1] + delta = min( + image_metadata.time - before.time, after.time - image_metadata.time + ) + # with the tolerance of 1ms, as at the ends of the track + if 0.001 < delta: + captured_at = datetime.datetime.fromtimestamp( + image_metadata.time, tz=datetime.timezone.utc + ).isoformat() + track_gap = describe_gap(before, after, before.time, after.time) + raise exceptions.MapillaryOutsideGPXTrackError( + f"The GPX track has {track_gap}, which the image time {captured_at} falls in", + image_time=build_capture_time(image_metadata.time), + gpx_start_time=build_capture_time(sorted_points[0].time), + gpx_end_time=build_capture_time(sorted_points[-1].time), + ) + interpolated = geo.interpolate(sorted_points, image_metadata.time) return dataclasses.replace( @@ -158,11 +188,17 @@ def to_description( # TODO: this time modification seems to affect final capture times image_metadata.time += image_time_offset + gaps = ( + track_gaps(sorted_points, constants.MAX_GPX_GAP_SECONDS) + if self.check_gaps + else [] + ) + for image_metadata in sorted_image_metadatas: try: final_metadatas.append( self._interpolate_image_metadata_along( - image_metadata, sorted_points + image_metadata, sorted_points, gaps ) ) except exceptions.MapillaryOutsideGPXTrackError as ex: diff --git a/mapillary_tools/geotag/geotag_images_from_nmea_file.py b/mapillary_tools/geotag/geotag_images_from_nmea_file.py index cb4ef27aa..eb779c736 100644 --- a/mapillary_tools/geotag/geotag_images_from_nmea_file.py +++ b/mapillary_tools/geotag/geotag_images_from_nmea_file.py @@ -6,6 +6,8 @@ from __future__ import annotations import datetime +import logging +import re from pathlib import Path import pynmea2 @@ -14,6 +16,13 @@ from .geotag_images_from_gpx import GeotagImagesFromGPX +LOG = logging.getLogger(__name__) + +# An RMC or a GGA sentence, from any talker: GP (GPS), GN (several systems), +# GL (GLONASS), GA (Galileo), ... +_SENTENCE_RE = re.compile(r"\$[A-Z]{2}(RMC|GGA),") + + class GeotagImagesFromNMEAFile(GeotagImagesFromGPX): def __init__( self, @@ -66,33 +75,69 @@ def _read_nmea_points(source_path: Path) -> list[geo.Point]: def get_lat_lon_time_from_nmea(nmea_file: Path) -> list[geo.Point]: - with nmea_file.open("r") as f: + """ + Read the fixes of an NMEA file: the time and position of each GGA + sentence, on the date of the last RMC sentence before it, or of the first + RMC sentence for the GGA sentences before any. + + Receivers log sentences without a fix, such as before they acquire one: + RMC sentences that are void or have no date or time, and GGA sentences + without a time or a fix, which give their position as 0, 0. They are + skipped. A file without the date of any fix gives no points. + + A receiver may log a fix from several talkers, such as $GPGGA and $GNGGA, + for the same time: only the first is read, so that no two points share a + time. + """ + # NMEA sentences are ASCII. Decode them as UTF-8 on every platform, so + # that a file that is not text fails to read, not only where UTF-8 is the + # default encoding + with nmea_file.open("r", encoding="utf-8") as f: lines = f.readlines() lines = [line.rstrip("\n\r") for line in lines] + messages = [pynmea2.parse(line) for line in lines if _SENTENCE_RE.search(line)] + # Get initial date - for line in lines: - if "GPRMC" in line: - data = pynmea2.parse(line) - date = data.datetime.date() - break + dates = [_fix_date(m) for m in messages if isinstance(m, pynmea2.RMC)] + date = next((d for d in dates if d is not None), None) + if date is None: + fixes = [m for m in messages if isinstance(m, pynmea2.GGA) and _has_fix(m)] + if fixes: + LOG.warning( + "Ignoring the %d fixes in the NMEA file %s, as no RMC sentence gives their date", + len(fixes), + nmea_file, + ) + return [] # Parse GPS trace points = [] - for line in lines: - if "GPRMC" in line: - data = pynmea2.parse(line) - date = data.datetime.date() - - if "$GPGGA" in line: - data = pynmea2.parse(line) - dt = datetime.datetime.combine(date, data.timestamp) - lat, lon, alt = data.latitude, data.longitude, data.altitude - points.append( - geo.Point( - time=geo.as_unix_time(dt), lat=lat, lon=lon, alt=alt, angle=None - ) - ) + times: set[float] = set() + for message in messages: + if isinstance(message, pynmea2.RMC): + date = _fix_date(message) or date + + elif isinstance(message, pynmea2.GGA) and _has_fix(message): + dt = datetime.datetime.combine(date, message.timestamp) + time = geo.as_unix_time(dt) + if time in times: + continue + times.add(time) + lat, lon, alt = message.latitude, message.longitude, message.altitude + points.append(geo.Point(time=time, lat=lat, lon=lon, alt=alt, angle=None)) points.sort() return points + + +def _fix_date(message: pynmea2.RMC) -> datetime.date | None: + """The date of an RMC sentence, or None if it is void or has no date or time""" + if message.status != "A" or message.timestamp is None: + return None + return message.datestamp + + +def _has_fix(message: pynmea2.GGA) -> bool: + """Whether a GGA sentence has a time and a fix (quality 0 is none)""" + return message.timestamp is not None and bool(message.gps_qual) diff --git a/mapillary_tools/geotag/geotag_images_from_video.py b/mapillary_tools/geotag/geotag_images_from_video.py index a6035740f..092fe2f42 100644 --- a/mapillary_tools/geotag/geotag_images_from_video.py +++ b/mapillary_tools/geotag/geotag_images_from_video.py @@ -5,7 +5,6 @@ from __future__ import annotations -import bisect import dataclasses import logging import sys @@ -21,74 +20,12 @@ from .base import GeotagImagesFromGeneric from .geotag_images_from_gpx import GeotagImagesFromGPX from .geotag_videos_from_video import GeotagVideosFromVideo -from .video_extractors.gpx import GPXVideoExtractor +from .video_extractors.gpx import GPSTimeline, GPXVideoExtractor LOG = logging.getLogger(__name__) -@dataclasses.dataclass(frozen=True) -class GPSTimeline: - """ - The Unix time of the video's GPS clock along the video: interpolated - between its GPS timestamps, and extrapolated beyond them at the clock's - rate. A timelapse captured its frames at those times, which one rate from - one of them does not follow: the rate drifts from them over the video, and - the first timestamp of a GoPro MAX timelapse can be seconds early. - - >>> timeline = GPSTimeline((0.0, 1.0, 2.0), (100.0, 104.0, 114.0), rate=10.0) - >>> timeline.to_unix_time(0.5) - 102.0 - >>> timeline.to_unix_time(3.0) - 124.0 - """ - - video_times: tuple[float, ...] - unix_times: tuple[float, ...] - # GPS seconds per video second beyond the timestamps - rate: float - - @classmethod - def read(cls, points: T.Sequence[geo.Point], rate: float) -> GPSTimeline | None: - """ - Read the timeline from the GPS timestamps of the points, or None if they - have none - """ - pairs: list[tuple[float, float]] = [] - for p in points: - unix_time = p.get_unix_time() - if unix_time is None: - continue - if pairs and unix_time == pairs[-1][1]: - # GoPro GPS5 stamps a whole block (~1s) of points with one - # timestamp. Keep the first point of the block - continue - pairs.append((p.time, unix_time)) - - if not pairs: - return None - - return cls( - video_times=tuple(v for v, _ in pairs), - unix_times=tuple(u for _, u in pairs), - rate=rate, - ) - - def to_unix_time(self, video_time: float) -> float: - idx = bisect.bisect_right(self.video_times, video_time) - if 0 < idx < len(self.video_times): - v0, v1 = self.video_times[idx - 1], self.video_times[idx] - u0, u1 = self.unix_times[idx - 1], self.unix_times[idx] - return u0 + (u1 - u0) * (video_time - v0) / (v1 - v0) - - # Before the first timestamp or after the last - anchor = 0 if idx == 0 else -1 - return ( - self.unix_times[anchor] - + (video_time - self.video_times[anchor]) * self.rate - ) - - class GeotagImagesFromVideo(GeotagImagesFromGeneric): def __init__( self, @@ -144,6 +81,9 @@ def to_description( use_image_start_time=True, offset_time=self.offset_time, num_processes=self.num_processes, + # Gaps between trips are those of a GPX file, not of the + # video's own track + check_gaps=False, ) image_metadatas = geotag.to_description(sample_paths) @@ -169,12 +109,12 @@ def _on_capture_timeline( ) -> T.Sequence[geo.Point]: """ Time the video's GPS points in GPS seconds from the start of the video, - like the samples, which are stamped with their capture times. A - timelapse packs several into each second of video, as many as its GPS - timestamps tell. + like the samples, which are stamped with their capture times, as its + GPS timestamps tell: a timelapse packs several into each second of + video, and some clocks gain on the video. """ clock = GPXVideoExtractor._read_video_clock(points) - if clock is None or clock.rate == 1.0: + if clock is None: return points timeline = GPSTimeline.read(points, clock.rate) diff --git a/mapillary_tools/geotag/geotag_videos_from_gpx.py b/mapillary_tools/geotag/geotag_videos_from_gpx.py index 8e0acd4e6..2f34418d9 100644 --- a/mapillary_tools/geotag/geotag_videos_from_gpx.py +++ b/mapillary_tools/geotag/geotag_videos_from_gpx.py @@ -25,6 +25,10 @@ class GeotagVideosFromGPX(GeotagVideosFromGeneric): + # Reads the track file of each video, and where to find it by default + extractor_class: type[GPXVideoExtractor] = GPXVideoExtractor + default_pattern = "%g.gpx" + def __init__( self, source_path: options.SourcePathOption | None = None, @@ -34,7 +38,7 @@ def __init__( ): super().__init__(num_processes=num_processes) if source_path is None: - source_path = options.SourcePathOption(pattern="%g.gpx") + source_path = options.SourcePathOption(pattern=self.default_pattern) self.source_path = source_path self.video_sync = video_sync self.interpolation = interpolation @@ -48,7 +52,7 @@ def _generate_video_extractors( source_path = self.source_path.resolve(video_path) if source_path.is_file(): results.append( - GPXVideoExtractor( + self.extractor_class( video_path, source_path, video_sync=self.video_sync, @@ -62,7 +66,7 @@ def _generate_video_extractors( results.append( types.describe_error_metadata( exceptions.MapillaryVideoGPSNotFoundError( - "GPX file not found for video" + f"{self.extractor_class.file_format} file not found for video" ), filename=video_path, filetype=types.FileType.VIDEO, diff --git a/mapillary_tools/geotag/geotag_videos_from_nmea.py b/mapillary_tools/geotag/geotag_videos_from_nmea.py new file mode 100644 index 000000000..a4c84e45f --- /dev/null +++ b/mapillary_tools/geotag/geotag_videos_from_nmea.py @@ -0,0 +1,19 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the BSD license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import annotations + +from .geotag_videos_from_gpx import GeotagVideosFromGPX +from .video_extractors.nmea import NMEAVideoExtractor + + +class GeotagVideosFromNMEA(GeotagVideosFromGPX): + """ + Geotag videos from NMEA files, which are placed on the videos as GPX + files are + """ + + extractor_class = NMEAVideoExtractor + default_pattern = "%g.nmea" diff --git a/mapillary_tools/geotag/image_extractors/exif.py b/mapillary_tools/geotag/image_extractors/exif.py index 01470964f..0997f7a16 100644 --- a/mapillary_tools/geotag/image_extractors/exif.py +++ b/mapillary_tools/geotag/image_extractors/exif.py @@ -6,6 +6,7 @@ from __future__ import annotations import contextlib +import datetime import sys import typing as T from pathlib import Path @@ -15,10 +16,25 @@ else: from typing_extensions import override -from ... import exceptions, exif_read, geo, types, utils +from ... import exceptions, exif_read, types, utils from .base import BaseImageExtractor +def _capture_unix_time(capture_time: datetime.datetime) -> float: + """ + The Unix time of the capture time of an image, assuming that a naive one + is in the local timezone. Some cameras write 0001:01:01 00:00:00 for no + date, which has no Unix time: raise as for an image with no capture time, + rather than let geo.as_unix_time() move it to 1970. + """ + try: + return capture_time.timestamp() + except (ValueError, OverflowError, OSError): + raise exceptions.MapillaryGeoTaggingError( + f"Unable to extract timestamp from the image: invalid capture time {capture_time}" + ) + + class ImageEXIFExtractor(BaseImageExtractor): def __init__(self, image_path: Path, skip_lonlat_error: bool = False): super().__init__(image_path) @@ -46,11 +62,12 @@ def extract(self) -> types.ImageMetadata: raise exceptions.MapillaryGeoTaggingError( "Unable to extract timestamp from the image" ) + unix_time = _capture_unix_time(capture_time) image_metadata = types.ImageMetadata( filename=self.image_path, filesize=utils.get_file_size(self.image_path), - time=geo.as_unix_time(capture_time), + time=unix_time, lat=lat, lon=lon, alt=exif.extract_altitude(), diff --git a/mapillary_tools/geotag/utils.py b/mapillary_tools/geotag/utils.py index 3c1e24103..e1b02e514 100644 --- a/mapillary_tools/geotag/utils.py +++ b/mapillary_tools/geotag/utils.py @@ -29,28 +29,42 @@ def parse_gpx(gpx_file: Path) -> list[Track]: tracks.append([]) for point in segment.points: if point.time is not None: - unix_time = geo.as_unix_time(point.time) tracks[-1].append( - telemetry.CAMMGPSPoint( - time=unix_time, + track_point( + geo.as_unix_time(point.time), lat=point.latitude, lon=point.longitude, alt=point.elevation, - angle=None, - epoch_time=unix_time, - gps_fix_type=3 if point.elevation is not None else 2, - horizontal_accuracy=0.0, - vertical_accuracy=0.0, - velocity_east=0.0, - velocity_north=0.0, - velocity_up=0.0, - speed_accuracy=0.0, ) ) return tracks +def track_point( + unix_time: float, lat: float, lon: float, alt: float | None +) -> telemetry.CAMMGPSPoint: + """ + A point of a track read from a file beside a video (GPX or NMEA), at its + Unix time, as it is written to the CAMM track uploaded with the video + """ + return telemetry.CAMMGPSPoint( + time=unix_time, + lat=lat, + lon=lon, + alt=alt, + angle=None, + epoch_time=unix_time, + gps_fix_type=3 if alt is not None else 2, + horizontal_accuracy=0.0, + vertical_accuracy=0.0, + velocity_east=0.0, + velocity_north=0.0, + velocity_up=0.0, + speed_accuracy=0.0, + ) + + def index_rdf_description_by_path(xml_paths: T.Sequence[Path]) -> dict[str, ET.Element]: rdf_by_path: dict[str, ET.Element] = {} diff --git a/mapillary_tools/geotag/video_extractors/gpx.py b/mapillary_tools/geotag/video_extractors/gpx.py index bd348bbea..02a0ebd1f 100644 --- a/mapillary_tools/geotag/video_extractors/gpx.py +++ b/mapillary_tools/geotag/video_extractors/gpx.py @@ -5,6 +5,7 @@ from __future__ import annotations +import bisect import dataclasses import datetime import enum @@ -23,7 +24,7 @@ else: from typing_extensions import override -from ... import exceptions, geo, telemetry, types, utils +from ... import constants, exceptions, geo, telemetry, types, utils from ...camm import camm_parser from ...mp4 import construct_mp4_parser as cparser, simple_mp4_parser as sparser from ...mp4.mp4_sample_parser import MovieBoxParser @@ -61,6 +62,14 @@ # second off at each end. _LOGGER_TOLERANCE_SECONDS = 2.0 +# The first GPS timestamps of a recording can be stale: GoPro stamps its first +# block of points up to 3 seconds before the rest place the start of the video, +# and a PANOX V2 its first fix 7.6 seconds before. The step from a stale +# timestamp to the next runs this much faster than the median step of the rest. +# Further on, steps that fast are kept: a LabPano Pilot One gains on the video +# in steps of a second per half second, 2 minutes over a 14-minute video +_STALE_STEP_RATE = 1.5 + class SyncMode(enum.Enum): # Sync by video GPS timestamps if found, otherwise rebase @@ -98,7 +107,164 @@ def to_unix_time(self, video_time: float) -> float: return self.unix_time + (video_time - self.video_time) * self.rate +@dataclasses.dataclass(frozen=True) +class GPSTimeline: + """ + The Unix time of the video's GPS clock along the video: interpolated + between its GPS timestamps, and extrapolated beyond them at the clock's + rate. A straight clock does not follow them: the clock of a timelapse + drifts from its rate over the video, the first timestamp of a GoPro MAX + timelapse can be seconds early, and some cameras gain on the video in + steps. + + >>> timeline = GPSTimeline((0.0, 1.0, 2.0), (100.0, 104.0, 114.0), rate=10.0) + >>> timeline.to_unix_time(0.5) + 102.0 + >>> timeline.to_unix_time(3.0) + 124.0 + >>> timeline.to_video_time(109.0) + 1.5 + >>> timeline.to_video_time(95.0) + -0.5 + >>> timeline.starting_at(1.0, 200.0).to_unix_time(2.0) + 210.0 + """ + + video_times: tuple[float, ...] + unix_times: tuple[float, ...] + # GPS seconds per video second beyond the timestamps + rate: float + + @classmethod + def read(cls, points: T.Sequence[geo.Point], rate: float) -> GPSTimeline | None: + """ + Read the timeline from the GPS timestamps of the points, or None if they + have none. Leave out the timestamps that the others show to be wrong: + those out of order, those before a jump of days (from a placeholder + date to the GPS time), and stale first ones. + """ + pairs: list[tuple[float, float]] = [] + for p in points: + unix_time = p.get_unix_time() + if unix_time is None: + continue + if pairs and unix_time == pairs[-1][1]: + # GoPro GPS5 stamps a whole block (~1s) of points with one + # timestamp. Keep the first point of the block + continue + if pairs and p.time <= pairs[-1][0]: + # Two timestamps at one video time + continue + pairs.append((p.time, unix_time)) + + # A timestamp out of order with the others is wrong, and could not be + # interpolated back to the video + pairs = _increasing_subsequence(pairs) + if not pairs: + return None + + # GoPro stamps a placeholder date until the GPS acquires the time + start = 0 + for idx in range(1, len(pairs)): + (v0, u0), (v1, u1) = pairs[idx - 1], pairs[idx] + if _IMPLAUSIBLE_OFFSET_SECONDS < (u1 - u0) - (v1 - v0) * rate: + start = idx + + # Stale first timestamps (see _STALE_STEP_RATE). Judged by the steps of + # the rest, not by the rate: that is 1 for timestamps over less than a + # second of video, as those of a short timelapse, too few to measure it + pace = GPXVideoExtractor._median_step_rate(pairs[start + 1 :]) + while pace is not None and start + 1 < len(pairs): + (v0, u0), (v1, u1) = pairs[start], pairs[start + 1] + if (u1 - u0) <= (v1 - v0) * pace * _STALE_STEP_RATE: + break + start += 1 + + return cls( + video_times=tuple(v for v, _ in pairs[start:]), + unix_times=tuple(u for _, u in pairs[start:]), + rate=rate, + ) + + def to_unix_time(self, video_time: float) -> float: + return _interpolate_line( + self.video_times, self.unix_times, video_time, self.rate + ) + + def to_video_time(self, unix_time: float) -> float: + return _interpolate_line( + self.unix_times, self.video_times, unix_time, 1 / self.rate + ) + + def starting_at(self, video_time: float, unix_time: float) -> GPSTimeline: + """ + The timeline shifted to be at **unix_time** at **video_time**: at the + pace of the GPS clock, from a start that the clock does not tell + """ + shift = unix_time - self.to_unix_time(video_time) + return dataclasses.replace( + self, unix_times=tuple(u + shift for u in self.unix_times) + ) + + +def _interpolate_line( + xs: T.Sequence[float], ys: T.Sequence[float], x: float, slope: float +) -> float: + """ + Interpolate y at x along the line through the points (xs, ys), xs in + increasing order, and extrapolate it beyond them at **slope** + """ + idx = bisect.bisect_right(xs, x) + if 0 < idx < len(xs): + x0, x1 = xs[idx - 1], xs[idx] + y0, y1 = ys[idx - 1], ys[idx] + return y0 + (y1 - y0) * (x - x0) / (x1 - x0) + + # Before the first point or after the last + anchor = 0 if idx == 0 else -1 + return ys[anchor] + (x - xs[anchor]) * slope + + +def _increasing_subsequence( + pairs: T.Sequence[tuple[float, float]], +) -> list[tuple[float, float]]: + """ + The longest subsequence of the (video time, Unix time) pairs whose Unix + times increase + + >>> _increasing_subsequence([(0, 10), (1, 50), (2, 12), (3, 13), (4, 11)]) + [(0, 10), (2, 12), (3, 13)] + >>> _increasing_subsequence([]) + [] + """ + # tails[k]: the index of the pair that ends the increasing subsequence of + # length k + 1 with the smallest Unix time found so far + tails: list[int] = [] + tail_unix_times: list[float] = [] + previous: list[int] = [] + for idx, (_, unix_time) in enumerate(pairs): + k = bisect.bisect_left(tail_unix_times, unix_time) + previous.append(tails[k - 1] if k else -1) + if k == len(tails): + tails.append(idx) + tail_unix_times.append(unix_time) + else: + tails[k] = idx + tail_unix_times[k] = unix_time + + subsequence: list[tuple[float, float]] = [] + idx = tails[-1] if tails else -1 + while 0 <= idx: + subsequence.append(pairs[idx]) + idx = previous[idx] + subsequence.reverse() + return subsequence + + class GPXVideoExtractor(BaseVideoExtractor): + # The format of the track file, as messages name it + file_format = "GPX" + def __init__( self, video_path: Path, @@ -139,8 +305,8 @@ def extract(self) -> types.VideoMetadata: rate = self._duration_ratio() self._apply_offset_time(gpx_points, rate=rate) self._check_coverage(gpx_points, gpx_unix_times, [], start_clock, rate) - if self.clip_to_video and start_clock is not None: - gpx_points = self._clip_to_video(gpx_points) + if self.clip_to_video: + gpx_points = self._clip_to_video(gpx_points, start_clock) return types.VideoMetadata( filename=self.video_path, filesize=utils.get_file_size(self.video_path), @@ -148,42 +314,35 @@ def extract(self) -> types.VideoMetadata: points=gpx_points, ) - clock = self._read_video_clock(native_video_metadata.points) - - if clock is not None and gpx_points and self.sync_mode is SyncMode.REBASE: - # The GPX is placed to start with the video, and runs on at the rate - # of the video's GPS clock, which in a timelapse packs several GPS - # seconds into each second of video - clock = VideoClock( - video_time=0.0, unix_time=gpx_points[0].time, rate=clock.rate + video_clock = self._read_video_clock(native_video_metadata.points) + clock: VideoClock | GPSTimeline | None = self._sync_clock( + video_clock, gpx_points, noisy + ) + if video_clock is not None and clock is None: + LOG.warning( + "The GPS clock of %s, whose GPS was rejected as noise, is %.1f days away from %s. " + "Syncing the %s from the start of the video instead", + self.video_path, + abs(gpx_points[0].time - video_clock.to_unix_time(0)) / 86400, + self.gpx_path, + self.file_format, ) - elif clock is not None and gpx_points: - offset = gpx_points[0].time - clock.to_unix_time(0) - if abs(offset) > _IMPLAUSIBLE_OFFSET_SECONDS: - # GPS rejected as noise may not have GPS time yet: GoPro - # stamps a placeholder date until it acquires it. That GPS is - # what the GPX replaces, so do not let a placeholder clock - # place the GPX days away from the video. A clock that agrees - # with the creation time of the video is no placeholder: it - # places the GPX as for any video with GPS, and a GPX days - # away from it misses the video - if ( - noisy - and self.sync_mode is SyncMode.SYNC - and not self._clock_matches_creation_time(clock) - ): - LOG.warning( - "The GPS clock of %s, whose GPS was rejected as noise, is %.1f days away from %s. " - "Syncing the GPX from the start of the video instead", - self.video_path, - abs(offset) / 86400, - self.gpx_path, - ) - clock = None + if clock is not None and not noisy: + # Through the GPS timestamps of the video, which no straight clock + # follows. GPS rejected as noise gives the clock only + timeline = GPSTimeline.read(native_video_metadata.points, clock.rate) + assert timeline is not None, "a GPS clock is read from GPS timestamps" + clock = timeline + if clock is not None and gpx_points and self.sync_mode is SyncMode.REBASE: + # At the pace of the GPS clock from the start of the track, which + # distance sampling, reading the frames by their composition times, + # puts at that of the first frame + clock = self._start_with_the_video(clock, gpx_points[0].time) if clock is None: # Its GPS has no clock to sync with, as if it had no GPS clock = self._place_without_video_gps(gpx_points) + self._apply_offset_time(gpx_points, rate=self._duration_ratio()) else: if self._duration_ratio() != 1.0: LOG.warning( @@ -192,6 +351,12 @@ def extract(self) -> types.VideoMetadata: self.video_path, clock.rate, ) + if self.video_sync is not None and self.video_sync.start_time is not None: + LOG.warning( + "Ignoring --video_start_time %s for %s, which has a GPS clock of its own", + build_capture_time(self.video_sync.start_time), + self.video_path, + ) if clock.rate != 1.0: LOG.info( "The GPS clock in %s runs %.1f times faster than the video (timelapse). Syncing %s accordingly", @@ -199,11 +364,12 @@ def extract(self) -> types.VideoMetadata: clock.rate, self.gpx_path, ) + # In GPS seconds, before the clock places the track + self._apply_offset_time(gpx_points) for p in gpx_points: p.time = clock.to_video_time(p.time) rate = self._duration_ratio() if clock is None else clock.rate - self._apply_offset_time(gpx_points, rate=rate) # Started with the video, the track is checked against it as one that # no clock placed: the clock gave it only its pace self._check_coverage( @@ -213,8 +379,8 @@ def extract(self) -> types.VideoMetadata: None if self.sync_mode is SyncMode.REBASE else clock, rate, ) - if self.clip_to_video and clock is not None: - gpx_points = self._clip_to_video(gpx_points) + if self.clip_to_video: + gpx_points = self._clip_to_video(gpx_points, clock) return dataclasses.replace(native_video_metadata, points=gpx_points) @@ -222,26 +388,21 @@ def _read_gpx_points(self) -> list[geo.Point]: """ Read the points of the GPX file, merged into one track in time order """ - try: - gpx_tracks = parse_gpx(self.gpx_path) - except (OSError, UnicodeError, gpxpy.gpx.GPXException) as ex: - raise exceptions.MapillaryInvalidGPXError( - f"{self.gpx_path.name} is not a valid GPX file. " - "Replace it with a valid GPX file." - ) from ex + gpx_tracks = self._parse_tracks() if 1 < len(gpx_tracks): LOG.warning( - f"Found {len(gpx_tracks)} tracks in the GPX file {self.gpx_path}. Will merge points in all the tracks as a single track for interpolation" + f"Found {len(gpx_tracks)} tracks in the {self.file_format} file {self.gpx_path}. Will merge points in all the tracks as a single track for interpolation" ) all_points: list[geo.Point] = sum(gpx_tracks, []) gpx_points = [p for p in all_points if _GPS_EPOCH <= p.time] if len(gpx_points) < len(all_points): LOG.warning( - "Ignoring %d of the %d points in the GPX file %s, which are timestamped before 1980", + "Ignoring %d of the %d points in the %s file %s, which are timestamped before 1980", len(all_points) - len(gpx_points), len(all_points), + self.file_format, self.gpx_path, ) @@ -250,21 +411,39 @@ def _read_gpx_points(self) -> list[geo.Point]: gpx_points.sort(key=lambda p: p.time) return gpx_points - def _clip_to_video(self, gpx_points: list[geo.Point]) -> list[geo.Point]: + def _parse_tracks(self) -> list[list[geo.Point]]: + """ + Parse the tracks of the GPX file, each a list of points with Unix times """ - Clip a GPX track to the video when a clock gives its pace along the - video, so that where the video ends along it is known: the GPS clock - of the video, which places the track too unless - --interpolation_use_gpx_start_time starts it with the video, or the - --video_start_time of sampling. Its points outside the video are the - positions of none of its frames, and a GPX of a longer trip can run on - for long after it. + try: + return parse_gpx(self.gpx_path) + except (OSError, UnicodeError, gpxpy.gpx.GPXException) as ex: + raise exceptions.MapillaryInvalidGPXError( + f"{self.gpx_path.name} is not a valid GPX file. " + "Replace it with a valid GPX file." + ) from ex - A track that is only assumed to start with a video without a GPS clock - is left as it is: it may be that of a timelapse, which runs on past - the end of the video at the pace of the GPS clock rather than that of - the video. + def _clip_to_video( + self, gpx_points: list[geo.Point], clock: VideoClock | GPSTimeline | None + ) -> list[geo.Point]: + """ + Clip a GPX track to the video, from video time 0 to where the video + ends along it, which is known when a **clock** gives its pace along + the video: the GPS clock of the video, which places the track too + unless --interpolation_use_gpx_start_time starts it with the video, or + the --video_start_time of sampling. Its points outside the video are + the positions of none of its frames, and a GPX of a longer trip can + run on for long after it. + + A track placed at the start of a video without a GPS clock, assumed to + start with it or by --interpolation_use_gpx_start_time, is clipped + only at video time 0, which --interpolation_offset_time may move it + past, so that the first frames keep a position. Its end is left as it + is: it may be that of a timelapse, which runs on past the end of the + video at the pace of the GPS clock rather than that of the video. """ + if clock is None: + return _clip_track(gpx_points, None) return _clip_track(gpx_points, _video_duration(self.video_path)) def _place_without_video_gps( @@ -297,12 +476,44 @@ def _place_without_video_gps( self._check_assumed_start(gpx_points, ratio) elif self.sync_mode is SyncMode.REBASE and self.video_sync is not None: self._check_given_start(gpx_points, ratio) + self._rebase_times( + gpx_points, offset=self._gpx_start_video_time(), rate=ratio + ) + return None elif self.sync_mode is SyncMode.SYNC: self._warn_of_a_long_track(gpx_points) self._rebase_times(gpx_points, rate=ratio) return None + def _start_with_the_video( + self, clock: VideoClock | GPSTimeline, gpx_start_time: float + ) -> VideoClock | GPSTimeline: + """ + The **clock** moved to start the track, at **gpx_start_time**, with + the video, as --interpolation_use_gpx_start_time says it does + """ + video_time = self._gpx_start_video_time() + if isinstance(clock, GPSTimeline): + return clock.starting_at(video_time, gpx_start_time) + return VideoClock( + video_time=video_time, unix_time=gpx_start_time, rate=clock.rate + ) + + def _gpx_start_video_time(self) -> float: + """ + The video time that --interpolation_use_gpx_start_time starts the GPX + track at. The geotagging stage of video_process puts the start of the + track at the first sample, which is the first frame, so distance + sampling starts the track at the first frame too: by its composition + time, which distance sampling reads the frames by, and which is not 0 + in a video with B-frames. In process, the track starts at 0 as before, + and so it does in interval sampling, which ffmpeg cuts from time 0. + """ + if self.video_sync is None: + return 0.0 + return _first_frame_time(self.video_path) + def _duration_ratio(self) -> float: """ GPS seconds per second of a video without a GPS clock of its own to @@ -346,7 +557,7 @@ def _check_coverage( gpx_points: T.Sequence[geo.Point], gpx_unix_times: tuple[float, float] | None, video_gps_points: T.Sequence[geo.Point], - clock: VideoClock | None, + clock: VideoClock | GPSTimeline | None, rate: float, ) -> None: """ @@ -357,7 +568,10 @@ def _check_coverage( A track that misses the video has no position for any moment of it, so it raises when its placement says so: the clock it was placed by, or --interpolation_offset_time. A track that covers only part of the video - warns, and one that covers all of it is silent. + warns, and one that covers all of it is silent. The video can fall + between the ends of the track and still miss it, or part of it, in a + gap of more than constants.MAX_GPX_GAP_SECONDS between two of its + points. Without the duration of the video, the video is known only up to its last GPS point, and a track that starts after it may still overlap the @@ -378,6 +592,18 @@ def _check_coverage( gpx_first = min(p.time for p in gpx_points) gpx_last = max(p.time for p in gpx_points) + # The gaps of the track, and its parts between them, in video seconds + gap_points = track_gaps( + sorted(gpx_points, key=lambda p: p.time), + constants.MAX_GPX_GAP_SECONDS / rate, + ) + gaps = [(before.time, after.time) for before, after in gap_points] + parts = list( + zip( + [gpx_first, *(end for _, end in gaps)], + [*(start for start, _ in gaps), gpx_last], + ) + ) offset_time = ( 0.0 if self.interpolation is None else self.interpolation.offset_time ) @@ -389,7 +615,7 @@ def _check_coverage( gpx_unix_times[0] if clock is None else clock.to_unix_time(0) ) + offset_time - gpx_track = f"The GPX track in {self.gpx_path} ({_isoformat(gpx_unix_times[0])} to {_isoformat(gpx_unix_times[1])})" + gpx_track = f"The {self.file_format} track in {self.gpx_path} ({_isoformat(gpx_unix_times[0])} to {_isoformat(gpx_unix_times[1])})" if clock is None: video = f"the video {self.video_path}" else: @@ -404,10 +630,17 @@ def _check_coverage( ) # In GPS seconds, which is where clock and time zone mistakes are made - gap = max(gpx_first - video_end, -gpx_last) * rate + gap = min(max(first - video_end, -last) for first, last in parts) * rate if gap <= 0: # In video seconds - uncovered = max(gpx_first, 0.0) + max(video_end - gpx_last, 0.0) + uncovered = ( + max(gpx_first, 0.0) + + max(video_end - gpx_last, 0.0) + + sum( + max(min(end, video_end) - max(start, 0.0), 0.0) + for start, end in gaps + ) + ) if _LOGGER_TOLERANCE_SECONDS < uncovered * rate: LOG.warning( f"{gpx_track} covers only part of {video}{offset_clause}: " @@ -420,19 +653,42 @@ def _check_coverage( advice = "Check --interpolation_offset_time" elif offset_time: advice = ( - "Check --interpolation_offset_time, that the GPX file belongs to " - "this video, then the camera clock and the time zone of the GPX " + f"Check --interpolation_offset_time, that the {self.file_format} file belongs to " + f"this video, then the camera clock and the time zone of the {self.file_format} " "timestamps" ) else: advice = ( - "Check that the GPX file belongs to this video, then the camera " - "clock and the time zone of the GPX timestamps" + f"Check that the {self.file_format} file belongs to this video, then the camera " + f"clock and the time zone of the {self.file_format} timestamps" ) - message = ( - f"{gpx_track} misses {video} by {_format_duration(gap)}{offset_clause}. " - f"{advice}" + # Between the ends of the track, the video falls in one of its gaps + surrounding_gap = next( + ( + (before, after) + for before, after in gap_points + if before.time < 0 and video_end < after.time + ), + None, ) + if surrounding_gap is None: + message = ( + f"{gpx_track} misses {video} by {_format_duration(gap)}{offset_clause}. " + f"{advice}" + ) + else: + before, after = surrounding_gap + # At the Unix times the points were recorded at + track_gap = describe_gap( + before, + after, + lookup_start + before.time * rate, + lookup_start + after.time * rate, + ) + message = ( + f"{gpx_track} has {track_gap}, which {video} falls in{offset_clause}. " + f"{advice}" + ) if (clock is not None or offset_time) and ( video_duration is not None or _IMPLAUSIBLE_OFFSET_SECONDS < gap @@ -467,21 +723,37 @@ def _check_assumed_start( excess = gpx_duration - video_span if video_span < excess: - raise exceptions.MapillaryGPXVideoStartUnknownError( - f"The GPX track in {self.gpx_path} lasts {_format_duration(gpx_duration)} " + message = ( + f"The {self.file_format} track in {self.gpx_path} lasts {_format_duration(gpx_duration)} " f"and {self._describe_video_span(video_span, ratio)}, so the video " - "could start anywhere along the track. Specify --video_start_time, or " - "--interpolation_use_gpx_start_time if the track was started with the " - "video, and if the video is a timelapse, --video_duration_ratio as well" + "could start anywhere along the track. " ) + # Not --interpolation_use_gpx_start_time, which starts the track + # with the video too, and refuses it for the same reason + if gpx_duration < _MAX_RECORDING_SECONDS: + message += ( + "If the video is a timelapse, specify --video_duration_ratio. If " + "the GPS logger was started with the video and left running, " + "specify --video_start_time " + f"{build_capture_time(min(p.time for p in gpx_points))}, when the " + "track starts. If the track covers more than this video, specify " + "the time the video started as --video_start_time instead" + ) + else: + message += ( + "A track that long covers more than this video: specify the time " + "the video started as --video_start_time" + ) + raise exceptions.MapillaryGPXVideoStartUnknownError(message) if _LOGGER_TOLERANCE_SECONDS < excess: LOG.warning( - "Assuming that the GPX track in %s starts with the video %s, although " + "Assuming that the %s track in %s starts with the video %s, although " "the track lasts %s longer. If it was started first, the frames are " "placed up to that much too early along it. Specify --video_start_time " "to place them exactly, or --interpolation_use_gpx_start_time if the " "track was started with the video", + self.file_format, self.gpx_path, self.video_path, _format_duration(excess), @@ -505,7 +777,7 @@ def _check_given_start( return message = ( - f"The GPX track in {self.gpx_path} lasts {_format_duration(gpx_duration)} " + f"The {self.file_format} track in {self.gpx_path} lasts {_format_duration(gpx_duration)} " f"and {self._describe_video_span(video_span, ratio)}, so it runs on long " "after the video ends. " ) @@ -563,6 +835,47 @@ def _durations( gpx_duration = max(p.time for p in gpx_points) - min(p.time for p in gpx_points) return video_duration * ratio, gpx_duration + def _sync_clock( + self, + clock: VideoClock | None, + gpx_points: T.Sequence[geo.Point], + noisy: bool, + ) -> VideoClock | None: + """ + The clock to place the GPX track by along the video, from the GPS + clock of the video (None if it has none), whose GPS was rejected as + noise if **noisy**. Return None if the track is to be placed as for a + video without a GPS clock. --interpolation_use_gpx_start_time starts + the track at video time 0. + """ + if clock is None or not gpx_points: + return clock + + if self.sync_mode is SyncMode.REBASE: + # The GPX is placed to start with the video, and runs on at the rate + # of the video's GPS clock, which in a timelapse packs several GPS + # seconds into each second of video + return VideoClock( + video_time=0.0, unix_time=gpx_points[0].time, rate=clock.rate + ) + + # GPS rejected as noise may not have GPS time yet: GoPro stamps a + # placeholder date until it acquires it. That GPS is what the GPX + # replaces, so do not let a placeholder clock place the GPX days away + # from the video. A clock that agrees with the creation time of the + # video is no placeholder: it places the GPX as for any video with GPS, + # and a GPX days away from it misses the video + offset = gpx_points[0].time - clock.to_unix_time(0) + if ( + abs(offset) > _IMPLAUSIBLE_OFFSET_SECONDS + and noisy + and self.sync_mode is SyncMode.SYNC + and not self._clock_matches_creation_time(clock) + ): + return None + + return clock + def _clock_matches_creation_time(self, clock: VideoClock) -> bool: """ Whether the GPS clock of the video starts within a day of the mvhd @@ -587,20 +900,40 @@ def _warn_of_a_long_track(self, gpx_points: T.Sequence[geo.Point]) -> None: _check_assumed_start): the video could start anywhere along it. Only warn, since the track of a timelapse without a GPS clock of its own runs on past the end of the video too. + + With --interpolation_offset_time, it is the offset that places the + video along the track, and one way to place it right. """ video_duration = _video_duration(self.video_path) if video_duration is None: return - gpx_duration = max(p.time for p in gpx_points) - min(p.time for p in gpx_points) - if 2 * video_duration < gpx_duration: + gpx_start = min(p.time for p in gpx_points) + gpx_duration = max(p.time for p in gpx_points) - gpx_start + if gpx_duration <= 2 * video_duration: + return + + track = ( + f"The {self.file_format} track in {self.gpx_path} lasts {_format_duration(gpx_duration)} " + f"and the video {self.video_path} only {_format_duration(video_duration)}, " + "so the video could start anywhere along the track" + ) + offset_time = ( + 0.0 if self.interpolation is None else self.interpolation.offset_time + ) + if offset_time: LOG.warning( - f"The GPX track in {self.gpx_path} lasts {_format_duration(gpx_duration)} " - f"and the video {self.video_path} only {_format_duration(video_duration)}, " - "so the video could start anywhere along the track, and is assumed to " - "start with it. Unless the video is a timelapse of the whole track, " - "trim the GPX to the video, or sample the video with video_process " - "and --video_start_time" + f"{track}, and --interpolation_offset_time {offset_time:g} places its " + f"start at {_isoformat(gpx_start + offset_time)} along it. Unless the " + "video is a timelapse of the track from there, check that the offset " + f"is where the video starts along the track, trim the {self.file_format} to the " + "video, or sample the video with video_process and --video_start_time" + ) + else: + LOG.warning( + f"{track}, and is assumed to start with it. Unless the video is a " + f"timelapse of the whole track, trim the {self.file_format} to the video, or sample " + "the video with video_process and --video_start_time" ) def _extract_native_video_metadata(self) -> tuple[types.VideoMetadata, bool]: @@ -730,6 +1063,46 @@ def _median_step_rate(cls, pairs: T.Sequence[tuple[float, float]]) -> float | No return statistics.median(steps) if steps else None +def track_gaps( + points: T.Sequence[geo.Point], max_gap: float +) -> list[tuple[geo.Point, geo.Point]]: + """ + The gaps of a track, sorted by time, between consecutive points more than + **max_gap** seconds apart, as the points either side + + >>> track = [geo.Point(time=t, lat=0, lon=0, alt=None, angle=None) for t in (0, 1, 5000, 5001, 9000)] + >>> [(before.time, after.time) for before, after in track_gaps(track, 3600)] + [(1, 5000), (5001, 9000)] + >>> [(before.time, after.time) for before, after in track_gaps(track, 4000)] + [(1, 5000)] + >>> track_gaps(track[:2], 3600) + [] + """ + return [ + (left, right) + for left, right in geo.pairwise(points) + if max_gap < right.time - left.time + ] + + +def describe_gap( + before: geo.Point, after: geo.Point, before_time: float, after_time: float +) -> str: + """ + Describe a gap of a track between the points **before** and **after**, + which were recorded at the Unix times **before_time** and **after_time** + + >>> p = lambda lat, lon: geo.Point(time=0, lat=lat, lon=lon, alt=None, angle=None) + >>> describe_gap(p(55.174859, 61.256901), p(55.172986, 61.256636), 1739074271, 1739124451) + 'a gap of 13.9 hours, from 2025-02-09T04:11:11+00:00 at (55.174859, 61.256901) to 2025-02-09T18:07:31+00:00 at (55.172986, 61.256636)' + """ + return ( + f"a gap of {_format_duration(after_time - before_time)}, " + f"from {_isoformat(before_time)} at ({round(before.lat, 7)}, {round(before.lon, 7)}) " + f"to {_isoformat(after_time)} at ({round(after.lat, 7)}, {round(after.lon, 7)})" + ) + + def _clip_track( points: T.Sequence[geo.Point], duration: float | None ) -> list[geo.Point]: @@ -782,6 +1155,41 @@ def _video_duration(video_path: Path) -> float | None: return mvhd.duration / mvhd.timescale +def _first_frame_time(video_path: Path) -> float: + """ + The composition time of the first frame of the video stream with the + highest resolution, which distance sampling samples by, or 0 if it cannot + be read + """ + try: + moov = MovieBoxParser.parse_file(video_path) + video_tracks = [ + track for track in moov.extract_tracks() if track.is_video_track() + ] + if not video_tracks: + return 0.0 + # As ffprobe picks the stream to sample: the first of the largest + track = max( + video_tracks, + key=lambda track: ( + track.extract_tkhd_boxdata()["width"] + * track.extract_tkhd_boxdata()["height"] + ), + ) + return min( + (sample.exact_composition_time for sample in track.extract_samples()), + default=0.0, + ) + except ( + OSError, + sparser.ParsingError, + cparser.BoxNotFoundError, + C.ConstructError, + ) as ex: + LOG.debug("Failed to read the first frame time of %s: %s", video_path, ex) + return 0.0 + + def _video_creation_time(video_path: Path) -> float | None: """ The creation time of the video as Unix time, from its mvhd box, or None if diff --git a/mapillary_tools/geotag/video_extractors/nmea.py b/mapillary_tools/geotag/video_extractors/nmea.py new file mode 100644 index 000000000..466d00f84 --- /dev/null +++ b/mapillary_tools/geotag/video_extractors/nmea.py @@ -0,0 +1,42 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the BSD license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import annotations + +import sys + +if sys.version_info >= (3, 12): + from typing import override +else: + from typing_extensions import override + +from ... import exceptions, geo +from ..geotag_images_from_nmea_file import get_lat_lon_time_from_nmea +from ..utils import track_point +from .gpx import GPXVideoExtractor + + +class NMEAVideoExtractor(GPXVideoExtractor): + """ + Geotag a video from the track in an NMEA file, placed on the video as a + GPX track is + """ + + file_format = "NMEA" + + @override + def _parse_tracks(self) -> list[list[geo.Point]]: + try: + points = get_lat_lon_time_from_nmea(self.gpx_path) + # OSError: unreadable; ValueError: not text (UnicodeError), a + # malformed sentence (pynmea2.ParseError) or position; TypeError: a + # malformed time, which pynmea2 leaves a string + except (OSError, ValueError, TypeError) as ex: + raise exceptions.MapillaryInvalidGPXError( + f"{self.gpx_path.name} is not a valid NMEA file. " + "Replace it with a valid NMEA file." + ) from ex + + return [[track_point(p.time, p.lat, p.lon, p.alt) for p in points]] diff --git a/mapillary_tools/gpmf/gpmf_gps_filter.py b/mapillary_tools/gpmf/gpmf_gps_filter.py index 61f2ff908..125bf4208 100644 --- a/mapillary_tools/gpmf/gpmf_gps_filter.py +++ b/mapillary_tools/gpmf/gpmf_gps_filter.py @@ -48,7 +48,17 @@ def remove_outliers( return sequence max_speed = gps_filter.upper_whisker(ground_speeds) - merged = gps_filter.dbscan(sequences, gps_filter.speed_le(max_speed)) + # Pieces whose ends are no farther apart than the points within a piece + # were split only by the points between them, such as a single sample + # far off the track. Over the fraction of a second such a sample lasts, + # GPS jitter alone can read faster than the camera moved, so merge them + # whatever the speed between them + merged = gps_filter.dbscan( + sequences, + gps_filter.either( + gps_filter.distance_le(max_distance), gps_filter.speed_le(max_speed) + ), + ) LOG.debug( "Found %d sequences after merging with max speed %f", len(merged), max_speed ) diff --git a/mapillary_tools/gpmf/gps_filter.py b/mapillary_tools/gpmf/gps_filter.py index 005191e40..f6cefae9e 100644 --- a/mapillary_tools/gpmf/gps_filter.py +++ b/mapillary_tools/gpmf/gps_filter.py @@ -80,6 +80,18 @@ def _split_or_not(p1, p2): return _split_or_not +def distance_le( + max_distance: float, +) -> Decider: + """Return a callable that checks if two points are no farther apart than the given distance.""" + + def _f(p1, p2): + distance = geo.gps_distance((p1.lat, p1.lon), (p2.lat, p2.lon)) + return distance <= max_distance + + return _f + + def speed_le(max_speed: float) -> Decider: """Return a callable that checks if the speed between two points are slower than the given speed.""" @@ -100,6 +112,16 @@ def _f(p1, p2): return _f +def either( + s1: Decider, + s2: Decider, +) -> Decider: + def _f(p1, p2): + return s1(p1, p2) or s2(p1, p2) + + return _f + + def dbscan( sequences: T.Sequence[PointSequence], merge_or_not: Decider, diff --git a/mapillary_tools/process_geotag_properties.py b/mapillary_tools/process_geotag_properties.py index 5a1b5493b..61ef91fb9 100644 --- a/mapillary_tools/process_geotag_properties.py +++ b/mapillary_tools/process_geotag_properties.py @@ -62,6 +62,17 @@ def process_geotag_properties( raise exceptions.MapillaryFileNotFoundError( f"Import file or directory not found: {path}" ) + # A directory is searched for the files it can process, but a file + # named explicitly is skipped only with a word. It is not failed: + # callers that pass the files a user selected find the skipped ones in + # the process report. A zip file is uploaded as it is, by + # process_and_upload too + if path.is_file() and not ( + utils.is_image_file(path) + or utils.is_video_file(path) + or path.suffix.lower() == ".zip" + ): + LOG.warning("Skipping %s, which is not a supported image or video", path) if geotag_source_path is None: geotag_source_path = video_import_path @@ -97,7 +108,7 @@ def process_geotag_properties( image_paths = utils.find_images(import_paths, skip_subfolders=skip_subfolders) video_paths = utils.find_videos(import_paths, skip_subfolders=skip_subfolders) - metadata_or_errors = process(image_paths + video_paths, options) + metadata_or_errors = process(image_paths + video_paths, options, filetypes) return metadata_or_errors diff --git a/mapillary_tools/sample_video.py b/mapillary_tools/sample_video.py index 4714844f8..2ebcf24ce 100644 --- a/mapillary_tools/sample_video.py +++ b/mapillary_tools/sample_video.py @@ -29,7 +29,6 @@ ) from .exif_write import ExifEdit from .geotag import factory -from .geotag.geotag_images_from_video import GPSTimeline from .geotag.options import ( InterpolationOption, SourceOption, @@ -37,9 +36,10 @@ SourceType, VideoSyncOption, ) -from .geotag.video_extractors.gpx import GPXVideoExtractor +from .geotag.video_extractors.gpx import GPSTimeline, GPXVideoExtractor, VideoClock from .geotag.video_extractors.native import NativeVideoExtractor -from .mp4 import mp4_sample_parser +from .geotag.video_extractors.nmea import NMEAVideoExtractor +from .mp4 import mp4_sample_parser, simple_mp4_parser as sparser from .serializer.description import build_capture_time, parse_capture_time LOG = logging.getLogger(__name__) @@ -67,6 +67,24 @@ def _normalize_path( return video_dir, video_list +def find_sample_dirs( + video_import_path: Path, import_path: Path, skip_subfolders: bool = False +) -> list[Path]: + """ + The folder in import_path that sample_video() samples each of the videos + in video_import_path into + """ + video_dir, video_list = _normalize_path(video_import_path, skip_subfolders) + return [ + _sample_dir(import_path, video_dir, video_path) for video_path in video_list + ] + + +def _sample_dir(import_path: Path, video_dir: Path, video_path: Path) -> Path: + # need to resolve video_path because video_dir might be absolute + return Path(import_path).joinpath(video_path.resolve().relative_to(video_dir)) + + def xor(a: bool, b: bool): # xor https://stackoverflow.com/a/433161 return bool(a) ^ bool(b) @@ -132,8 +150,8 @@ def _parse_video_sources( if option.source_path is None: option.source_path = SourcePathOption(source_path=video_import_path) - # exif, nmea and the legacy gopro/blackvue/camm sources have nothing to read - # a video with; in video_process they are there to geotag the frames. Sample + # exif and the legacy gopro/blackvue/camm sources have nothing to read a + # video with; in video_process they are there to geotag the frames. Sample # by the sources that can read videos, or by the video's own GPS, as before, # if none of them can video_options = [option for option in options if factory.can_geotag_videos(option)] @@ -218,9 +236,7 @@ def sample_video( # - video_path: foo/bar/zzz.mp4 # Then: # - sample_dir: mapillary_sampled_video_frames/bar/zzz.mp4/ - sample_dir = Path(import_path).joinpath( - video_path.resolve().relative_to(video_dir) - ) + sample_dir = _sample_dir(import_path, video_dir, video_path) LOG.info("Removing the sample directory %s", sample_dir) if sample_dir.is_dir(): shutil.rmtree(sample_dir) @@ -237,10 +253,7 @@ def sample_video( failures: list[tuple[Path, Exception]] = [] for video_path in video_list: - # need to resolve video_path because video_dir might be absolute - sample_dir = Path(import_path).joinpath( - video_path.resolve().relative_to(video_dir) - ) + sample_dir = _sample_dir(import_path, video_dir, video_path) if sample_dir.exists(): LOG.warning( "Skip sampling video %s as it has been sampled in %s. Specify --rerun to resample it", @@ -258,6 +271,7 @@ def sample_video( sample_distance=video_sample_distance, start_time=video_start_time_dt, geotag_options=geotag_options, + duration_ratio=video_duration_ratio, ) else: assert 0 < video_sample_interval, ( @@ -607,27 +621,35 @@ def _extract_video_start_time( return _creation_time_to_start_time(video_path, probe) +# The sources that place a track file on a video, by the extractor that reads it +_TRACK_FILE_EXTRACTORS: dict[SourceType, type[GPXVideoExtractor]] = { + SourceType.GPX: GPXVideoExtractor, + SourceType.NMEA: NMEAVideoExtractor, +} + + def _check_gpx_start_by_interval( video_path: Path, geotag_options: T.Sequence[SourceOption], video_sync: VideoSyncOption, ) -> None: """ - Check the GPX tracks that --interpolation_use_gpx_start_time says started - with a video without a GPS clock, as distance sampling does. The geotagging - stage puts the first frame at the start of the track and the others at - --video_duration_ratio, so a track that runs on long after the video was - not recorded for it alone, or needs the ratio. + Check the GPX (or NMEA) tracks that --interpolation_use_gpx_start_time says + started with a video without a GPS clock, as distance sampling does. The + geotagging stage puts the first frame at the start of the track and the + others at --video_duration_ratio, so a track that runs on long after the + video was not recorded for it alone, or needs the ratio. """ for option in geotag_options: - if option.source is not SourceType.GPX or option.source_path is None: + extractor_class = _TRACK_FILE_EXTRACTORS.get(option.source) + if extractor_class is None or option.source_path is None: continue if option.interpolation is None or not option.interpolation.use_gpx_start_time: continue gpx_path = option.source_path.resolve(video_path) if not gpx_path.is_file(): continue - extractor = GPXVideoExtractor( + extractor = extractor_class( video_path, gpx_path, video_sync=video_sync, @@ -657,6 +679,65 @@ def _gpx_sync_reads_a_clock(extractor: GPXVideoExtractor) -> bool: return extractor._read_video_clock(native_video_metadata.points) is not None +def _gpx_sync_clock( + video_path: Path, geotag_options: T.Sequence[SourceOption] +) -> VideoClock | None: + """ + The clock that GPX sync, in distance sampling and process, places the + track of the first GPX (or NMEA) source by along a video whose GPS was + rejected as noise, which interval sampling has no telemetry from. None if + it places the track as for a video without a GPS clock. + """ + for option in geotag_options: + extractor_class = _TRACK_FILE_EXTRACTORS.get(option.source) + if extractor_class is None or option.source_path is None: + continue + gpx_path = option.source_path.resolve(video_path) + if not gpx_path.is_file(): + continue + extractor = extractor_class( + video_path, + gpx_path, + video_sync=option.video_sync, + interpolation=option.interpolation, + ) + try: + native_video_metadata, noisy = extractor._extract_native_video_metadata() + gpx_points = extractor._read_gpx_points() + except Exception as ex: + # The geotagging stage reports what keeps it from reading the + # GPS of the video or the GPX + LOG.debug("No GPX sync clock for %s: %s", video_path.name, ex) + return None + if not noisy: + return None + video_clock = extractor._read_video_clock(native_video_metadata.points) + return extractor._sync_clock(video_clock, gpx_points, noisy) + + return None + + +def _probe_video(ffmpeg: ffmpeglib.FFMPEG, video_path: Path) -> ffmpeglib.Probe: + """ + Probe the streams of a video with ffprobe. A video that ffprobe fails to + read because it has no movie box (moov), such as an MP4 cut short before + it, is invalid, as process reports it + """ + try: + return ffmpeglib.Probe(ffmpeg.probe_format_and_streams(video_path)) + except ffmpeglib.FFmpegCalledProcessError as ex: + try: + with video_path.open("rb") as fp: + sparser.parse_box_data_firstx(fp, [b"moov"]) + except sparser.BoxNotFoundError as box_ex: + raise exceptions.MapillaryInvalidVideoError( + f"Invalid video: {box_ex}" + ) from ex + except (OSError, sparser.ParsingError): + pass + raise ex + + def _interval_stream_specifier(probe: ffmpeglib.Probe) -> str: """ The video stream to sample by interval: the one with the highest @@ -693,6 +774,14 @@ def _sample_single_video_by_interval( # timestamps tell, and for a video without a GPS clock, at # --video_duration_ratio clock_rate = _telemetry_clock_rate(video_path, telemetry_points) + # GPS rejected as noise gives no telemetry, but GPX sync reads its clock, + # and the frames are stamped by it, where GPX sync places them along the + # GPX, so that the geotagging stage finds them there + gpx_clock: VideoClock | None = None + if clock_rate is None: + gpx_clock = _gpx_sync_clock(video_path, geotag_options) + if gpx_clock is not None: + clock_rate = gpx_clock.rate timeline: GPSTimeline | None = None if clock_rate is None: rate = duration_ratio @@ -705,7 +794,7 @@ def _sample_single_video_by_interval( video_path.name, clock_rate, ) - if clock_rate != 1.0: + if gpx_clock is None: # As distance samples are stamped, and as the geotagging stage # times the video's track to place the samples along it timeline = GPSTimeline.read(telemetry_points, clock_rate) @@ -713,13 +802,17 @@ def _sample_single_video_by_interval( if clock_rate is None and video_sync is not None: _check_gpx_start_by_interval(video_path, geotag_options, video_sync) - probe = ffmpeglib.Probe(ffmpeg.probe_format_and_streams(video_path)) + probe = _probe_video(ffmpeg, video_path) if start_time is None: if timeline is not None: start_time = datetime.datetime.fromtimestamp( timeline.to_unix_time(0.0), tz=datetime.timezone.utc ) + elif gpx_clock is not None: + start_time = datetime.datetime.fromtimestamp( + gpx_clock.to_unix_time(0.0), tz=datetime.timezone.utc + ) else: start_time = _extract_video_start_time(video_path, probe, telemetry_points) if start_time is None: @@ -728,10 +821,18 @@ def _sample_single_video_by_interval( ) stream_specifier = _interval_stream_specifier(probe) + # The stream sampled, whether it is the only one or not + video_stream = probe.probe_video_with_max_resolution() with wip_dir_context(wip_sample_dir(sample_dir), sample_dir) as wip_dir: ffmpeg.extract_frames_by_interval( - video_path, wip_dir, sample_interval, stream_specifier=stream_specifier + video_path, + wip_dir, + sample_interval, + stream_specifier=stream_specifier, + sample_aspect_ratio=( + video_stream.get("sample_aspect_ratio") if video_stream else None + ), ) frame_samples = ffmpeglib.FFMPEG.sort_selected_samples( wip_dir, video_path, selected_stream_specifiers=[stream_specifier] @@ -838,19 +939,59 @@ def _describe_track_time_range(points: T.Sequence[geo.Point]) -> str: return f"which covers {start:.3f} to {end:.3f} seconds of the video" +def _warn_of_options_the_clock_overrides( + video_path: Path, + points: T.Sequence[geo.Point], + sample_points: T.Sequence[geo.Point], + duration_ratio: float, + start_time: datetime.datetime | None, +) -> None: + """ + Warn of --video_duration_ratio and --video_start_time, which distance + sampling ignores for a video sampled by its own GPS, if it has a GPS clock: + the clock paces the video, and the frames are stamped with the GPS + timestamps of the points they are sampled at (**sample_points**). A frame + sampled where the track has no timestamp is stamped from the start time. + """ + if duration_ratio == 1.0 and start_time is None: + return + + clock = GPXVideoExtractor._read_video_clock(points) + if clock is None: + return + + if duration_ratio != 1.0: + LOG.warning( + "Ignoring --video_duration_ratio %g for %s, whose own GPS clock runs at %.3gx", + duration_ratio, + video_path.name, + clock.rate, + ) + + if start_time is not None and all( + p.get_unix_time() is not None for p in sample_points + ): + LOG.warning( + "Ignoring --video_start_time %s for %s, which has a GPS clock of its own", + build_capture_time(start_time), + video_path.name, + ) + + def _sample_single_video_by_distance( video_path: Path, sample_dir: Path, sample_distance: float, start_time: datetime.datetime | None = None, geotag_options: T.Sequence[SourceOption] | None = None, + duration_ratio: float = 1.0, ) -> None: if geotag_options is None: geotag_options = [SourceOption(SourceType.NATIVE)] ffmpeg = ffmpeglib.FFMPEG(constants.FFMPEG_PATH, constants.FFPROBE_PATH) - probe = ffmpeglib.Probe(ffmpeg.probe_format_and_streams(video_path)) + probe = _probe_video(ffmpeg, video_path) LOG.info("Extracting video metdata") @@ -915,11 +1056,35 @@ def _sample_single_video_by_distance( first_unix_time = min(track_unix_times, default=None) last_unix_time = max(track_unix_times, default=None) - # Frames at points with an absolute timestamp are timestamped from it - # below, so only the others need the start time - if start_time is None and any( - interp.get_unix_time() is None - for _, interp in sample_points_by_frame_idx.values() + # Read from the video itself, a track with a GPS clock of its own paces the + # video, and stamps the frames along its timeline, as interval samples are. + # A GPX source warns of the options that its sync ignores itself, and its + # sync placed the track along the video's clock already: its timestamps + # stamp the frames as they are + track_timeline: GPSTimeline | None = None + if not any( + option.source in factory.EXTERNAL_GPS_SOURCES for option in geotag_options + ): + _warn_of_options_the_clock_overrides( + video_path, + video_metadata.points, + [interp for _, interp in sample_points_by_frame_idx.values()], + duration_ratio, + start_time, + ) + track_clock = GPXVideoExtractor._read_video_clock(video_metadata.points) + if track_clock is not None: + track_timeline = GPSTimeline.read(video_metadata.points, track_clock.rate) + + # Frames along the timeline, or at points with an absolute timestamp, are + # timestamped from it below, so only the others need the start time + if ( + start_time is None + and track_timeline is None + and any( + interp.get_unix_time() is None + for _, interp in sample_points_by_frame_idx.values() + ) ): start_time = _gps_clock_start_time(video_metadata.points) if start_time is None: @@ -935,6 +1100,7 @@ def _sample_single_video_by_distance( wip_dir, frame_indices=set(sorted_sample_indices), stream_specifier=str(video_stream_idx), + sample_aspect_ratio=video_stream.get("sample_aspect_ratio"), ) frame_samples = ffmpeglib.FFMPEG.sort_selected_samples( @@ -962,8 +1128,11 @@ def _sample_single_video_by_distance( f"interpolated time {interp.time} should match the video sample time {video_sample.exact_composition_time}" ) - # Try to use the GPS timestamp if available (for timelapse videos) - gps_unix_time = interp.get_unix_time() + gps_unix_time = ( + interp.get_unix_time() + if track_timeline is None + else track_timeline.to_unix_time(interp.time) + ) if gps_unix_time is not None: assert first_unix_time is not None and last_unix_time is not None # Frames up to 1 ms of video past either end of the track are diff --git a/mapillary_tools/utils.py b/mapillary_tools/utils.py index b363e70b0..be8fc864e 100644 --- a/mapillary_tools/utils.py +++ b/mapillary_tools/utils.py @@ -260,6 +260,11 @@ def configure_logger( logger.setLevel(level) + # A worker process that is forked inherits the handler of its parent, + # and a second one would print each of its messages twice + if logger.handlers: + return logger + formatter = logging.Formatter( "%(asctime)s.%(msecs)03d - %(levelname)-7s - %(message)s", datefmt="%H:%M:%S", diff --git a/tests/cli/gps_filter.py b/tests/cli/gps_filter.py index f63ae045b..0907581c1 100644 --- a/tests/cli/gps_filter.py +++ b/tests/cli/gps_filter.py @@ -122,7 +122,12 @@ def _filter_outliers( return points max_speed = gps_filter.upper_whisker(ground_speeds) - merged = gps_filter.dbscan(subseqs, gps_filter.speed_le(max_speed)) + merged = gps_filter.dbscan( + subseqs, + gps_filter.either( + gps_filter.distance_le(max_distance), gps_filter.speed_le(max_speed) + ), + ) return T.cast( T.List[telemetry.GPSPoint], diff --git a/tests/integration/test_gopro.py b/tests/integration/test_gopro.py index 32cba7066..115d79507 100644 --- a/tests/integration/test_gopro.py +++ b/tests/integration/test_gopro.py @@ -31,16 +31,16 @@ "MAPILLARY_TOOLS_GOPRO_GPS_PRECISION": "10000000", "MAPILLARY_TOOLS_MAX_CAPTURE_SPEED_KMH": "2000000", # km/h } -# The capture times come from the GPMF GPS clock (the median of the first fixes -# puts the start at 2019-11-18T23:42:08.539Z), not from the container's creation -# time. This camera writes the creation time in local time, so reading the start -# time from there would timestamp every sample 8 hours off. +# The capture times come from the GPMF GPS timestamps (which put the start at +# 2019-11-18T23:42:10.009Z, without the first, 1.35 seconds stale), not from the +# container's creation time. This camera writes the creation time in local time, +# so reading the start time from there would timestamp every sample 8 hours off. EXPECTED_DESCS: T.List[T.Any] = [ { "filename": "hero8.mp4/hero8_v_000001.jpg", "filetype": "image", "MAPAltitude": 9540.24, - "MAPCaptureTime": "2019_11_18_23_42_08_539", + "MAPCaptureTime": "2019_11_18_23_42_10_009", "MAPCompassHeading": { "TrueHeading": 123.93587938690177, "MagneticHeading": 123.93587938690177, @@ -55,7 +55,7 @@ "filename": "hero8.mp4/hero8_v_000002.jpg", "filetype": "image", "MAPAltitude": 7112.573717404068, - "MAPCaptureTime": "2019_11_18_23_42_10_539", + "MAPCaptureTime": "2019_11_18_23_42_12_052", "MAPCompassHeading": { "TrueHeading": 140.8665026186285, "MagneticHeading": 140.8665026186285, @@ -70,7 +70,7 @@ "filename": "hero8.mp4/hero8_v_000003.jpg", "filetype": "image", "MAPAltitude": 7463.642846094319, - "MAPCaptureTime": "2019_11_18_23_42_12_539", + "MAPCaptureTime": "2019_11_18_23_42_13_976", "MAPCompassHeading": { "TrueHeading": 138.44255851085705, "MagneticHeading": 138.44255851085705, @@ -85,7 +85,7 @@ "filename": "hero8.mp4/hero8_v_000004.jpg", "filetype": "image", "MAPAltitude": 6909.8168472111465, - "MAPCaptureTime": "2019_11_18_23_42_14_539", + "MAPCaptureTime": "2019_11_18_23_42_16_009", "MAPCompassHeading": { "TrueHeading": 142.23462669862568, "MagneticHeading": 142.23462669862568, @@ -100,7 +100,7 @@ "filename": "hero8.mp4/hero8_v_000005.jpg", "filetype": "image", "MAPAltitude": 7212.594480737465, - "MAPCaptureTime": "2019_11_18_23_42_16_539", + "MAPCaptureTime": "2019_11_18_23_42_17_992", "MAPCompassHeading": { "TrueHeading": 164.70819093235514, "MagneticHeading": 164.70819093235514, @@ -115,7 +115,7 @@ "filename": "hero8.mp4/hero8_v_000006.jpg", "filetype": "image", "MAPAltitude": 7274.361994963208, - "MAPCaptureTime": "2019_11_18_23_42_18_539", + "MAPCaptureTime": "2019_11_18_23_42_20_019", "MAPCompassHeading": { "TrueHeading": 139.71549328876722, "MagneticHeading": 139.71549328876722, diff --git a/tests/integration/test_process.py b/tests/integration/test_process.py index f21c093ba..a14abe558 100644 --- a/tests/integration/test_process.py +++ b/tests/integration/test_process.py @@ -617,11 +617,11 @@ def test_process_videos_with_an_image_only_geotag_source(setup_data: py.path.loc def test_process_unsupported_filetypes(setup_data: py.path.local): video_dir = setup_data.join("gopro_data") for filetypes in ["blackvue"]: + # GoPro videos, which are not processed, as images are not descs = run_process_for_descs( ["--filetypes", filetypes, "--geotag_source=native", str(video_dir)] ) - assert 2 == len(descs) - assert 2 == len(find_desc_errors(descs)) + assert 0 == len(descs) for filetypes in ["image"]: descs = run_process_for_descs( @@ -630,6 +630,39 @@ def test_process_unsupported_filetypes(setup_data: py.path.local): assert 0 == len(descs) +def test_process_filetypes_of_videos(setup_data: py.path.local): + """ + The type of a video is known only once it is read. ExifTool, which the + default sources fall back to, processed the videos of other types anyway + """ + videos = [ + str(setup_data.join("gopro_data").join("max-360mode.mp4")), + str(setup_data.join("videos").join("sample-5s.mp4")), + ] + descs = run_process_for_descs(["--filetypes=gopro", *videos]) + assert [(Path(d["filename"]).name, d["filetype"]) for d in descs] == [ + ("max-360mode.mp4", "gopro") + ] + assert "error" not in descs[0] + + assert [] == run_process_for_descs(["--filetypes=blackvue", *videos]) + + # Limited by its own filetypes, a source leaves the video to the next + descs = run_process_for_descs( + [ + *[ + "--video_geotag_source", + '{"source": "native", "filetypes": ["blackvue"]}', + ], + *["--video_geotag_source", "native"], + videos[0], + ] + ) + assert [(Path(d["filename"]).name, d["filetype"]) for d in descs] == [ + ("max-360mode.mp4", "gopro") + ] + + def test_process_video_geotag_source_with_gpx_specified(setup_data: py.path.local): video_path = setup_data.join("videos").join("sample-5s.mp4") gpx_file = setup_data.join("gpx").join("sf_30km_h.gpx") @@ -728,7 +761,7 @@ def test_process_video_geotag_source_with_gpx_outside_video( assert descs[0]["error"]["vars"] == { # Its first GPS timestamp is 23:44:42.590, a second before the rest # place the start of the video - "image_time": "2019_11_18_23_44_43_613", + "image_time": "2019_11_18_23_44_43_624", "gpx_start_time": "2025_03_14_07_00_00_000", "gpx_end_time": "2025_03_14_07_01_33_624", } @@ -742,7 +775,7 @@ def test_process_video_from_gpx_with_interpolation_options( images. They were ignored for videos. """ video_path = setup_data.join("gopro_data").join("max-360mode.mp4") - # 3.613 seconds before the GPS clock of the video starts + # 3.624 seconds before the GPS timestamps of the video place its start start = datetime.datetime(2019, 11, 18, 23, 44, 40, tzinfo=datetime.timezone.utc) points = "".join( f'' @@ -776,19 +809,20 @@ def video_time_of_recorded(*options: str) -> float: [point] = [p for p in describe(*options)["MAPGPSTrack"] if p[5] == recorded] return point[0] + # In ms of video, which the GPS timestamps of the video pace at a second + # per second, give or take their jitter of some 40 ms synced = video_time_of_recorded() - assert synced == pytest.approx(10000 - 3613, abs=1) + assert synced == pytest.approx(10000 - 3624, abs=50) - assert video_time_of_recorded("--interpolation_use_gpx_start_time") == 10000 - assert ( - video_time_of_recorded( - "--interpolation_use_gpx_start_time", "--interpolation_offset_time=2.5" - ) - == 7500 + assert video_time_of_recorded("--interpolation_use_gpx_start_time") == ( + pytest.approx(10000, abs=50) ) + assert video_time_of_recorded( + "--interpolation_use_gpx_start_time", "--interpolation_offset_time=2.5" + ) == pytest.approx(7500, abs=50) shifted = video_time_of_recorded("--interpolation_offset_time=2") - assert shifted == pytest.approx(synced - 2000, abs=1) + assert shifted == pytest.approx(synced - 2000, abs=50) # an offset that moves the track off the video fails it, as it does a # track placed by the clock of the video @@ -799,6 +833,58 @@ def video_time_of_recorded(*options: str) -> float: assert "by 40 seconds, as --interpolation_offset_time 60 " in error["message"] +def test_process_video_from_nmea(setup_data: py.path.local): + """ + A video is geotagged from an NMEA file as from a GPX file of the same + track. It got no description at all. + """ + video_path = setup_data.join("gopro_data").join("max-360mode.mp4") + start = datetime.datetime(2019, 11, 18, 23, 44, 40, tzinfo=datetime.timezone.utc) + trkpts = [] + sentences = [] + for i in range(21): + time = start + datetime.timedelta(seconds=i) + trkpts.append( + f'' + f"" + ) + hhmmss = time.strftime("%H%M%S.00") + position = f"3700.{i * 100:04d},N,12200.0000,W" + for body in [ + f"GPRMC,{hhmmss},A,{position},0.0,0.0,{time.strftime('%d%m%y')},,,A", + f"GPGGA,{hhmmss},{position},1,08,1.0,,M,,M,,", + ]: + checksum = 0 + for char in body: + checksum ^= ord(char) + sentences.append(f"${body}*{checksum:02X}\r\n") + gpx_file = setup_data.join("track.gpx") + gpx_file.write( + '\n' + '' + f"{''.join(trkpts)}\n" + ) + nmea_file = setup_data.join("track.nmea") + nmea_file.write("".join(sentences)) + + def track(source: str, source_path: py.path.local): + [desc] = run_process_for_descs( + [ + *["--geotag_source", source], + *["--geotag_source_path", str(source_path)], + str(video_path), + ] + ) + return desc["MAPGPSTrack"] + + from_nmea = track("nmea", nmea_file) + from_gpx = track("gpx", gpx_file) + + assert len(from_nmea) == len(from_gpx) > 0 + for nmea_point, gpx_point in zip(from_nmea, from_gpx): + assert nmea_point == pytest.approx(gpx_point, abs=1e-6) + + def test_process_geotag_with_gpx_pattern_not_found(setup_data: py.path.local): video_path = setup_data.join("gopro_data").join("max-360mode.mp4") diff --git a/tests/integration/test_process_report.py b/tests/integration/test_process_report.py index 437fefe9e..1a1151684 100644 --- a/tests/integration/test_process_report.py +++ b/tests/integration/test_process_report.py @@ -112,3 +112,24 @@ def test_process_without_report_preserves_unsupported_omission(tmpdir: py.path.l assert json.loads(desc_path.read_text(encoding="utf-8")) == [] assert not (root / "process-report.json").exists() + + +def test_process_warns_of_an_explicit_unsupported_file(tmpdir: py.path.local): + """It was skipped without a word""" + root = Path(str(tmpdir)) + unsupported = root / "unsupported.webp" + desc_path = root / "description.json" + unsupported.write_bytes(b"not supported") + + result = run_command( + ["--desc_path", str(desc_path), str(unsupported)], + command="process", + capture_output=True, + text=True, + ) + + assert json.loads(desc_path.read_text(encoding="utf-8")) == [] + assert ( + f"Skipping {unsupported}, which is not a supported image or video" + in result.stderr + ) diff --git a/tests/integration/test_upload.py b/tests/integration/test_upload.py index 3bfa1e357..1afe1bb45 100644 --- a/tests/integration/test_upload.py +++ b/tests/integration/test_upload.py @@ -10,7 +10,7 @@ import py.path import pytest -from mapillary_tools.geotag.video_extractors.gpx import GPXVideoExtractor +from mapillary_tools.geotag.video_extractors.gpx import GPSTimeline, GPXVideoExtractor from mapillary_tools.geotag.video_extractors.native import NativeVideoExtractor from .fixtures import ( @@ -52,7 +52,9 @@ def test_upload_video_with_gpx_running_past_both_ends( points = NativeVideoExtractor(Path(video_path)).extract().points clock = GPXVideoExtractor._read_video_clock(points) assert clock is not None - video_start = clock.to_unix_time(0) + timeline = GPSTimeline.read(points, clock.rate) + assert timeline is not None + video_start = timeline.to_unix_time(0) # From 12.5 seconds before the video to about 47 seconds after it trkpts = "".join( f'