diff --git a/src/virtualship/cli/_initialise.py b/src/virtualship/cli/_initialise.py new file mode 100644 index 000000000..6fea21809 --- /dev/null +++ b/src/virtualship/cli/_initialise.py @@ -0,0 +1,260 @@ +import os +import re +import warnings +from datetime import timedelta +from pathlib import Path + +import click +import pandas as pd +import yaml + +from virtualship.models import ( + Expedition, + InstrumentsConfig, + Location, + Port, + Schedule, + Waypoint, +) +from virtualship.utils import EXPEDITION, _get_example_expedition + +ERR_SUPPLEMENT = "If the MFP export format has changed, please submit an issue at: https://github.com/Parcels-code/virtualship/issues." + + +def _initialise( + path: str | Path, from_mfp: str | None = None, start_date: str | None = None +): + path = Path(path) + path.mkdir(exist_ok=True) + + expedition = path / EXPEDITION + + if expedition.exists(): + raise FileExistsError( + f"File '{expedition}' already exists. Please remove it or choose another directory." + ) + + if from_mfp: + mfp_file = Path(from_mfp) + click.echo(f"Generating schedule from {mfp_file}...") + + # catch warnings raised to propagate them via click.echo + with warnings.catch_warnings(record=True) as captured_warnings: + warnings.simplefilter("always") + _mfp_to_yaml(mfp_file, start_date, expedition) + + indent = " " * 4 + click.echo( + "\n⚠️ The generated schedule does not contain INSTRUMENT selections. ⚠️" + "\n\nNow please either use the `\033[4mvirtualship plan\033[0m` app to complete the configuration, " + "\nOR edit 'expedition.yaml' and manually add the instrument selections under the 'schedule' heading." + "\n\nIf editing 'expedition.yaml' manually:" + "\n\n🌡️ Expected instrument(s) format: one line per instrument e.g." + f"\n\n{indent * 4}waypoints:\n{indent * 4}- instrument:\n{indent * 5}- CTD\n{indent * 5}- ARGO_FLOAT\n" + ) + + # output captured warnings to the terminal + if captured_warnings: + click.echo("\n❗️ WARNINGS:") + for w in captured_warnings: + click.echo(f"{indent}• {w.message}") + click.echo( + f"\n{indent}If you believe any of these warnings are incorrect (e.g. you have selected departure/arrival ports), and {ERR_SUPPLEMENT.replace('If ', '')}\n" + ) + else: + expedition.write_text(_get_example_expedition()) + + click.echo(f"Created '{expedition.name}' at {path}.") + + +def _mfp_to_yaml(file_path: Path, start_date: str, output_path: Path): + """Generates an expedition.yaml file from MFP Excel export.""" + mfp_data = _validate_mfp_data(file_path) + + # convert start_date string to datetime object if needed, ensuring it's standard Python datetime + if isinstance(start_date, str): + current_time = pd.to_datetime(start_date).to_pydatetime() + elif isinstance(start_date, pd.Timestamp): + current_time = start_date.to_pydatetime() + else: + current_time = start_date + + waypoints = [] + previous_timedelta = None + + for i, row in mfp_data.iterrows(): + if i > 0: + current_time += previous_timedelta + + is_port = "Port" in str(row["Station"]) or "Port" in str(row["Type"]) + lat = None if pd.isna(row["Latitude"]) else float(row["Latitude"]) + lon = None if pd.isna(row["Longitude"]) else float(row["Longitude"]) + loc = Location(latitude=lat, longitude=lon) + + # Ensure timestamp passed is a native python datetime (or string) to prevent PyYAML pandas pickle tags + time_val = ( + current_time.to_pydatetime() + if isinstance(current_time, pd.Timestamp) + else current_time + ) + + if is_port: + has_latlon = lat is not None and lon is not None + waypoints.append(Port(location=loc, time=time_val if has_latlon else None)) + else: + waypoints.append(Waypoint(instrument=None, location=loc, time=time_val)) + + previous_timedelta = ( + row["Total Time"] if pd.notna(row["Total Time"]) else timedelta(0) + ) + + # build and dump expedition YAML + static_yaml = yaml.safe_load(_get_example_expedition()) + expedition = Expedition( + schedule=Schedule(waypoints=waypoints), + instruments_config=InstrumentsConfig.model_validate( + static_yaml.get("instruments_config") + ), + ship_config=static_yaml.get("ship_config"), + ) + expedition.to_yaml(output_path) + + +def _validate_mfp_data(file_path: Path) -> pd.DataFrame: + """Load and validate MFP CruiseData export.""" + mfp_data = _load_mfp_export(file_path) + + # clean up column names + mfp_data.columns = mfp_data.columns.astype(str).str.strip() + junk_col_pattern = r"^(Unnamed:.*||\.\d+)$" + mfp_data = mfp_data.loc[:, ~mfp_data.columns.str.match(junk_col_pattern)] + + expected_columns = [ + "Station", + "Type", + "Latitude", + "Longitude", + "Sea Depth", + "Time at Station", + "Travel Time to Next", + "Distance to Next (NM)", + "Ship Speed (kn)", + "EEZ", + ] + expected_set = set(expected_columns) + actual_set = set(mfp_data.columns) + + missing_columns = expected_set - actual_set + if missing_columns: + raise ValueError( + f"Error: Found columns {list(actual_set)}, but expected columns {list(expected_columns)}. " + f"Are you sure that you're using the correct export from MFP?\n\n{ERR_SUPPLEMENT}" + ) + + extra_columns = actual_set - expected_set + if extra_columns: + warnings.warn( + f"Found additional unexpected columns {list(extra_columns)}. Manually added columns have no effect.", + stacklevel=2, + ) + + # safe float conversion for lat/lon + for coord in ["Latitude", "Longitude"]: + if mfp_data[coord].dtype in ["object", "string"]: + mfp_data[coord] = pd.to_numeric( + mfp_data[coord].astype(str).str.replace(",", "."), errors="coerce" + ) + + # check for missing departure/arrival ports and add placeholders if necessary + # check against both 'Station' and 'Type' columns; variations can occur when importing to MFP before re-exporting + has_departure = ( + "Departure Port" in mfp_data["Station"].values + or "Departure Port" in mfp_data["Type"].values + ) + has_arrival = ( + "Arrival Port" in mfp_data["Station"].values + or "Arrival Port" in mfp_data["Type"].values + ) + + if not has_departure or not has_arrival: + warnings.warn( + "The MFP export is missing either a 'Departure Port' or 'Arrival Port', or both. " + "Any missing port will be replaced with an empty placeholder in `expedition.yaml` but will be ignored in the simulation. " + "If missing the 'Departure Port', the prescribed start date will be used for Waypoint #1 instead. ", + stacklevel=2, + ) + + if not has_departure: + dept_row = _create_port_row(expected_columns, "Departure Port") + mfp_data = pd.concat([dept_row, mfp_data], ignore_index=True) # first row + + if not has_arrival: + arr_row = _create_port_row(expected_columns, "Arrival Port") + mfp_data = pd.concat([mfp_data, arr_row], ignore_index=True) # last row + + # Drop unexpected columns + mfp_data = mfp_data[list(expected_columns)] + + # convert 'Travel Time to Next' and 'Time at Station' to timedelta + mfp_data["Travel Time to Next"] = mfp_data["Travel Time to Next"].apply( + _mfp_string_to_timedelta + ) + mfp_data["Time at Station"] = mfp_data["Time at Station"].apply( + _mfp_string_to_timedelta + ) + + # combine 'Travel Time to Next' and 'Time at Station' into a single 'Total Time' column + # add 0 when Time at Station is NaN, to avoid NaT in Total Time, but not to Travel Time to keep NaT at the arrival port + mfp_data["Total Time"] = mfp_data["Travel Time to Next"] + mfp_data[ + "Time at Station" + ].fillna(pd.Timedelta(0)) + + return mfp_data + + +def _load_mfp_export(file_path: Path) -> pd.DataFrame: + if not os.path.isfile(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + + try: + return pd.read_excel(file_path).dropna(how="all", axis=1) # drop empty columns + except Exception as e: + raise RuntimeError( + "Could not read coordinates data from the provided file. " + "Ensure it is an exported .xlsx file from MFP." + ) from e + + +def _create_port_row(columns, port_type: str) -> pd.DataFrame: + """Generate a single placeholder row for missing departure/arrival ports.""" + row = {col: None for col in columns} + row["Station"] = port_type + row["Type"] = port_type + return pd.DataFrame([row]) + + +def _mfp_string_to_timedelta(value: str | None) -> timedelta | None: + """Parse MFP duration string (e.g., '0d 13h 13m') to timedelta.""" + if pd.isna(value): + return None + + match = re.search(r"(\d+)d\s*(\d+)h\s*(\d+)m", str(value)) + if match: + days, hours, minutes = map(int, match.groups()) + return timedelta(days=days, hours=hours, minutes=minutes) + + else: + raise ValueError( + f"Invalid MFP duration format: '{value}'. Expected format: 'Xd Yh Zm' (e.g., '0d 13h 13m'). {ERR_SUPPLEMENT}" + ) + + +def _validate_start_date(ctx, param, value): + """Callback to enforce and validate --start-date when --from-mfp is used.""" + if ctx.params.get("from_mfp"): + if not value: + raise click.BadParameter( + "The '--start-date' option is required when using '--from-mfp'." + "\n\nExpected format: 'YYYY-MM-DD HH:MM:SS' (with quotes, e.g., '2023-10-20 01:00:00'). If only the date is provided, the time will default to 00:00:00." + ) + return value diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 41f4d519e..3442fb1bf 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -2,14 +2,12 @@ import click +from virtualship.cli._initialise import _initialise, _validate_start_date from virtualship.cli._plan import _plan from virtualship.cli._run import _run from virtualship.utils import ( COPERNICUSMARINE_BGC_VARIABLES, COPERNICUSMARINE_PHYS_VARIABLES, - EXPEDITION, - get_example_expedition, - mfp_to_yaml, ) @@ -26,41 +24,21 @@ 'Marine Facilities Planning tool (specifically the "Export Coordinates > DD" option). ' "User edits are required after initialisation.", ) -def init(path, from_mfp): +@click.option( + "--start-date", + type=click.DateTime(formats=["%Y-%m-%d %H:%M:%S", "%Y-%m-%d"]), + default=None, + callback=_validate_start_date, + help="The departure/start date of the expedition (required when using --from-mfp). " + "Expected format: 'YYYY-MM-DD HH:MM:SS' (with quotes, e.g., '2023-10-20 01:00:00'). If only the date is provided, the time will default to 00:00:00.", +) +def init(path, from_mfp, start_date): """ Initialize a directory for a new expedition, with an expedition.yaml file. - If --mfp-file is provided, it will generate the expedition.yaml from the MPF file instead. + If --mfp-file is provided (and --start-date is also provided), it will generate the expedition.yaml from the MPF file instead. """ - path = Path(path) - path.mkdir(exist_ok=True) - - expedition = path / EXPEDITION - - if expedition.exists(): - raise FileExistsError( - f"File '{expedition}' already exist. Please remove it or choose another directory." - ) - - if from_mfp: - mfp_file = Path(from_mfp) - # Generate expedition.yaml from the MPF file - click.echo(f"Generating schedule from {mfp_file}...") - mfp_to_yaml(mfp_file, expedition) - click.echo( - "\n⚠️ The generated schedule does not contain TIME values or INSTRUMENT selections. ⚠️" - "\n\nNow please either use the `\033[4mvirtualship plan\033[0m` app to complete the schedule configuration, " - "\nOR edit 'expedition.yaml' and manually add the necessary time values and instrument selections under the 'schedule' heading." - "\n\nIf editing 'expedition.yaml' manually:" - "\n\n🕒 Expected time format: 'YYYY-MM-DD HH:MM:SS' (e.g., '2023-10-20 01:00:00')." - "\n\n🌡️ Expected instrument(s) format: one line per instrument e.g." - f"\n\n{' ' * 15}waypoints:\n{' ' * 15}- instrument:\n{' ' * 19}- CTD\n{' ' * 19}- ARGO_FLOAT\n" - ) - else: - # Create a default example expedition YAML - expedition.write_text(get_example_expedition()) - - click.echo(f"Created '{expedition.name}' at {path}.") + _initialise(Path(path), from_mfp, start_date) @click.command() diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 6af9d80ce..6f1fed056 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -16,6 +16,7 @@ from virtualship.models import ( Expedition, Location, + Port, Spacetime, Waypoint, ) @@ -133,7 +134,7 @@ def simulate(self) -> ScheduleOk | ScheduleProblem: ) # wait at the waypoint until ship is scheduled to be there # note measurements made at waypoint - time_passed = self._make_measurements(waypoint) + time_passed = self._get_instrument_timescosts(waypoint) # wait while measurements are being done self._progress_time_stationary(time_passed) @@ -247,9 +248,13 @@ def _get_underway_stationary_times( for i in range(1, int(npts) + 1) ] - def _make_measurements(self, waypoint: Waypoint) -> timedelta: - # if there are no instruments, there is no time cost - if waypoint.instrument is None: + def _get_instrument_timescosts(self, waypoint: Waypoint | Port) -> timedelta: + # port stops have no instruments; if there are no instruments, there is no time cost + if isinstance(waypoint, Port): + return timedelta() + + # if proper waypoint but there are no instruments, there is no time cost + if isinstance(waypoint, Waypoint) and waypoint.instrument is None: return timedelta() # make instruments a list even if it's only a single one diff --git a/src/virtualship/make_realistic/problems/simulator.py b/src/virtualship/make_realistic/problems/simulator.py index dcffbbff8..e08e0827b 100644 --- a/src/virtualship/make_realistic/problems/simulator.py +++ b/src/virtualship/make_realistic/problems/simulator.py @@ -23,6 +23,7 @@ InstrumentProblem, ) from virtualship.models.checkpoint import Checkpoint +from virtualship.models.expedition import Port from virtualship.utils import ( CACHE, EXPEDITION, @@ -74,6 +75,8 @@ def select_problems( Map each selected problem to a random waypoint (or None if pre-departure). Finally, cache the suite of problems to a directory (expedition-specific) for reference. """ + waypoints = self.expedition.schedule.waypoints + valid_instrument_problems = [ problem for problem in INSTRUMENT_PROBLEMS @@ -86,12 +89,9 @@ def select_problems( if isinstance(p, GeneralProblem) and p.pre_departure ] - num_waypoints = len(self.expedition.schedule.waypoints) + num_waypoints = len(waypoints) num_instruments = len(instruments_in_expedition) - expedition_duration_days = ( - self.expedition.schedule.waypoints[-1].time - - self.expedition.schedule.waypoints[0].time - ).days + expedition_duration_days = (waypoints[-1].time - waypoints[0].time).days # if only one waypoint, return just a pre-departure problem if num_waypoints < 2: @@ -166,13 +166,12 @@ def select_problems( random.shuffle(available_replacements) selected_problems.extend(available_replacements[:num_to_replace]) - # map each problem to a [random] waypoint (or None if pre-departure) + # map each problem to a [random, non-port waypoint] (or None if pre-departure) # limited to one per waypoint, else complicates scheduling and contingency checking waypoint_idxs = [] unassigned_problems = [] - available_idxs = list( - range(len(self.expedition.schedule.waypoints) - 1) - ) # exclude last waypoint (problem there would have no impact on scheduling) + is_port = [isinstance(wp, Port) for wp in waypoints] + available_idxs = [i for i, port in enumerate(is_port) if not port] # TODO: if incorporate departure and arrival port/waypoints in future, bear in mind index selection here may need to change for problem in selected_problems: @@ -181,13 +180,21 @@ def select_problems( else: if available_idxs: wp_select = random.choice(available_idxs) + wp_instruments = waypoints[wp_select].instrument + wp_instruments = wp_instruments if wp_instruments else [] # noqa; handle when waypoint instruments set to "null" in expedition.yaml - # fmt: off # check waypoint actually deploys the instrument associated with the problem...if not, replace it with a general (non-instrument related) problem # rather than a different waypoint, because it's possible no applicable waypoint is still available - wp_instruments = self.expedition.schedule.waypoints[wp_select].instrument - if isinstance(problem, InstrumentProblem) and problem.instrument_type not in wp_instruments: - available_general = [p for p in GENERAL_PROBLEMS if not p.pre_departure and p not in selected_problems] + needs_replacement = ( + isinstance(problem, InstrumentProblem) + and problem.instrument_type not in wp_instruments + ) + if needs_replacement: + available_general = [ + p + for p in GENERAL_PROBLEMS + if not p.pre_departure and p not in selected_problems + ] if not available_general: unassigned_problems.append(problem) @@ -196,15 +203,12 @@ def select_problems( replacement = random.choice(available_general) problem_idx = selected_problems.index(problem) selected_problems[problem_idx] = replacement - # fmt: on waypoint_idxs.append(wp_select) available_idxs.remove(wp_select) # each waypoint only used once else: - unassigned_problems.append( - problem - ) # if run out of available waypoints, remove problem from selection + unassigned_problems.append(problem) # noqa; if run out of available waypoints, remove problem from selection # remove any problems that couldn't be assigned a waypoint (i.e. if more problems than available waypoints) if unassigned_problems: diff --git a/src/virtualship/models/__init__.py b/src/virtualship/models/__init__.py index dd4b2bf14..b95544c89 100644 --- a/src/virtualship/models/__init__.py +++ b/src/virtualship/models/__init__.py @@ -8,6 +8,7 @@ DrifterConfig, Expedition, InstrumentsConfig, + Port, Schedule, SensorConfig, ShipConfig, @@ -23,6 +24,7 @@ __all__ = [ # noqa: RUF022 "Location", + "Port", "Schedule", "SensorConfig", "ShipConfig", diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index b72693731..5b5193166 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -37,9 +37,11 @@ class Expedition(pydantic.BaseModel): model_config = pydantic.ConfigDict(extra="forbid") def to_yaml(self, file_path: str) -> None: - """Write exepedition object to yaml file.""" + """Write expedition object to yaml file, with port/waypoint number comments.""" + annotated = self._annotate() + with open(file_path, "w") as file: - yaml.dump(self.model_dump(by_alias=True), file) + file.writelines(annotated) @classmethod def from_yaml(cls, file_path: str) -> Expedition: @@ -53,6 +55,8 @@ def get_instruments(self) -> set[InstrumentType]: instruments_in_expedition = [] # from waypoints for waypoint in self.schedule.waypoints: + if isinstance(waypoint, Port): + continue if waypoint.instrument: for instrument in waypoint.instrument: if instrument: @@ -70,6 +74,36 @@ def get_instruments(self) -> set[InstrumentType]: "Underway instrument config attribute(s) are missing from YAML. Must be Config object or None." ) from e + def _annotate(self): + """Add port/waypoint comments/annotations to the expedition.yaml file.""" + assert isinstance(self.schedule.waypoints[0], Port) & isinstance( + self.schedule.waypoints[-1], Port + ), ( + "First and last waypoints must be Ports." + ) # commenting logic below assumes first and last waypoints are ports + + raw = yaml.dump(self.model_dump(by_alias=True), default_flow_style=False) + + lines = raw.splitlines(keepends=True) + annotated = [] + waypoint_number = 0 + for line in lines: + stripped = line.lstrip() + indent = " " * (len(line) - len(stripped)) + + # waypoints start with "- instrument:" and Ports start with "- location:" (no instrument field). + if stripped.startswith("- instrument:"): + waypoint_number += 1 + annotated.append(f"{indent}# Waypoint {waypoint_number}\n") + + if stripped.startswith("- location:"): + arrival_departure = "Departure" if waypoint_number == 0 else "Arrival" + annotated.append(f"{indent}# Port of {arrival_departure}\n") + + annotated.append(line) + + return annotated + class ShipConfig(pydantic.BaseModel): """Configuration of the ship.""" @@ -84,7 +118,7 @@ class ShipConfig(pydantic.BaseModel): class Schedule(pydantic.BaseModel): """Schedule of the virtual ship.""" - waypoints: list[Waypoint] + waypoints: list[Port | Waypoint] model_config = pydantic.ConfigDict(extra="forbid") @@ -137,6 +171,8 @@ def verify( ) from e for wp_i, wp in enumerate(self.waypoints): + if isinstance(wp, Port): + continue # ports are in harbour; skip bathymetry land check try: value = bathymetry_field.eval( np.float64(0.0), # time @@ -162,7 +198,8 @@ def verify( zip(self.waypoints, self.waypoints[1:], strict=False) ): stationkeeping_time = _calc_wp_stationkeeping_time( - wp.instrument, instruments_config + wp.instrument if isinstance(wp, Waypoint) else None, + instruments_config, ) time_to_reach = _calc_sail_time( @@ -188,6 +225,15 @@ def verify( print("... All good to go!") +class Port(pydantic.BaseModel): + """A port stop: a location the ship visits with no instrument deployments made.""" + + location: Location | None = None + time: datetime | None = None + + model_config = pydantic.ConfigDict(extra="forbid") + + class Waypoint(pydantic.BaseModel): """A Waypoint to sail to with an optional time and an optional instrument.""" diff --git a/src/virtualship/models/location.py b/src/virtualship/models/location.py index 793e5312c..1c40bb8b4 100644 --- a/src/virtualship/models/location.py +++ b/src/virtualship/models/location.py @@ -7,26 +7,29 @@ class Location: """A location on a sphere.""" - latitude: float - longitude: float + latitude: float | None = None + longitude: float | None = None def __post_init__(self) -> None: """ - Verify this location has valid latitude and longitude. + Verify this location has valid latitude and longitude if provided. :raises ValueError: If latitude and/or longitude are not valid. """ - if self.lat < -90: - raise ValueError("Latitude cannot be smaller than -90.") - if self.lat > 90: - raise ValueError("Latitude cannot be larger than 90.") - if self.lon < -180: - raise ValueError("Longitude cannot be smaller than -180.") - if self.lon > 360: - raise ValueError("Longitude cannot be larger than 360.") + if self.lat is not None: + if self.lat < -90: + raise ValueError("Latitude cannot be smaller than -90.") + if self.lat > 90: + raise ValueError("Latitude cannot be larger than 90.") + + if self.lon is not None: + if self.lon < -180: + raise ValueError("Longitude cannot be smaller than -180.") + if self.lon > 360: + raise ValueError("Longitude cannot be larger than 360.") @property - def lat(self) -> float: + def lat(self) -> float | None: """ Shorthand for latitude variable. @@ -35,7 +38,7 @@ def lat(self) -> float: return self.latitude @property - def lon(self) -> float: + def lon(self) -> float | None: """ Shorthand for longitude variable. diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index acb16dcf0..1e201543c 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -1,36 +1,5 @@ # see https://virtualship.readthedocs.io/en/latest/user-guide/tutorials/working_with_expedition_yaml.html for more details on how to edit this file # -schedule: - waypoints: - - instrument: - - CTD - location: - latitude: 0 - longitude: 0 - time: 1998-01-01 00:00:00 - - instrument: - - DRIFTER - - CTD - location: - latitude: 0.01 - longitude: 0.01 - time: 1998-01-02 01:00:00 - - instrument: - - ARGO_FLOAT - location: - latitude: 0.02 - longitude: 0.02 - time: 1998-01-03 02:00:00 - - instrument: - - XBT - location: - latitude: 0.03 - longitude: 0.03 - time: 1998-01-04 03:00:00 - - location: - latitude: 0.03 - longitude: 0.03 - time: 1998-01-05 03:00:00 instruments_config: adcp_config: num_bins: 40 @@ -82,5 +51,52 @@ instruments_config: sensors: - TEMPERATURE - SALINITY +schedule: + waypoints: + # Port of Departure + - location: + latitude: 0 + longitude: 0 + time: 1998-01-01 00:00:00 + # Waypoint 1 + - instrument: + - CTD + location: + latitude: 0.01 + longitude: 0.01 + time: 1998-01-02 00:00:00 + # Waypoint 2 + - instrument: + - DRIFTER + - CTD + location: + latitude: 0.02 + longitude: 0.02 + time: 1998-01-03 01:00:00 + # Waypoint 3 + - instrument: + - ARGO_FLOAT + location: + latitude: 0.03 + longitude: 0.03 + time: 1998-01-04 02:00:00 + # Waypoint 4 + - instrument: + - XBT + location: + latitude: 0.04 + longitude: 0.04 + time: 1998-01-05 03:00:00 + # Waypoint 5 + - instrument: [] + location: + latitude: 0.05 + longitude: 0.05 + time: 1998-01-06 04:00:00 + # Port of Arrival + - location: + latitude: 0.06 + longitude: 0.06 + time: 1998-01-07 05:00:00 ship_config: ship_speed_knots: 10.0 diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index cd497657f..10f5993a7 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -2,10 +2,8 @@ import glob import hashlib -import os import re import sys -import warnings from datetime import datetime, timedelta from functools import lru_cache from importlib.resources import files @@ -29,7 +27,6 @@ from virtualship.models.checkpoint import Checkpoint from virtualship.models.expedition import SensorConfig -import pandas as pd import yaml from pydantic import BaseModel from yaspin import Spinner @@ -158,16 +155,15 @@ def decorator(cls): # ===================================================== -def load_static_file(name: str) -> str: +def _load_static_file(name: str) -> str: """Load static file from the ``virtualship.static`` module by file name.""" return files("virtualship.static").joinpath(name).read_text(encoding="utf-8") @lru_cache(None) -@lru_cache(None) -def get_example_expedition() -> str: +def _get_example_expedition() -> str: """Get the example unified expedition configuration file.""" - return load_static_file(EXPEDITION) + return _load_static_file(EXPEDITION) def _dump_yaml(model: BaseModel, stream: TextIO) -> str | None: @@ -182,137 +178,6 @@ def _generic_load_yaml(data: str, model: BaseModel) -> BaseModel: return model.model_validate(yaml.safe_load(data)) -def load_coordinates(file_path): - """Loads coordinates from a file based on its extension.""" - if not os.path.isfile(file_path): - raise FileNotFoundError(f"File not found: {file_path}") - - ext = os.path.splitext(file_path)[-1].lower() - - try: - if ext in [".xls", ".xlsx"]: - return pd.read_excel(file_path) - - if ext == ".csv": - return pd.read_csv(file_path) - - raise ValueError(f"Unsupported file extension {ext}.") - - except Exception as e: - raise RuntimeError( - "Could not read coordinates data from the provided file. " - "Ensure it is either a csv or excel file." - ) from e - - -def validate_coordinates(coordinates_data): - # Expected column headers - expected_columns = {"Station Type", "Name", "Latitude", "Longitude"} - - # Check if the headers match the expected ones - actual_columns = set(coordinates_data.columns) - - missing_columns = expected_columns - actual_columns - if missing_columns: - raise ValueError( - f"Error: Found columns {list(actual_columns)}, but expected columns {list(expected_columns)}. " - "Are you sure that you're using the correct export from MFP?" - ) - - extra_columns = actual_columns - expected_columns - if extra_columns: - warnings.warn( - f"Found additional unexpected columns {list(extra_columns)}. " - "Manually added columns have no effect. " - "If the MFP export format changed, please submit an issue: " - "https://github.com/OceanParcels/virtualship/issues.", - stacklevel=2, - ) - - # Drop unexpected columns (optional, only if you want to ensure strict conformity) - coordinates_data = coordinates_data[list(expected_columns)] - - # Continue with the rest of the function after validation... - coordinates_data = coordinates_data.dropna() - - # Convert latitude and longitude to floats, replacing commas with dots - # Handles case when the latitude and longitude have decimals with commas - if coordinates_data["Latitude"].dtype in ["object", "string"]: - coordinates_data["Latitude"] = coordinates_data["Latitude"].apply( - lambda x: float(x.replace(",", ".")) - ) - - if coordinates_data["Longitude"].dtype in ["object", "string"]: - coordinates_data["Longitude"] = coordinates_data["Longitude"].apply( - lambda x: float(x.replace(",", ".")) - ) - - return coordinates_data - - -def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D417 - """ - Generates an expedition.yaml file with schedule information based on data from MFP excel file. The ship and instrument configurations entries in the YAML file are sourced from the static version. - - Parameters - ---------- - - excel_file_path (str): Path to the Excel file containing coordinate and instrument data. - - The function: - 1. Reads instrument and location data from the Excel file. - 2. Determines the maximum depth and buffer based on the instruments present. - 3. Ensures longitude and latitude values remain valid after applying buffer adjustments. - 4. returns the yaml information. - - """ - # avoid circular imports - from virtualship.models import ( - Expedition, - InstrumentsConfig, - Location, - Schedule, - Waypoint, - ) - - # Read data from file - coordinates_data = load_coordinates(coordinates_file_path) - - coordinates_data = validate_coordinates(coordinates_data) - - # Generate waypoints - waypoints = [] - for _, row in coordinates_data.iterrows(): - waypoints.append( - Waypoint( - instrument=None, # instruments blank, to be built by user using `virtualship plan` UI or by interacting directly with YAML files - location=Location(latitude=row["Latitude"], longitude=row["Longitude"]), - ) - ) - - # Create Schedule object - schedule = Schedule( - waypoints=waypoints, - ) - - # extract instruments config from static - instruments_config = InstrumentsConfig.model_validate( - yaml.safe_load(get_example_expedition()).get("instruments_config") - ) - - # extract ship config from static - ship_config = yaml.safe_load(get_example_expedition()).get("ship_config") - - # combine to Expedition object - expedition = Expedition( - schedule=schedule, - instruments_config=instruments_config, - ship_config=ship_config, - ) - - # Save to YAML file - expedition.to_yaml(yaml_output_path) - - def _validate_numeric_to_timedelta( value: int | float | timedelta, unit: Literal["minutes", "days"] ) -> timedelta: @@ -638,7 +503,7 @@ def _calc_sail_time( def _calc_wp_stationkeeping_time( - wp_instrument_types: list, + wp_instrument_types: list | None, instruments_config: InstrumentsConfig, instrument_config_map: dict = INSTRUMENT_CONFIG_MAP, ) -> timedelta: diff --git a/tests/cli/test_initialise.py b/tests/cli/test_initialise.py new file mode 100644 index 000000000..0037c7b6d --- /dev/null +++ b/tests/cli/test_initialise.py @@ -0,0 +1,202 @@ +import pandas as pd +import pytest + +from virtualship.cli._initialise import _mfp_to_yaml +from virtualship.models import Expedition, Port, Waypoint +from virtualship.utils import _get_example_expedition + + +def test_get_example_expedition(): + assert len(_get_example_expedition()) > 0 + + +def test_valid_example_expedition(tmp_path): + path = tmp_path / "test.yaml" + with open(path, "w") as file: + file.write(_get_example_expedition()) + + Expedition.from_yaml(path) + + +def valid_mfp_data(): + return pd.DataFrame( + { + "Station": [ + "Departure Port", + "Station1", + "Station2", + "Station3", + "Arrival Port", + ], + "Type": ["Departure Port", "CTD", "CTD", "CTD", "Arrival Port"], + "Latitude": [30.8, 31.2, 32.5, 33.1, 34.0], + "Longitude": [-44.3, -45.1, -46.7, -47.2, -48.0], + "Sea Depth": [100, 200, 300, 400, 500], + "Time at Station": [ + "0d 00h 00m", + "0d 01h 00m", + "0d 01h 00m", + "0d 01h 00m", + "0d 00h 00m", + ], + "Travel Time to Next": [ + "0d 05h 00m", + "0d 06h 00m", + "0d 04h 00m", + "0d 03h 00m", + None, + ], + "Distance to Next (NM)": [50, 60, 40, 30, None], + "Ship Speed (kn)": [10, 10, 10, 10, None], + "EEZ": ["EEZ1", "EEZ1", "EEZ2", "EEZ2", "EEZ2"], + } + ) + + +@pytest.fixture +def valid_excel_mfp_file(tmp_path): + path = tmp_path / "file.xlsx" + valid_mfp_data().to_excel(path, index=False) + return path + + +@pytest.fixture +def valid_excel_mfp_file_with_commas(tmp_path): + path = tmp_path / "file.xlsx" + df = valid_mfp_data() + df["Latitude"] = df["Latitude"].astype(str).str.replace(".", ",") + df["Longitude"] = df["Longitude"].astype(str).str.replace(".", ",") + df.to_excel(path, index=False) + return path + + +@pytest.fixture +def invalid_mfp_file(tmp_path): + """File missing required MFP columns.""" + path = tmp_path / "file.xlsx" + df = pd.DataFrame({"WrongColumn": [1, 2, 3]}) + df.to_excel(path, index=False) + return path + + +@pytest.fixture +def unsupported_extension_mfp_file(tmp_path): + path = tmp_path / "file.unsupported" + valid_mfp_data().to_csv(path, index=False) + return path + + +@pytest.fixture +def nonexistent_mfp_file(tmp_path): + return tmp_path / "non_file.xlsx" + + +@pytest.fixture +def missing_columns_mfp_file(tmp_path): + path = tmp_path / "file.xlsx" + valid_mfp_data().drop(columns=["Longitude"]).to_excel(path, index=False) + return path + + +@pytest.fixture +def missing_ports_mfp_file(tmp_path): + path = tmp_path / "file.xlsx" + # remove rows marked as departure or arrival ports + df = valid_mfp_data() + df = df[~df["Station"].str.contains("Port")] + df.to_excel(path, index=False) + return path + + +@pytest.fixture +def unexpected_header_mfp_file(tmp_path): + path = tmp_path / "file.xlsx" + df = valid_mfp_data() + df["Unexpected Column"] = ["Extra1", "Extra2", "Extra3", "Extra4", "Extra5"] + df.to_excel(path, index=False) + return path + + +@pytest.mark.parametrize( + "fixture_name", + ["valid_excel_mfp_file", "valid_excel_mfp_file_with_commas"], +) +def test_mfp_to_yaml_success(request, fixture_name, tmp_path): + """Test that _mfp_to_yaml correctly processes a valid MFP Excel export.""" + valid_mfp_file = request.getfixturevalue(fixture_name) + yaml_output_path = tmp_path / "expedition.yaml" + start_date = "2023-10-20 01:00:00" + + _mfp_to_yaml(valid_mfp_file, start_date, yaml_output_path) + + # Ensure the YAML file was written + assert yaml_output_path.exists() + + # Load YAML and validate contents + data = Expedition.from_yaml(yaml_output_path) + + # 3 waypoints + 2 ports (departure & arrival) + assert len(data.schedule.waypoints) == 5 + assert isinstance(data.schedule.waypoints[0], Port) + assert isinstance(data.schedule.waypoints[-1], Port) + assert isinstance(data.schedule.waypoints[1], Waypoint) + + +@pytest.mark.parametrize( + "fixture_name,error,match", + [ + pytest.param( + "nonexistent_mfp_file", + FileNotFoundError, + r"File not found:", + id="FileNotFound", + ), + pytest.param( + "unsupported_extension_mfp_file", + RuntimeError, + "Could not read coordinates data from the provided file. Ensure it is an exported .xlsx file from MFP.", + id="UnsupportedExtension", + ), + pytest.param( + "invalid_mfp_file", + ValueError, + r"Error: Found columns .* but expected columns .*", + id="InvalidFile", + ), + pytest.param( + "missing_columns_mfp_file", + ValueError, + r"Error: Found columns .* but expected columns .*", + id="MissingColumns", + ), + ], +) +def test_mfp_to_yaml_exceptions(request, fixture_name, error, match, tmp_path): + """Test that _mfp_to_yaml raises an error when input file is not valid.""" + fixture = request.getfixturevalue(fixture_name) + yaml_output_path = tmp_path / "expedition.yaml" + start_date = "1998-05-01 01:00:00" + + with pytest.raises(error, match=match): + _mfp_to_yaml(fixture, start_date, yaml_output_path) + + +def test_mfp_to_yaml_extra_headers(unexpected_header_mfp_file, tmp_path): + """Test that _mfp_to_yaml prints a warning when extra columns are found.""" + yaml_output_path = tmp_path / "expedition.yaml" + start_date = "1998-05-01 01:00:00" + + with pytest.warns(UserWarning, match="Found additional unexpected columns.*"): + _mfp_to_yaml(unexpected_header_mfp_file, start_date, yaml_output_path) + + +def test_mfp_to_yaml_missing_ports_warning(missing_ports_mfp_file, tmp_path): + """Test that _mfp_to_yaml warns when departure or arrival ports are missing.""" + yaml_output_path = tmp_path / "expedition.yaml" + start_date = "1998-05-01 01:00:00" + + with pytest.warns( + UserWarning, + match="The MFP export is missing either a 'Departure Port' or 'Arrival Port'", + ): + _mfp_to_yaml(missing_ports_mfp_file, start_date, yaml_output_path) diff --git a/tests/cli/test_plan.py b/tests/cli/test_plan.py index 294592237..ec994ae60 100644 --- a/tests/cli/test_plan.py +++ b/tests/cli/test_plan.py @@ -17,7 +17,7 @@ SensorConfig, Waypoint, ) -from virtualship.utils import EXPEDITION, get_example_expedition +from virtualship.utils import EXPEDITION, _get_example_expedition NEW_SPEED = "8.0" NEW_LAT = "0.015" @@ -32,9 +32,9 @@ def _make_expedition( """Write a minimal expedition YAML.""" if instruments_config is None: instruments_config = InstrumentsConfig.model_validate( - yaml.safe_load(get_example_expedition()).get("instruments_config") + yaml.safe_load(_get_example_expedition()).get("instruments_config") ) - ship_config = yaml.safe_load(get_example_expedition()).get("ship_config") + ship_config = yaml.safe_load(_get_example_expedition()).get("ship_config") Expedition( schedule=Schedule(waypoints=waypoints), instruments_config=instruments_config, @@ -346,7 +346,7 @@ async def test_sensor_initial_state_reflects_config(tmp_path): sensors=[SensorConfig(sensor_type=SensorType.TEMPERATURE)], ) instruments_config = InstrumentsConfig.model_validate( - yaml.safe_load(get_example_expedition()).get("instruments_config") + yaml.safe_load(_get_example_expedition()).get("instruments_config") ) instruments_config.ctd_config = ctd_config _make_expedition( diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index 0b6978616..2124eb52b 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -8,7 +8,7 @@ ScheduleOk, ) from virtualship.instruments.types import InstrumentType -from virtualship.utils import EXPEDITION, EXPEDITION_IDENTIFIER, get_example_expedition +from virtualship.utils import EXPEDITION, EXPEDITION_IDENTIFIER, _get_example_expedition def _simulate_schedule(projection, expedition): @@ -47,7 +47,7 @@ def test_run(tmp_path, monkeypatch): expedition_dir = tmp_path / "expedition_dir" expedition_dir.mkdir() - (expedition_dir / EXPEDITION).write_text(get_example_expedition()) + (expedition_dir / EXPEDITION).write_text(_get_example_expedition()) monkeypatch.setattr("virtualship.cli._run.simulate_schedule", _simulate_schedule) diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index 4bde12bdd..456ab0540 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -7,6 +7,7 @@ import pyproj import pytest import xarray as xr +import yaml from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.models import ( @@ -18,8 +19,8 @@ ) from virtualship.utils import ( EXPEDITION, + _get_example_expedition, _get_expedition, - get_example_expedition, ) projection = pyproj.Geod(ellps="WGS84") @@ -237,7 +238,7 @@ def test_verify_schedule_errors(schedule: Schedule, error, match) -> None: @pytest.fixture def expedition(tmp_file): with open(tmp_file, "w") as file: - file.write(get_example_expedition()) + file.write(_get_example_expedition()) return Expedition.from_yaml(tmp_file) @@ -371,3 +372,40 @@ def test_all_instrument_configs_use_mixin(expedition): assert iconfig.__class__._instrument_type == iconfig._instrument_type, ( f"{iconfig.__class__.__name__}._instrument_type does not match its registered InstrumentType" ) + + +def test_waypoint_yaml_line() -> None: + """Each waypoint entry in the raw YAML dump should start with '- instrument:'.""" + base_time = datetime.strptime("1950-01-01", "%Y-%m-%d") + schedule = Schedule( + waypoints=[ + Waypoint(location=Location(0, 0), time=base_time, instrument=None), + Waypoint( + location=Location(1, 1), + time=base_time + timedelta(hours=1), + instrument=None, + ), + Waypoint( + location=Location(2, 2), + time=base_time + timedelta(hours=2), + instrument=["CTD"], + ), + ] + ) + raw = yaml.dump( + { + "schedule": { + "waypoints": [wp.model_dump(by_alias=True) for wp in schedule.waypoints] + } + }, + default_flow_style=False, + ) + + lines = [ + line for line in raw.splitlines() if line.lstrip().startswith("- instrument:") + ] + assert len(lines) == len(schedule.waypoints), ( + f"Expected {len(schedule.waypoints)} lines starting with '- instrument:' in the YAML dump, " + f"got {len(lines)}. The Waypoint field order or teminology may have changed. " + "Note this can have implications for the placement of waypoint number comments in Expedition.to_yaml()." + ) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index f84693c96..d501b72d4 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -7,13 +7,13 @@ from virtualship.models.checkpoint import Checkpoint from virtualship.models.expedition import Expedition, Schedule, Waypoint from virtualship.models.location import Location -from virtualship.utils import get_example_expedition +from virtualship.utils import _get_example_expedition @pytest.fixture def expedition(tmp_file): with open(tmp_file, "w") as file: - file.write(get_example_expedition()) + file.write(_get_example_expedition()) return Expedition.from_yaml(tmp_file) diff --git a/tests/test_mfp_to_yaml.py b/tests/test_mfp_to_yaml.py deleted file mode 100644 index 4eab16c29..000000000 --- a/tests/test_mfp_to_yaml.py +++ /dev/null @@ -1,152 +0,0 @@ -import os - -import pandas as pd -import pytest - -from virtualship.models import Expedition -from virtualship.utils import mfp_to_yaml - - -def valid_mfp_data(): - return pd.DataFrame( - { - "Station Type": ["A", "B", "C"], - "Name": ["Station1", "Station2", "Station3"], - "Latitude": [30.8, 31.2, 32.5], - "Longitude": [-44.3, -45.1, -46.7], - } - ) - - -# Fixture for Excel file -@pytest.fixture -def valid_excel_mfp_file(tmp_path): - path = tmp_path / "file.xlsx" - valid_mfp_data().to_excel(path, index=False) - return path - - -# Fixture for CSV file -@pytest.fixture -def valid_csv_mfp_file(tmp_path): - path = tmp_path / "file.csv" - valid_mfp_data().to_csv(path, index=False) - return path - - -@pytest.fixture -def valid_csv_mfp_file_with_commas(tmp_path): - path = tmp_path / "file.csv" - valid_mfp_data().to_csv(path, decimal=",", index=False) - return path - - -@pytest.fixture -def invalid_mfp_file(tmp_path): - path = tmp_path / "file.csv" - valid_mfp_data().to_csv(path, decimal=",", sep="|", index=False) - - return path - - -@pytest.fixture -def unsupported_extension_mfp_file(tmp_path): - path = tmp_path / "file.unsupported" - valid_mfp_data().to_csv(path, index=False) - - return path - - -@pytest.fixture -def nonexistent_mfp_file(tmp_path): - path = tmp_path / "non_file.csv" - - return path - - -@pytest.fixture -def missing_columns_mfp_file(tmp_path): - path = tmp_path / "file.xlsx" - valid_mfp_data().drop(columns=["Longitude"]).to_excel(path, index=False) - return path - - -@pytest.fixture -def unexpected_header_mfp_file(tmp_path): - path = tmp_path / "file.xlsx" - df = valid_mfp_data() - df["Unexpected Column"] = ["Extra1", "Extra2", "Extra3"] - df.to_excel(path, index=False) - yield path - - -@pytest.mark.parametrize( - "fixture_name", - ["valid_excel_mfp_file", "valid_csv_mfp_file", "valid_csv_mfp_file_with_commas"], -) -def test_mfp_to_yaml_success(request, fixture_name, tmp_path): - """Test that mfp_to_yaml correctly processes a valid MFP file.""" - valid_mfp_file = request.getfixturevalue(fixture_name) - - yaml_output_path = tmp_path / "expedition.yaml" - - # Run function (No need to mock open() for YAML, real file is created) - mfp_to_yaml(valid_mfp_file, yaml_output_path) - - # Ensure the YAML file was written - assert yaml_output_path.exists() - - # Load YAML and validate contents - data = Expedition.from_yaml(yaml_output_path) - - assert len(data.schedule.waypoints) == 3 - - -@pytest.mark.parametrize( - "fixture_name,error,match", - [ - pytest.param( - "nonexistent_mfp_file", - FileNotFoundError, - os.path.basename("/non_file.csv"), - id="FileNotFound", - ), - pytest.param( - "unsupported_extension_mfp_file", - RuntimeError, - "Could not read coordinates data from the provided file. Ensure it is either a csv or excel file.", - id="UnsupportedExtension", - ), - pytest.param( - "invalid_mfp_file", - ValueError, - r"Error: Found columns \['Station Type\|Name\|Latitude\|Longitude'\], but expected columns \[.*('Station Type'|'Longitude'|'Latitude'|'Name').*\]. Are you sure that you're using the correct export from MFP\?", - id="InvalidFile", - ), - pytest.param( - "missing_columns_mfp_file", - ValueError, - ( - r"Error: Found columns \[.*?('Station Type'| 'Name'| 'Latitude').*?\], " - r"but expected columns \[.*?('Station Type'| 'Name'| 'Latitude'| 'Longitude').*?\]." - ), - id="MissingColumns", - ), - ], -) -def test_mfp_to_yaml_exceptions(request, fixture_name, error, match, tmp_path): - """Test that mfp_to_yaml raises an error when input file is not valid.""" - fixture = request.getfixturevalue(fixture_name) - - yaml_output_path = tmp_path / "expedition.yaml" - - with pytest.raises(error, match=match): - mfp_to_yaml(fixture, yaml_output_path) - - -def test_mfp_to_yaml_extra_headers(unexpected_header_mfp_file, tmp_path): - """Test that mfp_to_yaml prints a warning when extra columns are found.""" - yaml_output_path = tmp_path / "expedition.yaml" - - with pytest.warns(UserWarning, match="Found additional unexpected columns.*"): - mfp_to_yaml(unexpected_header_mfp_file, yaml_output_path) diff --git a/tests/test_utils.py b/tests/test_utils.py index 63b5c4e97..4fd456714 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -18,17 +18,17 @@ _calc_wp_stationkeeping_time, _find_nc_file_with_variable, _get_bathy_data, + _get_example_expedition, _select_product_id, _start_end_in_product_timerange, build_particle_class_from_sensors, - get_example_expedition, ) @pytest.fixture def expedition(tmp_file): with open(tmp_file, "w") as file: - file.write(get_example_expedition()) + file.write(_get_example_expedition()) return Expedition.from_yaml(tmp_file) @@ -68,18 +68,6 @@ def fake_open_dataset(*args, **kwargs): yield -def test_get_example_expedition(): - assert len(get_example_expedition()) > 0 - - -def test_valid_example_expedition(tmp_path): - path = tmp_path / "test.yaml" - with open(path, "w") as file: - file.write(get_example_expedition()) - - Expedition.from_yaml(path) - - def test_instrument_registry_updates(dummy_instrument): from virtualship import utils