-
Notifications
You must be signed in to change notification settings - Fork 32
Correct provisional action-fault attribution via supersedes_source_id #536
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
eedf23f
3730328
5397309
348444c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -139,8 +139,17 @@ class ActionStatusBridgeNode : public rclcpp::Node { | |
| /// Resolve the action server's node FQN from its status-topic publisher, for | ||
| /// use as the fault source_id so faults associate with the gateway's SOVD | ||
| /// entity. Returns "" if no publisher with a discovery-resolved node name is | ||
| /// visible yet (the caller falls back and re-resolves later). | ||
| std::string server_fqn_for_action(const std::string & action_name); | ||
| /// visible yet (the caller falls back and re-resolves later). virtual so a unit | ||
| /// test can drive the re-attribution path without a live action graph. | ||
| virtual std::string server_fqn_for_action(const std::string & action_name); | ||
|
|
||
| /// Re-attribute faults first reported under a provisional (action-name) source | ||
| /// to the resolved server FQN. For each provisional action still reported failed, | ||
| /// once the FQN resolves it re-reports the fault under the FQN with | ||
| /// supersedes_source_id set to the provisional source, so the FaultManager swaps | ||
| /// the source and the fault resolves to the server node's SOVD entity. A no-op | ||
| /// once every provisional attribution has been corrected. Runs on the rescan tick. | ||
| void reattribute_provisional(); | ||
|
|
||
| /// Returns true if this (goal, status) pair was not logged before, marking it | ||
| /// logged. Bounded to avoid unbounded growth. Suppresses duplicate LOG lines | ||
|
|
@@ -172,6 +181,17 @@ class ActionStatusBridgeNode : public rclcpp::Node { | |
| std::map<std::string, std::unique_ptr<ros2_medkit_fault_reporter::FaultReporter>> reporters_; | ||
| std::mutex reporters_mutex_; | ||
|
|
||
| // Actions whose fault was first reported under a provisional (action-name) source | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Two docs just above still state the invariant this PR removes: reporter_for (lines 132-136) says "It is never re-attributed afterwards (reporting_sources is append-only...)" and the reporters_ comment (lines 178-180) says the source_id is "never changed". reattribute_provisional() now swaps the map entry (reporters_[action_name] = std::move(...fqn_reporter), action_status_bridge_node.cpp:580) and supersede is exactly what makes reporting_sources non-append-only. reporter_identity's doc has the same problem (line 275, "never swapped out") now that the test is StickyCreatedOncePerCall. The equivalent comment in test_action_status_bridge.cpp got updated ("it never swaps the source on its own"); update these to match. |
||
| // because the server node FQN was not yet discovered. reattribute_provisional() | ||
| // corrects each to the FQN once discovery resolves. Guarded by reporters_mutex_. | ||
| struct Provisional { | ||
| std::string old_source; // the provisional source_id to supersede | ||
| // The FQN-sourced reporter, created once the FQN resolves and reused across | ||
| // rescan ticks until its service client is ready (avoids re-discovery churn). | ||
| std::unique_ptr<ros2_medkit_fault_reporter::FaultReporter> fqn_reporter; | ||
| }; | ||
| std::map<std::string, Provisional> provisional_; | ||
|
|
||
| // Desired (latest observed) action-level state derived from status messages: | ||
| // the source of truth the bridge tries to make the FaultManager reflect. | ||
| // `canceled` records whether a failure was a CANCELED (vs ABORTED), to pick | ||
|
|
@@ -187,6 +207,10 @@ class ActionStatusBridgeNode : public rclcpp::Node { | |
| // so reconcile_pending() retries it on the next rescan instead of dropping it. | ||
| std::map<std::string, DesiredState> desired_state_; | ||
| std::map<std::string, ActionState> last_reported_state_; | ||
| // The `canceled` flag as of the report that put each action into kFailed, so a | ||
| // deferred re-attribution supersedes the fault code that was actually reported | ||
| // (not a value that flipped afterwards). Guarded by state_mutex_. | ||
| std::map<std::string, bool> reported_canceled_; | ||
| std::mutex state_mutex_; | ||
|
|
||
| // Bounded dedup of logged (goal_id:status) keys (LOG suppression only). | ||
|
|
@@ -251,6 +275,22 @@ class ActionStatusBridgeTestAccess { | |
| /// test assert the reporter is created once and never swapped out. | ||
| const void * reporter_identity(const std::string & action_name); | ||
|
|
||
| /// Run the re-attribution pass (what the rescan tick fires) so a test can drive | ||
| /// the provisional -> FQN correction without a live rescan timer. | ||
| void run_reattribute_provisional(); | ||
|
|
||
| /// True if the action currently has a provisional (action-name) attribution | ||
| /// pending correction to the server FQN. | ||
| bool is_provisional(const std::string & action_name) const; | ||
|
|
||
| /// True if re-attribution has resolved the FQN and created (but not yet | ||
| /// delivered, e.g. service not ready) the FQN-sourced reporter for this action. | ||
| bool provisional_has_fqn_reporter(const std::string & action_name) const; | ||
|
|
||
| /// The `canceled` flag captured for the action's currently-reported fault | ||
| /// (what a deferred re-attribution will supersede under). | ||
| bool reported_canceled(const std::string & action_name) const; | ||
|
|
||
| private: | ||
| ActionStatusBridgeNode * node_; | ||
| }; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -214,6 +214,10 @@ void ActionStatusBridgeNode::rescan_actions() { | |
| // arming can never strand a pending report. The fast timer is what keeps the | ||
| // FaultManager freeze-frame contemporaneous; this is just belt-and-suspenders. | ||
| reconcile_pending(); | ||
|
|
||
| // Correct any faults reported under a provisional (action-name) source now that | ||
| // discovery has had another rescan period to resolve the server FQN. | ||
| reattribute_provisional(); | ||
| } | ||
|
|
||
| void ActionStatusBridgeNode::prune_vanished(const std::map<std::string, std::string> & present_topics) { | ||
|
|
@@ -257,11 +261,13 @@ void ActionStatusBridgeNode::prune_vanished(const std::map<std::string, std::str | |
| } | ||
| } | ||
| reporters_.erase(action_name); | ||
| provisional_.erase(action_name); | ||
| } | ||
| { | ||
| std::lock_guard<std::mutex> lock(state_mutex_); | ||
| last_reported_state_.erase(action_name); | ||
| desired_state_.erase(action_name); | ||
| reported_canceled_.erase(action_name); | ||
| } | ||
| RCLCPP_INFO(get_logger(), "Action '%s' vanished; dropped (topic %s)", action_name.c_str(), topic.c_str()); | ||
| it = subs_.erase(it); | ||
|
|
@@ -358,6 +364,9 @@ ActionStatusBridgeNode::reconcile(const std::string & action_name, | |
| RCLCPP_INFO(get_logger(), "Action %s -> fault %s", action_name.c_str(), code.c_str()); | ||
| } | ||
| last_reported_state_[action_name] = ActionState::kFailed; | ||
| // Remember which code (aborted vs canceled) was actually raised, so a later | ||
| // re-attribution supersedes that exact fault, not one derived from a flipped flag. | ||
| reported_canceled_[action_name] = desired.canceled; | ||
| return ActionState::kFailed; | ||
| } | ||
|
|
||
|
|
@@ -462,27 +471,117 @@ ros2_medkit_fault_reporter::FaultReporter * ActionStatusBridgeNode::reporter_for | |
| std::lock_guard<std::mutex> lock(reporters_mutex_); | ||
| auto it = reporters_.find(action_name); | ||
| if (it != reporters_.end()) { | ||
| return it->second.get(); // created on the first report; source is fixed thereafter | ||
| return it->second.get(); // created on the first report; source fixed until re-attributed | ||
| } | ||
| // First report for this action: attribute it to the action SERVER's node FQN so | ||
| // the gateway can resolve the fault to its SOVD entity. During DDS discovery the | ||
| // FQN may be unresolved (rcl reports placeholders); fall back to the action name | ||
| // so the fault still fires on time. Reporting the fault on time takes priority | ||
| // over entity attribution when discovery is slow. | ||
| // | ||
| // The source is NOT re-attributed later: reporting_sources is append-only on the | ||
| // manager side and the per-entity /faults scope filter is strict-AND, so a | ||
| // provisional action-name source cannot be swapped for the FQN afterwards. | ||
| // Correct attribution for the slow-discovery case is a separate concern (it | ||
| // needs a way to supersede a provisional source). | ||
| // A fallback (action-name) attribution is recorded as provisional and later | ||
| // corrected to the FQN by reattribute_provisional(): reporting_sources is | ||
| // append-only and the per-entity /faults scope filter is strict-AND, so the | ||
| // provisional source cannot merely be added-to - it is swapped for the FQN via | ||
| // ReportFault.supersedes_source_id once discovery resolves. | ||
| const std::string fqn = server_fqn_for_action(action_name); | ||
| const std::string & source_id = fqn.empty() ? action_name : fqn; | ||
| if (fqn.empty()) { | ||
| provisional_[action_name] = Provisional{action_name, nullptr}; | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. provisional_ is process memory guarding a correction to state that lives in the FaultManager (persistent with the SQLite backend), and prune_vanished erases it too (line 264). Once the entry is gone the fault has no fix path, and the next report makes the misattribution permanent: restart the bridge inside the provisional window, the latched status replays the abort (or the next abort arrives), reporter_for() now resolves the FQN and reports without supersede, and the store appends (fault_storage.cpp:204) -> reporting_sources = {/navigate_to_pose, /bt_navigator}, which the strict-AND scope filter keeps off every entity forever. Nothing ever drops the stale source after that; ReportWithoutSupersedeKeepsBothSources is exactly this pair. Related corner: one provisional entry covers both codes but reported_canceled_ is last-writer-wins (line 369), so with canceled_is_fault a CANCELED raise + heal + ABORTED raise supersedes only _ABORTED; the healed _CANCELED fault keeps the provisional source, and a later re-raise of it appends the FQN - same permanent hide. So "at most once per fault" (line 574) is really at most once per action. The provisional source is deterministic (it is the action name), so a stateless option: pass supersedes_source_id = action_name on every FAILED report. Self and absent supersedes are proven no-ops (SelfSupersedeKeepsTheSource, SupersedeAbsentSourceJustAddsNew), and the provisional_/reported_canceled_ bookkeeping goes away. Trade-off to check: report() skips the local filter whenever supersede is set; this bridge pre-declares filtering off, so it only bites if someone re-enables it. Failing that, the restart window belongs in the README limitations. |
||
| } | ||
| auto reporter = std::make_unique<ros2_medkit_fault_reporter::FaultReporter>(this->shared_from_this(), source_id); | ||
| auto * raw = reporter.get(); | ||
| reporters_[action_name] = std::move(reporter); | ||
| return raw; | ||
| } | ||
|
|
||
| void ActionStatusBridgeNode::reattribute_provisional() { | ||
| // Snapshot the provisional action names under reporters_mutex_, then release it | ||
| // so the per-action pass does not iterate provisional_ while it mutates the map | ||
| // (it erases corrected entries below). Each action's delivered state is read | ||
| // under state_mutex_; the graph query and re-report then run under reporters_mutex_ | ||
| // - held while we look up, cache the FQN reporter in, and erase from provisional_, | ||
| // the same lock reporter_for() takes, so the correction stays atomic against | ||
| // concurrent status callbacks. Neither server_fqn_for_action() nor | ||
| // FaultReporter::report() re-acquires reporters_mutex_, so there is no deadlock. | ||
| std::vector<std::string> actions; | ||
| { | ||
| std::lock_guard<std::mutex> lock(reporters_mutex_); | ||
| actions.reserve(provisional_.size()); | ||
| for (const auto & [action_name, prov] : provisional_) { | ||
| actions.push_back(action_name); | ||
| } | ||
| } | ||
| if (actions.empty()) { | ||
| return; | ||
| } | ||
|
|
||
| for (const auto & action_name : actions) { | ||
| // Re-attribution applies only to a fault that is actually LIVE in the | ||
| // FaultManager under the provisional source. Read the delivered state: | ||
| // - not delivered yet (no last_reported entry): the fault does not exist yet | ||
| // (its FAILED is still deferred on service discovery). Keep the entry and | ||
| // wait - erasing here would strand the report so it later lands under the | ||
| // provisional source with nothing left to correct it. | ||
| // - healed (kHealthy): nothing active to correct now, but a re-failure would | ||
| // reactivate under the still-provisional source (reporter_for reuses the | ||
| // frozen reporter), so keep the entry to correct it then. | ||
| // - failed (kFailed): correct it. | ||
| // reported_canceled_ carries the canceled flag AS OF the report that raised the | ||
| // fault, so the supersede targets the code actually reported, not a later flip. | ||
| bool has_reported = false; | ||
| ActionState reported = ActionState::kUnknown; | ||
| bool reported_canceled = false; | ||
| { | ||
| std::lock_guard<std::mutex> lock(state_mutex_); | ||
| auto sit = last_reported_state_.find(action_name); | ||
| has_reported = sit != last_reported_state_.end(); | ||
| if (has_reported) { | ||
| reported = sit->second; | ||
| } | ||
| auto cit = reported_canceled_.find(action_name); | ||
| reported_canceled = cit != reported_canceled_.end() && cit->second; | ||
| } | ||
|
|
||
| std::lock_guard<std::mutex> lock(reporters_mutex_); | ||
| auto pit = provisional_.find(action_name); | ||
| if (pit == provisional_.end()) { | ||
| continue; // raced away (e.g. pruned) | ||
| } | ||
| if (!has_reported || reported != ActionState::kFailed) { | ||
| continue; // not delivered yet, or healed: keep the entry, nothing to correct now | ||
| } | ||
|
|
||
| // Resolve the server FQN once, then keep the FQN reporter across ticks until | ||
| // its service client is ready (recreating it every tick would reset discovery). | ||
| if (!pit->second.fqn_reporter) { | ||
| const std::string fqn = server_fqn_for_action(action_name); | ||
| if (fqn.empty() || fqn == pit->second.old_source) { | ||
| continue; // still unresolved; retry on the next rescan | ||
| } | ||
| pit->second.fqn_reporter = | ||
| std::make_unique<ros2_medkit_fault_reporter::FaultReporter>(this->shared_from_this(), fqn); | ||
| } | ||
| if (!pit->second.fqn_reporter->is_service_ready()) { | ||
| continue; // FaultManager service not discovered by the new client yet; retry | ||
| } | ||
|
|
||
| // Re-report the same fault (the code actually reported, per reported_canceled_) | ||
| // under the FQN, superseding the provisional source, so the FaultManager ends | ||
| // with reporting_sources = {FQN} and the fault resolves to the server node's | ||
| // SOVD entity under the strict-AND scope filter. The fault is still active, so this | ||
| // second FAILED report does not inflate occurrence_count (the store counts occurrence | ||
| // edges, not reports) - it only swaps the source. Happens at most once per fault. | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. "It only swaps the source" is true for occurrence_count but not for the rest of the FAILED update this re-report runs through:
None of the new tests (either e2e, storage units) assert last_occurred, debounce_counter, or occurrence_count across a supersede. Worth either scoping this comment down to the occurrence_count claim, or teaching the store to skip the timestamp/debounce mutation when a supersede re-report hits an already-active fault. |
||
| const std::string code = fault_code_for(action_name, reported_canceled); | ||
| const std::string desc = std::string("Action ") + action_name + (reported_canceled ? " canceled" : " aborted"); | ||
| pit->second.fqn_reporter->report(code, aborted_severity_, desc, pit->second.old_source); | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The one-shot correction is committed on an unacked send. report() here is fire-and-forget (async_send_request, no callback), send_report early-returns if service_is_ready() flipped after the check at 565, and the swap+erase below plus the "Re-attributed" INFO run either way. Realistic trigger: the FaultManager restarts and this tick lands in the down window, or it dies with the request in flight. The supersede is lost, provisional_ no longer has the entry, and every later FAILED goes out under the FQN with no supersedes_source_id, so reporting_sources keeps the provisional name and strict-AND hides the fault from the owning entity again - the pre-fix #467 state, permanent, logged as corrected. Ordering is unguarded too: last_reported_state_ == kFailed only means the FAILED was sent (reconcile commits right after its async send), and the supersede travels on a different service client, with no cross-writer ordering guarantee. On heal-then-refail the rescan tick can fire ms after the re-raise FAILED (fqn_reporter is already cached and service-ready), so the manager can process the supersede first and the refail then re-adds the provisional source with nothing left to correct it. Narrower window, same permanent outcome. Committing the swap+erase in the async_send_request response callback closes the lost-request case - reattribute already retries every rescan tick until then. It does not close the reorder case (the response proves the supersede was processed, not that it landed after the FAILED it names); that one needs the supersede sent on the same client as the FAILED, e.g. a source_id override in send_report, or a read-back before erasing. |
||
| RCLCPP_INFO(get_logger(), "Re-attributed action '%s' fault '%s' from provisional '%s' to server FQN", | ||
| action_name.c_str(), code.c_str(), pit->second.old_source.c_str()); | ||
| reporters_[action_name] = std::move(pit->second.fqn_reporter); | ||
| provisional_.erase(pit); | ||
| } | ||
| } | ||
|
|
||
| std::string ActionStatusBridgeNode::server_fqn_from_endpoint(const std::string & node_name, | ||
| const std::string & node_namespace) { | ||
| // During DDS discovery the participant is known before its node name/namespace | ||
|
|
@@ -670,4 +769,25 @@ const void * ActionStatusBridgeTestAccess::reporter_identity(const std::string & | |
| return node_->reporter_for(action_name); | ||
| } | ||
|
|
||
| void ActionStatusBridgeTestAccess::run_reattribute_provisional() { | ||
| node_->reattribute_provisional(); | ||
| } | ||
|
|
||
| bool ActionStatusBridgeTestAccess::is_provisional(const std::string & action_name) const { | ||
| std::lock_guard<std::mutex> lock(node_->reporters_mutex_); | ||
| return node_->provisional_.count(action_name) > 0; | ||
| } | ||
|
|
||
| bool ActionStatusBridgeTestAccess::provisional_has_fqn_reporter(const std::string & action_name) const { | ||
| std::lock_guard<std::mutex> lock(node_->reporters_mutex_); | ||
| auto it = node_->provisional_.find(action_name); | ||
| return it != node_->provisional_.end() && it->second.fqn_reporter != nullptr; | ||
| } | ||
|
|
||
| bool ActionStatusBridgeTestAccess::reported_canceled(const std::string & action_name) const { | ||
| std::lock_guard<std::mutex> lock(node_->state_mutex_); | ||
| auto it = node_->reported_canceled_.find(action_name); | ||
| return it != node_->reported_canceled_.end() && it->second; | ||
| } | ||
|
|
||
| } // namespace ros2_medkit_action_status_bridge | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
docs/api/rest.rst wasn't updated to match: the per-entity fault scope note (rest.rst:828-833) still promises REST consumers that reporting_sources "is the union of every app that has reported that code". Supersede breaks that invariant - a source can vanish between two GETs of the same fault, and a fault that 404'd under an entity can flip to 200 after it was already CONFIRMED (the exact flip test_fault_source_supersede.test.py asserts). rest.rst deserves the same paragraph this file got. While there, one sentence for event consumers would help: when the correction lands on an already-confirmed fault the only signal is a plain fault_updated (fault_manager_node.cpp:662), so a client building per-entity fault lists from fault_confirmed events alone never sees the fault move to the right entity.