From 5e4c28254f8faa338930004ffa284fecddf977b5 Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Tue, 18 Aug 2026 17:42:19 -0600 Subject: [PATCH 1/4] Log EventDataIds through the SND event and attribute data ETL steps The _SND Event Data step clears attribute values and only the _SND Attribute Data step restores them, and each step picks its rows independently, so a mismatch silently leaves event data with no attributes. Logging the ids on both sides makes the gap visible in the job log. --- snd/src/org/labkey/snd/SNDManager.java | 27 +++++++++ .../labkey/snd/query/AttributeDataTable.java | 59 +++++++++++++++++-- .../org/labkey/snd/query/EventDataTable.java | 55 ++++++++++++++++- 3 files changed, 133 insertions(+), 8 deletions(-) diff --git a/snd/src/org/labkey/snd/SNDManager.java b/snd/src/org/labkey/snd/SNDManager.java index b6f37af5b..a7074ff96 100644 --- a/snd/src/org/labkey/snd/SNDManager.java +++ b/snd/src/org/labkey/snd/SNDManager.java @@ -105,6 +105,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TreeMap; @@ -153,6 +154,9 @@ public static UserSchema getSndUserSchemaAdminRole(Container c, User u) public static int MAX_MERGE_ROWS = 2000; + private static final int MAX_LOGGED_IDS = 5000; + private static final int LOGGED_IDS_PER_LINE = 250; + public static Logger getLogger(Map configParameters, Class clazz) { Logger log = null; @@ -164,6 +168,29 @@ public static Logger getLogger(Map configParameters, Class claz return log; } + /** + * Writes an id set to the ETL job log so that the id sets logged by different ETL steps of the same run can be + * diffed against each other. Chunked because a single line of thousands of ids is unreadable, and capped because + * the initial full data load would otherwise write the entire table to the log. + */ + public static void logIds(Logger log, String message, Collection ids) + { + log.info(message + " Count: " + ids.size() + "."); + + if (ids.isEmpty()) + return; + + if (ids.size() > MAX_LOGGED_IDS) + { + log.info("Id list omitted, more than " + MAX_LOGGED_IDS + " ids."); + return; + } + + List sorted = ids.stream().filter(Objects::nonNull).sorted().collect(Collectors.toList()); + for (List chunk : ListUtils.partition(sorted, LOGGED_IDS_PER_LINE)) + log.info(" " + StringUtils.join(chunk, ", ")); + } + public static String getPackageName(int id) { return PackageDomainKind.getPackageKindName() + "-" + id; diff --git a/snd/src/org/labkey/snd/query/AttributeDataTable.java b/snd/src/org/labkey/snd/query/AttributeDataTable.java index 81d343671..18f4ba029 100644 --- a/snd/src/org/labkey/snd/query/AttributeDataTable.java +++ b/snd/src/org/labkey/snd/query/AttributeDataTable.java @@ -149,6 +149,10 @@ public QueryUpdateService getUpdateService() protected class UpdateService extends SNDQueryUpdateService { + /** Bounds the source ordering check below. It costs one retained URI per distinct EventDataId, and an ungrouped source would otherwise warn once per row. */ + private static final int MAX_TRACKED_URIS = 50_000; + private static final int MAX_ORDER_WARNINGS = 10; + private final SNDManager _sndManager = SNDManager.get(); private final SNDService _sndService = SNDService.get(); private final DbSchema _expSchema = OntologyManager.getExpSchema(); @@ -230,7 +234,11 @@ private int insertObject(Container c, User u, String uri, List p private List> updateObjectProperty(User user, Container container, List> data, boolean isInsertOnly, boolean isUpdate, Logger logger) { - logger.info("Begin updating exp.ObjectProperty."); + Set incomingEventDataIds = new HashSet<>(); + for (Map row : data) + incomingEventDataIds.add((Integer) row.get("EventDataId")); + + SNDManager.logIds(logger, "Begin updating exp.ObjectProperty. Source rows: " + data.size() + ". EventDataIds in this batch:", incomingEventDataIds); int inserted = 0; @@ -240,6 +248,10 @@ private List> updateObjectProperty(User user, Container cont boolean found = false; Set cacheEventIds = new HashSet<>(); + Set writtenEventDataIds = new HashSet<>(); + Set flushedUris = new HashSet<>(); + boolean checkOrdering = true; + int outOfOrderFlushes = 0; for(Map row : data) { @@ -255,7 +267,8 @@ private List> updateObjectProperty(User user, Container cont //add to list of cached narrative rows to delete cacheEventIds.add((Integer) row.get("EventId")); - String objectURI = getObjectURI((Integer) row.get("EventDataId"), container); + Integer eventDataId = (Integer) row.get("EventDataId"); + String objectURI = getObjectURI(eventDataId, container); if (prevUri == null) prevUri = objectURI; @@ -320,10 +333,27 @@ else if (stringValue != null) if (!prevUri.equals(objectURI)) { inserted = insertObject(container, user, prevUri, prevObjProps, pkgId, inserted, logger); + + // Properties are only flushed when the URI changes, so a URI seen twice means the source + // did not arrive grouped by EventDataId and the ORDER BY in v_snd_attributeData was lost. + if (checkOrdering) + { + if (!flushedUris.add(prevUri) && ++outOfOrderFlushes <= MAX_ORDER_WARNINGS) + logger.warn("Source rows are not grouped by EventDataId; exp.ObjectProperty for {} was written in more than one pass.", prevUri); + + if (flushedUris.size() >= MAX_TRACKED_URIS) + { + logger.info("More than {} EventDataIds in this batch; ending the source ordering check.", MAX_TRACKED_URIS); + flushedUris.clear(); + checkOrdering = false; + } + } + prevUri = objectURI; prevObjProps = new ArrayList<>(); } prevObjProps.add(oprop); + writtenEventDataIds.add(eventDataId); } } @@ -332,12 +362,16 @@ else if (stringValue != null) } if (!found) { - throw new RuntimeException("Attribute metadata not found for key: '" + key + "' in package: " + pkgId); + throw new RuntimeException("Attribute metadata not found for key: '" + key + "' in package: " + pkgId + + ", EventDataId: " + eventDataId + ". Aborting, leaving all " + incomingEventDataIds.size() + + " EventDataIds in this batch with the attribute values the _SND Event Data step already cleared."); } } else { - throw new RuntimeException("Package metadata not found for package id: " + pkgId); + throw new RuntimeException("Package metadata not found for package id: " + pkgId + + ", EventDataId: " + eventDataId + ". Aborting, leaving all " + incomingEventDataIds.size() + + " EventDataIds in this batch with the attribute values the _SND Event Data step already cleared."); } } @@ -347,7 +381,22 @@ else if (stringValue != null) } OntologyManager.clearPropertyCache(); - logger.info("End updating exp.ObjectProperty. Inserted/Updated " + inserted + " rows."); + + SNDManager.logIds(logger, "End updating exp.ObjectProperty. Inserted/Updated " + inserted + " rows. EventDataIds written:", writtenEventDataIds); + + if (outOfOrderFlushes > MAX_ORDER_WARNINGS) + logger.warn("{} objectURIs in total were written in more than one pass; further warnings were suppressed.", outOfOrderFlushes); + + // Collect only the misses; copying the incoming set would double its footprint on a full load. + Set unwritten = new HashSet<>(); + for (Integer id : incomingEventDataIds) + { + if (!writtenEventDataIds.contains(id)) + unwritten.add(id); + } + + if (!unwritten.isEmpty()) + SNDManager.logIds(logger, "EventDataIds present in the source rows but left with no attribute values written:", unwritten); _sndManager.updateNarrativeCache(container, user, cacheEventIds, logger); diff --git a/snd/src/org/labkey/snd/query/EventDataTable.java b/snd/src/org/labkey/snd/query/EventDataTable.java index 1c9bce9d6..d6ca1ad6f 100644 --- a/snd/src/org/labkey/snd/query/EventDataTable.java +++ b/snd/src/org/labkey/snd/query/EventDataTable.java @@ -25,6 +25,7 @@ import org.labkey.api.data.JdbcType; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; import org.labkey.api.dataiterator.DataIteratorBuilder; import org.labkey.api.dataiterator.DataIteratorContext; @@ -48,7 +49,9 @@ import java.io.IOException; import java.sql.SQLException; +import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -108,6 +111,9 @@ public QueryUpdateService getUpdateService() protected static class UpdateService extends SNDQueryUpdateService { + /** Keeps the ObjectURI IN clause well under the SQL Server parameter limit. */ + private static final int URI_CHUNK_SIZE = 500; + private final SNDManager _sndManager = SNDManager.get(); private final SNDService _sndService = SNDService.get(); private final DbSchema _expSchema = OntologyManager.getExpSchema(); @@ -122,6 +128,36 @@ private String getObjectURI(Integer eventDataId, Container c) return _sndManager.generateLsid(c, String.valueOf(eventDataId)); } + /** + * EventDataIds in this batch whose exp.Object currently carries attribute values. Deleting the exp.Object row + * cascades to exp.ObjectProperty, so these are the values the merge destroys; only the _SND Attribute Data ETL + * step re-inserts them, and it computes its incremental window independently of this step's. + */ + private Set getEventDataIdsWithAttributeData(Container container, Map eventDataIdsByUri) + { + Set withAttributeData = new HashSet<>(); + List uris = new ArrayList<>(eventDataIdsByUri.keySet()); + + for (int i = 0; i < uris.size(); i += URI_CHUNK_SIZE) + { + List chunk = uris.subList(i, Math.min(i + URI_CHUNK_SIZE, uris.size())); + + // EXISTS rather than a join: both indexes (UQ_Object on ObjectURI, PK_ObjectProperty on ObjectId) + // are seeks, and the semi-join stops at the first property instead of reading all of them per object. + SQLFragment sql = new SQLFragment("SELECT o.ObjectURI FROM ") + .append(OntologyManager.getTinfoObject(), "o") + .append(" WHERE o.Container = ?").add(container.getId()) + .append(" AND EXISTS (SELECT 1 FROM ").append(OntologyManager.getTinfoObjectProperty(), "op") + .append(" WHERE op.ObjectId = o.ObjectId)") + .append(" AND o.ObjectURI").appendInClause(chunk, _expSchema.getSqlDialect()); + + new SqlSelector(_expSchema, sql).getCollection(String.class) + .forEach(uri -> withAttributeData.add(eventDataIdsByUri.get(uri))); + } + + return withAttributeData; + } + @Override public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) @@ -158,14 +194,27 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B log.info("Merging rows."); log.info("Begin updating exp.Object table."); - int count = 0; - for(Map map : data) + + Map eventDataIdsByUri = new LinkedHashMap<>(); + for (Map map : data) { - String objectURI = getObjectURI((Integer) map.get("EventDataId"), container); + Integer eventDataId = (Integer) map.get("EventDataId"); + String objectURI = getObjectURI(eventDataId, container); //update snd.EventData row with objectURI map.put("ObjectURI", objectURI); + eventDataIdsByUri.put(objectURI, eventDataId); + } + + SNDManager.logIds(log, "Attribute values about to be cleared by this merge; the _SND Attribute Data step must re-insert them.", + getEventDataIdsWithAttributeData(container, eventDataIdsByUri)); + + int count = 0; + for(Map map : data) + { + String objectURI = (String) map.get("ObjectURI"); + //delete row from exp.Object OntologyManager.deleteOntologyObjects(container, objectURI); From 23c45e35e3f4757ac80fa4de4929df0ccbf887a8 Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Tue, 18 Aug 2026 19:24:10 -0600 Subject: [PATCH 2/4] Keep the SND attribute-clearing diagnostic off the merge critical path The exp.Object lookup added for this branch's id logging ran unguarded inside mergeRows, so a failure reading it would abort a merge that previously did no reads there at all. It now runs inside logAttributeDataToBeCleared, which logs a warning and lets the merge continue. The lookup is also skipped once the batch exceeds MAX_LOGGED_IDS, the same threshold above which logIds suppresses the id list, capping the chunked queries at ten round trips instead of the hundreds a full merge batch would spend to produce a bare count. MAX_LOGGED_IDS is public now so both sites share one constant. --- snd/src/org/labkey/snd/SNDManager.java | 2 +- .../org/labkey/snd/query/EventDataTable.java | 26 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/snd/src/org/labkey/snd/SNDManager.java b/snd/src/org/labkey/snd/SNDManager.java index a7074ff96..9a2e43805 100644 --- a/snd/src/org/labkey/snd/SNDManager.java +++ b/snd/src/org/labkey/snd/SNDManager.java @@ -154,7 +154,7 @@ public static UserSchema getSndUserSchemaAdminRole(Container c, User u) public static int MAX_MERGE_ROWS = 2000; - private static final int MAX_LOGGED_IDS = 5000; + public static final int MAX_LOGGED_IDS = 5000; private static final int LOGGED_IDS_PER_LINE = 250; public static Logger getLogger(Map configParameters, Class clazz) diff --git a/snd/src/org/labkey/snd/query/EventDataTable.java b/snd/src/org/labkey/snd/query/EventDataTable.java index d6ca1ad6f..dfe340a63 100644 --- a/snd/src/org/labkey/snd/query/EventDataTable.java +++ b/snd/src/org/labkey/snd/query/EventDataTable.java @@ -158,6 +158,29 @@ private Set getEventDataIdsWithAttributeData(Container container, Map eventDataIdsByUri, Logger log) + { + if (eventDataIdsByUri.size() > SNDManager.MAX_LOGGED_IDS) + { + log.info("More than " + SNDManager.MAX_LOGGED_IDS + " EventDataIds in this batch; skipping the check for attribute values about to be cleared."); + return; + } + + try + { + SNDManager.logIds(log, "Attribute values about to be cleared by this merge; the _SND Attribute Data step must re-insert them.", + getEventDataIdsWithAttributeData(container, eventDataIdsByUri)); + } + catch (Exception e) + { + log.warn("Could not determine which EventDataIds have attribute values; continuing with the merge.", e); + } + } + @Override public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) @@ -207,8 +230,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B eventDataIdsByUri.put(objectURI, eventDataId); } - SNDManager.logIds(log, "Attribute values about to be cleared by this merge; the _SND Attribute Data step must re-insert them.", - getEventDataIdsWithAttributeData(container, eventDataIdsByUri)); + logAttributeDataToBeCleared(container, eventDataIdsByUri, log); int count = 0; for(Map map : data) From 74267b5d51bfdf492081862fd1341daa8ed1f3fe Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Tue, 18 Aug 2026 20:53:01 -0600 Subject: [PATCH 3/4] Name the EventDataId in the skipped-attribute-value log lines These two lines carried the property name but nothing identifying the row, so a single attribute dropped from an otherwise healthy EventDataId could not be joined back to the id sets the rest of this branch logs. Per-attribute loss was invisible to every one of them, since they all reconcile at EventDataId granularity and an id with any resolving value counts as written. --- snd/src/org/labkey/snd/query/AttributeDataTable.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snd/src/org/labkey/snd/query/AttributeDataTable.java b/snd/src/org/labkey/snd/query/AttributeDataTable.java index 18f4ba029..f666f6a76 100644 --- a/snd/src/org/labkey/snd/query/AttributeDataTable.java +++ b/snd/src/org/labkey/snd/query/AttributeDataTable.java @@ -305,11 +305,11 @@ else if (stringValue != null) { if (pd.getLookupSchema() != null && pd.getLookupQuery() != null) { - logger.info("Value null for property " + pd.getName() + ". Value skipped. Verify lookup " + pd.getLookupSchema() + "." + pd.getLookupQuery() + " contains " + stringValue); + logger.info("Value null for property " + pd.getName() + ", EventDataId: " + eventDataId + ". Value skipped. Verify lookup " + pd.getLookupSchema() + "." + pd.getLookupQuery() + " contains " + stringValue); } else { - logger.info("Value null for property " + pd.getName() + ". Value skipped."); + logger.info("Value null for property " + pd.getName() + ", EventDataId: " + eventDataId + ". Value skipped."); } } From dbd6710eec923c410f394f25cde9c0bdf1f77de7 Mon Sep 17 00:00:00 2001 From: Marty Pradere Date: Thu, 20 Aug 2026 09:22:40 -0600 Subject: [PATCH 4/4] Log EventDataIds on the non-merge paths and drop diagnostics to debug The event data step logged ids only from mergeRows, and only the subset that already carried attribute values, so there was nothing to diff against the id sets the attribute data step logs. importRows and deleteRows now log theirs as well: a newly inserted row the _SND Attribute Data step misses ends up just as stripped as a cleared one, and a deleted row otherwise reads as an attribute data miss. Everything this branch adds now logs at debug, which the ETL job logger runs at by default. The pre-existing Begin/End updating exp.ObjectProperty lines are back at info, and the two diagnostics that cost real work - the exp.Object lookup for values about to be cleared, and the source ordering check - are skipped entirely when debug is off. MAX_LOGGED_IDS drops to 2000, below the 5000 row ETL batch size, so the full batches of an initial load suppress the id lists while the smaller batches of an incremental run keep them. --- snd/src/org/labkey/snd/SNDManager.java | 12 ++++++--- .../labkey/snd/query/AttributeDataTable.java | 18 ++++++++----- .../org/labkey/snd/query/EventDataTable.java | 26 ++++++++++++++++--- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/snd/src/org/labkey/snd/SNDManager.java b/snd/src/org/labkey/snd/SNDManager.java index 9a2e43805..eb24300d5 100644 --- a/snd/src/org/labkey/snd/SNDManager.java +++ b/snd/src/org/labkey/snd/SNDManager.java @@ -154,7 +154,8 @@ public static UserSchema getSndUserSchemaAdminRole(Container c, User u) public static int MAX_MERGE_ROWS = 2000; - public static final int MAX_LOGGED_IDS = 5000; + /** Below the ETL batch size so that the full batches of an initial load suppress the id lists while the smaller batches of an incremental run keep them. */ + public static final int MAX_LOGGED_IDS = 2000; private static final int LOGGED_IDS_PER_LINE = 250; public static Logger getLogger(Map configParameters, Class clazz) @@ -175,20 +176,23 @@ public static Logger getLogger(Map configParameters, Class claz */ public static void logIds(Logger log, String message, Collection ids) { - log.info(message + " Count: " + ids.size() + "."); + if (!log.isDebugEnabled()) + return; + + log.debug(message + " Count: " + ids.size() + "."); if (ids.isEmpty()) return; if (ids.size() > MAX_LOGGED_IDS) { - log.info("Id list omitted, more than " + MAX_LOGGED_IDS + " ids."); + log.debug("Id list omitted, more than " + MAX_LOGGED_IDS + " ids."); return; } List sorted = ids.stream().filter(Objects::nonNull).sorted().collect(Collectors.toList()); for (List chunk : ListUtils.partition(sorted, LOGGED_IDS_PER_LINE)) - log.info(" " + StringUtils.join(chunk, ", ")); + log.debug(" " + StringUtils.join(chunk, ", ")); } public static String getPackageName(int id) diff --git a/snd/src/org/labkey/snd/query/AttributeDataTable.java b/snd/src/org/labkey/snd/query/AttributeDataTable.java index f666f6a76..e810cd1d7 100644 --- a/snd/src/org/labkey/snd/query/AttributeDataTable.java +++ b/snd/src/org/labkey/snd/query/AttributeDataTable.java @@ -149,7 +149,7 @@ public QueryUpdateService getUpdateService() protected class UpdateService extends SNDQueryUpdateService { - /** Bounds the source ordering check below. It costs one retained URI per distinct EventDataId, and an ungrouped source would otherwise warn once per row. */ + /** Bounds the source ordering check below. It costs one retained URI per distinct EventDataId, and an ungrouped source would otherwise log once per row. */ private static final int MAX_TRACKED_URIS = 50_000; private static final int MAX_ORDER_WARNINGS = 10; @@ -234,11 +234,13 @@ private int insertObject(Container c, User u, String uri, List p private List> updateObjectProperty(User user, Container container, List> data, boolean isInsertOnly, boolean isUpdate, Logger logger) { + logger.info("Begin updating exp.ObjectProperty."); + Set incomingEventDataIds = new HashSet<>(); for (Map row : data) incomingEventDataIds.add((Integer) row.get("EventDataId")); - SNDManager.logIds(logger, "Begin updating exp.ObjectProperty. Source rows: " + data.size() + ". EventDataIds in this batch:", incomingEventDataIds); + SNDManager.logIds(logger, "Source rows: " + data.size() + ". EventDataIds in this batch:", incomingEventDataIds); int inserted = 0; @@ -250,7 +252,7 @@ private List> updateObjectProperty(User user, Container cont Set cacheEventIds = new HashSet<>(); Set writtenEventDataIds = new HashSet<>(); Set flushedUris = new HashSet<>(); - boolean checkOrdering = true; + boolean checkOrdering = logger.isDebugEnabled(); int outOfOrderFlushes = 0; for(Map row : data) @@ -339,11 +341,11 @@ else if (stringValue != null) if (checkOrdering) { if (!flushedUris.add(prevUri) && ++outOfOrderFlushes <= MAX_ORDER_WARNINGS) - logger.warn("Source rows are not grouped by EventDataId; exp.ObjectProperty for {} was written in more than one pass.", prevUri); + logger.debug("Source rows are not grouped by EventDataId; exp.ObjectProperty for {} was written in more than one pass.", prevUri); if (flushedUris.size() >= MAX_TRACKED_URIS) { - logger.info("More than {} EventDataIds in this batch; ending the source ordering check.", MAX_TRACKED_URIS); + logger.debug("More than {} EventDataIds in this batch; ending the source ordering check.", MAX_TRACKED_URIS); flushedUris.clear(); checkOrdering = false; } @@ -382,10 +384,12 @@ else if (stringValue != null) OntologyManager.clearPropertyCache(); - SNDManager.logIds(logger, "End updating exp.ObjectProperty. Inserted/Updated " + inserted + " rows. EventDataIds written:", writtenEventDataIds); + logger.info("End updating exp.ObjectProperty. Inserted/Updated " + inserted + " rows."); + + SNDManager.logIds(logger, "EventDataIds written:", writtenEventDataIds); if (outOfOrderFlushes > MAX_ORDER_WARNINGS) - logger.warn("{} objectURIs in total were written in more than one pass; further warnings were suppressed.", outOfOrderFlushes); + logger.debug("{} objectURIs in total were written in more than one pass; further messages were suppressed.", outOfOrderFlushes); // Collect only the misses; copying the incoming set would double its footprint on a full load. Set unwritten = new HashSet<>(); diff --git a/snd/src/org/labkey/snd/query/EventDataTable.java b/snd/src/org/labkey/snd/query/EventDataTable.java index dfe340a63..ca832e3b0 100644 --- a/snd/src/org/labkey/snd/query/EventDataTable.java +++ b/snd/src/org/labkey/snd/query/EventDataTable.java @@ -164,9 +164,12 @@ private Set getEventDataIdsWithAttributeData(Container container, Map eventDataIdsByUri, Logger log) { + if (!log.isDebugEnabled()) + return; + if (eventDataIdsByUri.size() > SNDManager.MAX_LOGGED_IDS) { - log.info("More than " + SNDManager.MAX_LOGGED_IDS + " EventDataIds in this batch; skipping the check for attribute values about to be cleared."); + log.debug("More than " + SNDManager.MAX_LOGGED_IDS + " EventDataIds in this batch; skipping the check for attribute values about to be cleared."); return; } @@ -177,7 +180,7 @@ private void logAttributeDataToBeCleared(Container container, Map eventDataIds = new HashSet<>(); for(Map map : data) { - String objectURI = getObjectURI((Integer) map.get("EventDataId"), container); + Integer eventDataId = (Integer) map.get("EventDataId"); + String objectURI = getObjectURI(eventDataId, container); //update snd.EventData row with objectURI map.put("ObjectURI", objectURI); @@ -299,6 +305,8 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, //add to list of cached narrative rows to delete cacheData.add((Integer) map.get("EventId")); + eventDataIds.add(eventDataId); + count++; //TODO: Count in exp.Object is not going to be the same as in snd.EventData - need to figure out how to get the count to log if(count % 1000 == 0) @@ -306,6 +314,9 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, } log.info("End inserting into exp.Object. Inserted total of " + count + " rows."); + // These rows get a fresh exp.Object with no properties, so they depend on the _SND Attribute Data step just as much as the merged ones do. + SNDManager.logIds(log, "EventDataIds inserted into snd.EventData by this batch:", eventDataIds); + DataIteratorBuilder rowsWithObjectURI = new ListofMapsDataIterator.Builder(data.get(0).keySet(), data); _sndManager.updateNarrativeCache(container, user, cacheData, log); @@ -405,11 +416,15 @@ private void deleteFromExpTables(List> oldRows, Container co { log.info("Begin deleting from exp.ObjectProperty and exp.Object."); int count = 0; + Set eventDataIds = new HashSet<>(); //This will be a cascading delete across exp.ObjectProperty, exp.Object, and snd.EventData for (Map map : oldRows) { - String objectURI = getObjectURI((Integer) map.get("EventDataId"), container); + Integer eventDataId = (Integer) map.get("EventDataId"); + String objectURI = getObjectURI(eventDataId, container); + + eventDataIds.add(eventDataId); OntologyObject obj = OntologyManager.getOntologyObject(container, objectURI); //delete row from exp.ObjectProperty @@ -426,6 +441,9 @@ private void deleteFromExpTables(List> oldRows, Container co } log.info("End deleting from exp.ObjectProperty and exp.Object. Deleted total of " + count + " rows."); + + // Without these the deleted rows read as attribute data the _SND Attribute Data step failed to write. + SNDManager.logIds(log, "EventDataIds deleted from snd.EventData by this batch:", eventDataIds); } private int deleteAllFromExpTables(Logger log)