diff --git a/testresults/src/org/labkey/testresults/TestResultsController.java b/testresults/src/org/labkey/testresults/TestResultsController.java index 199fb36f..345ff79e 100644 --- a/testresults/src/org/labkey/testresults/TestResultsController.java +++ b/testresults/src/org/labkey/testresults/TestResultsController.java @@ -111,6 +111,8 @@ import java.util.Date; import java.util.Enumeration; import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; import java.util.List; import java.util.Locale; import java.util.Map; @@ -133,6 +135,60 @@ public class TestResultsController extends SpringActionController private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(TestResultsController.class); + // Tab name constants for menu highlighting + public static class TabNames + { + // Request attribute name + public static final String ACTIVE_TAB_ATTR = "activeTab"; + // CSS classes + private static final String NAV_TAB_CLASS = "nav-tab"; + private static final String ACTIVE_CSS_CLASS = " active"; + + // Tab identifiers + public static final String OVERVIEW = "overview"; + public static final String USER = "user"; + public static final String RUN = "run"; + public static final String LONGTERM = "longterm"; + public static final String FLAGS = "flags"; + public static final String TRAINING_DATA = "trainingdata"; + public static final String ERRORS = "errors"; + + /** Returns CSS class for a tab link: "nav-tab" or "nav-tab active" */ + public static String getTabClass(String tabName, String activeTab) + { + return NAV_TAB_CLASS + (tabName.equals(activeTab) ? ACTIVE_CSS_CLASS : ""); + } + } + + // Form class for RetrainAllAction + public static class RetrainAllForm + { + private String _mode = "reset"; + private int _maxRuns = 20; + private int _minRuns = 5; + private Integer _targetRuns; // backwards compatibility + + public String getMode() { return _mode; } + public void setMode(String mode) { _mode = mode; } + + public int getMaxRuns() + { + // Support legacy targetRuns parameter + if (_targetRuns != null) + return _targetRuns; + return _maxRuns; + } + public void setMaxRuns(int maxRuns) { _maxRuns = maxRuns; } + + public int getMinRuns() { return _minRuns; } + public void setMinRuns(int minRuns) { _minRuns = minRuns; } + + public Integer getTargetRuns() { return _targetRuns; } + public void setTargetRuns(Integer targetRuns) { _targetRuns = targetRuns; } + + public boolean isIncremental() { return "incremental".equalsIgnoreCase(_mode); } + } + public TestResultsController() { setActionResolver(_actionResolver); @@ -1037,6 +1093,110 @@ public Object execute(Object o, BindException errors) } } + @RequiresSiteAdmin + public static class RetrainAllAction extends MutatingApiAction + { + @Override + public Object execute(RetrainAllForm form, BindException errors) + { + // Validate and clamp parameters + int maxRuns = Math.max(1, Math.min(100, form.getMaxRuns())); + int minRuns = Math.max(1, Math.min(maxRuns, form.getMinRuns())); + boolean incremental = form.isIncremental(); + + Container c = getContainer(); + String containerPath = c.getPath(); + int expectedDuration = containerPath.toLowerCase().contains("perf") ? 720 : 540; + + // Only look back 1.5x maxRuns days to avoid ancient data + int lookbackDays = (int) Math.ceil(maxRuns * 1.5); + java.sql.Timestamp cutoffDate = new java.sql.Timestamp( + System.currentTimeMillis() - (long) lookbackDays * 24 * 60 * 60 * 1000); + + TestResultsManager mgr = TestResultsManager.get(); + DbScope scope = TestResultsSchema.getSchema().getScope(); + + try (DbScope.Transaction transaction = scope.ensureTransaction()) + { + List allUserIds = mgr.getRecentUserIds(c, cutoffDate); + + if (!incremental) + { + mgr.deleteTrainRunsForContainer(c); + mgr.deleteUserDataForContainer(c); + } + + int usersRetrained = 0; + int totalTrainRuns = 0; + + for (int userId : allUserIds) + { + Set existingRunIds = mgr.getExistingTrainRunIds(userId, c); + List recentCleanRunIds = mgr.getCleanRunIds(userId, c, cutoffDate, expectedDuration); + + // Determine final set of trainrun IDs + List finalRunIds; + if (incremental && !existingRunIds.isEmpty()) + { + finalRunIds = mgr.getCandidateRunIds(userId, c, existingRunIds, recentCleanRunIds, maxRuns); + } + else + { + finalRunIds = recentCleanRunIds.size() > maxRuns + ? new ArrayList<>(recentCleanRunIds.subList(0, maxRuns)) + : new ArrayList<>(recentCleanRunIds); + } + + // Skip if total runs is below minimum threshold + if (finalRunIds.size() < minRuns) + continue; + + // Determine runs to add and remove + Set finalRunSet = new HashSet<>(finalRunIds); + List runsToAdd = new ArrayList<>(); + List runsToRemove = new ArrayList<>(); + + for (int runId : finalRunIds) + { + if (!existingRunIds.contains(runId)) + runsToAdd.add(runId); + } + for (int runId : existingRunIds) + { + if (!finalRunSet.contains(runId)) + runsToRemove.add(runId); + } + + mgr.removeTrainRuns(runsToRemove); + mgr.addTrainRuns(runsToAdd); + + boolean isActive = finalRunIds.size() >= maxRuns; + mgr.upsertUserData(userId, c, finalRunIds, isActive); + + usersRetrained++; + totalTrainRuns += runsToAdd.size(); + } + + transaction.commit(); + + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("success", true); + response.put("usersRetrained", usersRetrained); + response.put("totalTrainRuns", totalTrainRuns); + response.put("mode", form.getMode()); + return response; + } + catch (Exception e) + { + _log.error("Error in RetrainAllAction", e); + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("success", false); + response.put("error", e.getMessage()); + return response; + } + } + } + /** * action for posting test output as an xml file */ diff --git a/testresults/src/org/labkey/testresults/TestResultsManager.java b/testresults/src/org/labkey/testresults/TestResultsManager.java index 0398c6b1..605541a3 100644 --- a/testresults/src/org/labkey/testresults/TestResultsManager.java +++ b/testresults/src/org/labkey/testresults/TestResultsManager.java @@ -16,6 +16,20 @@ package org.labkey.testresults; +import org.labkey.api.data.Container; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * User: Yuval Boss, yuval(at)uw.edu * Date: 1/14/2015 @@ -33,4 +47,212 @@ public static TestResultsManager get() { return _instance; } -} \ No newline at end of file + + /** + * Get distinct user IDs from recent test runs in a container. + */ + public List getRecentUserIds(Container container, Timestamp cutoffDate) + { + DbScope scope = TestResultsSchema.getSchema().getScope(); + SQLFragment sql = new SQLFragment(); + sql.append( + "SELECT DISTINCT userid FROM " + TestResultsSchema.getTableInfoTestRuns() + + " WHERE container = ? AND posttime >= ?"); + sql.add(container.getEntityId()); + sql.add(cutoffDate); + + List userIds = new ArrayList<>(); + new SqlSelector(scope, sql).forEach(rs -> userIds.add(rs.getInt("userid"))); + return userIds; + } + + /** + * Get trainrun counts per user for a container. + */ + public Map getTrainRunCounts(Container container) + { + DbScope scope = TestResultsSchema.getSchema().getScope(); + SQLFragment sql = new SQLFragment(); + sql.append( + "SELECT r.userid, COUNT(tr.runid) as traincount" + + " FROM " + TestResultsSchema.getTableInfoTrain() + " tr" + + " JOIN " + TestResultsSchema.getTableInfoTestRuns() + " r ON tr.runid = r.id" + + " WHERE r.container = ?" + + " GROUP BY r.userid"); + sql.add(container.getEntityId()); + + Map counts = new HashMap<>(); + new SqlSelector(scope, sql).forEach(rs -> + counts.put(rs.getInt("userid"), rs.getInt("traincount"))); + return counts; + } + + /** + * Delete all trainruns for a container. + */ + public void deleteTrainRunsForContainer(Container container) + { + DbScope scope = TestResultsSchema.getSchema().getScope(); + SQLFragment sql = new SQLFragment(); + sql.append( + "DELETE FROM " + TestResultsSchema.getTableInfoTrain() + + " WHERE runid IN (SELECT id FROM " + TestResultsSchema.getTableInfoTestRuns() + + " WHERE container = ?)"); + sql.add(container.getEntityId()); + new SqlExecutor(scope).execute(sql); + } + + /** + * Delete all userdata for a container. + */ + public void deleteUserDataForContainer(Container container) + { + DbScope scope = TestResultsSchema.getSchema().getScope(); + SQLFragment sql = new SQLFragment(); + sql.append("DELETE FROM " + TestResultsSchema.getTableInfoUserData() + " WHERE container = ?"); + sql.add(container.getEntityId()); + new SqlExecutor(scope).execute(sql); + } + + /** + * Get existing trainrun IDs for a user in a container. + */ + public Set getExistingTrainRunIds(int userId, Container container) + { + DbScope scope = TestResultsSchema.getSchema().getScope(); + SQLFragment sql = new SQLFragment(); + sql.append( + "SELECT tr.runid FROM " + TestResultsSchema.getTableInfoTrain() + " tr" + + " JOIN " + TestResultsSchema.getTableInfoTestRuns() + " r ON tr.runid = r.id" + + " WHERE r.userid = ? AND r.container = ?"); + sql.add(userId); + sql.add(container.getEntityId()); + + Set runIds = new HashSet<>(); + new SqlSelector(scope, sql).forEach(rs -> runIds.add(rs.getInt("runid"))); + return runIds; + } + + /** + * Get clean run IDs for a user within lookback period. + * Clean = 0 failures, 0 leaks, passedtests > 0, not flagged, full duration, no hangs. + */ + public List getCleanRunIds(int userId, Container container, Timestamp cutoffDate, int expectedDuration) + { + DbScope scope = TestResultsSchema.getSchema().getScope(); + SQLFragment sql = new SQLFragment(); + sql.append( + "SELECT tr.id FROM " + TestResultsSchema.getTableInfoTestRuns() + " tr" + + " WHERE tr.userid = ? AND tr.container = ?" + + " AND tr.posttime >= ?" + + " AND tr.failedtests = 0 AND tr.leakedtests = 0" + + " AND tr.passedtests > 0 AND tr.flagged = false" + + " AND tr.duration >= ?" + + " AND NOT EXISTS (SELECT 1 FROM " + TestResultsSchema.getTableInfoHangs() + + " h WHERE h.testrunid = tr.id)" + + " ORDER BY tr.posttime DESC"); + sql.add(userId); + sql.add(container.getEntityId()); + sql.add(cutoffDate); + sql.add(expectedDuration); + + List runIds = new ArrayList<>(); + new SqlSelector(scope, sql).forEach(rs -> runIds.add(rs.getInt("id"))); + return runIds; + } + + /** + * Get combined candidate run IDs (existing + recent) sorted by posttime, limited to maxRuns. + */ + public List getCandidateRunIds(int userId, Container container, + Set existingIds, List recentIds, int maxRuns) + { + if (existingIds.isEmpty() && recentIds.isEmpty()) + return new ArrayList<>(); + + DbScope scope = TestResultsSchema.getSchema().getScope(); + SQLFragment sql = new SQLFragment(); + sql.append( + "SELECT id FROM " + TestResultsSchema.getTableInfoTestRuns() + + " WHERE userid = ? AND container = ?" + + " AND (id = ANY(?) OR id = ANY(?))" + + " ORDER BY posttime DESC"); + sql.add(userId); + sql.add(container.getEntityId()); + sql.add(existingIds.toArray(new Integer[0])); + sql.add(recentIds.toArray(new Integer[0])); + + List result = new ArrayList<>(); + new SqlSelector(scope, sql).forEach(rs -> { + if (result.size() < maxRuns) + result.add(rs.getInt("id")); + }); + return result; + } + + /** + * Delete trainruns by run IDs. + */ + public void removeTrainRuns(List runIds) + { + if (runIds.isEmpty()) + return; + + DbScope scope = TestResultsSchema.getSchema().getScope(); + for (int runId : runIds) + { + SQLFragment sql = new SQLFragment(); + sql.append("DELETE FROM " + TestResultsSchema.getTableInfoTrain() + " WHERE runid = ?"); + sql.add(runId); + new SqlExecutor(scope).execute(sql); + } + } + + /** + * Insert trainruns by run IDs. + */ + public void addTrainRuns(List runIds) + { + if (runIds.isEmpty()) + return; + + DbScope scope = TestResultsSchema.getSchema().getScope(); + for (int runId : runIds) + { + SQLFragment sql = new SQLFragment(); + sql.append("INSERT INTO " + TestResultsSchema.getTableInfoTrain() + " (runid) VALUES (?)"); + sql.add(runId); + new SqlExecutor(scope).execute(sql); + } + } + + /** + * Upsert userdata with calculated stats from specified run IDs. + */ + public void upsertUserData(int userId, Container container, List runIds, boolean active) + { + if (runIds.isEmpty()) + return; + + DbScope scope = TestResultsSchema.getSchema().getScope(); + SQLFragment sql = new SQLFragment(); + sql.append( + "INSERT INTO " + TestResultsSchema.getTableInfoUserData() + + " (userid, container, meantestsrun, meanmemory, stddevtestsrun, stddevmemory, active)" + + " SELECT ?, ?, avg(passedtests), avg(averagemem)," + + " stddev_pop(passedtests), stddev_pop(averagemem), ?" + + " FROM " + TestResultsSchema.getTableInfoTestRuns() + + " WHERE id = ANY(?)" + + " ON CONFLICT(userid, container) DO UPDATE SET" + + " meantestsrun = excluded.meantestsrun," + + " meanmemory = excluded.meanmemory," + + " stddevtestsrun = excluded.stddevtestsrun," + + " stddevmemory = excluded.stddevmemory," + + " active = excluded.active"); + sql.add(userId); + sql.add(container.getEntityId()); + sql.add(active); + sql.add(runIds.toArray(new Integer[0])); + new SqlExecutor(scope).execute(sql); + } +} diff --git a/testresults/src/org/labkey/testresults/view/errorFiles.jsp b/testresults/src/org/labkey/testresults/view/errorFiles.jsp index 015d84aa..7eb70766 100644 --- a/testresults/src/org/labkey/testresults/view/errorFiles.jsp +++ b/testresults/src/org/labkey/testresults/view/errorFiles.jsp @@ -24,6 +24,7 @@ Container c = getContainer(); %> +<% request.setAttribute(TestResultsController.TabNames.ACTIVE_TAB_ATTR, TestResultsController.TabNames.ERRORS); %> <%@include file="menu.jsp" %>

All the files listed below at one point or another failed to post. When a run is successfully posted through this page it gets removed from the list.

diff --git a/testresults/src/org/labkey/testresults/view/failureDetail.jsp b/testresults/src/org/labkey/testresults/view/failureDetail.jsp index e5986926..6537303b 100644 --- a/testresults/src/org/labkey/testresults/view/failureDetail.jsp +++ b/testresults/src/org/labkey/testresults/view/failureDetail.jsp @@ -188,6 +188,7 @@ } %> +<% request.setAttribute(TestResultsController.TabNames.ACTIVE_TAB_ATTR, ""); %> <%@include file="menu.jsp" %> + + diff --git a/testresults/src/org/labkey/testresults/view/multiFailureDetail.jsp b/testresults/src/org/labkey/testresults/view/multiFailureDetail.jsp index 6f457e75..573165d9 100644 --- a/testresults/src/org/labkey/testresults/view/multiFailureDetail.jsp +++ b/testresults/src/org/labkey/testresults/view/multiFailureDetail.jsp @@ -63,8 +63,8 @@ } %> +<% request.setAttribute(TestResultsController.TabNames.ACTIVE_TAB_ATTR, ""); %> <%@include file="menu.jsp" %> -
View Type: