Only restore jobIds added within last 3 days (#983)

* Only restore jobIds added within last 3 days * Remove unused exception variable e * Fix pylint errors * Refactor restore jobIds code Changed constant to more descriptive name, changed multi line text to use implicit joins, fixed arg type. * Fixed MonitorJobPlugin restore constants Previous commit mixed constant job threshold's limit with days, corrected it. Added comment description for timestamp in addJob.
AcademySoftwareFoundation · Dec 14, 2021 · 5122518 · 5122518
1 parent 7e05c4e
commit 5122518
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 16 deletions.
diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py
@@ -226,23 +226,44 @@ def setLoadMine(self, value):
  @type value: boolean or QtCore.Qt.Checked or QtCore.Qt.Unchecked"""
  self.__loadMine = (value is True or value == QtCore.Qt.Checked)
 
- def addJob(self, job):
+ def addJob(self, job, timestamp=None):
  """Adds a job to the list. With locking"
  @param job: Job can be None, a job object, or a job name.
- @type job: job, string, None"""
+ @type job: job, string, None
+ @param timestamp: UTC time of the specific date the job was
+ added to be monitored
+ @type timestamp: float"""
  newJobObj = cuegui.Utils.findJob(job)
  self.ticksLock.lock()
  try:
  if newJobObj:
- objectKey = cuegui.Utils.getObjectKey(newJobObj)
- self.__load[objectKey] = newJobObj
- self.__jobTimeLoaded[objectKey] = time.time()
+ jobKey = cuegui.Utils.getObjectKey(newJobObj)
+ if not self.__groupDependent:
+ self.__load[jobKey] = newJobObj
+ self.__jobTimeLoaded[jobKey] = timestamp if timestamp else time.time()
  finally:
  self.ticksLock.unlock()
 
  def getJobProxies(self):
- """Gets a list of IDs of monitored jobs."""
- return list(self._items.keys())
+ """Get a list of the JobProxies that are being monitored in the session
+ which will be saved to the config file
+
+ Returning a sorted list based on the most recent timestamp - restoring jobs is capped
+ by LOAD_LIMIT, so restore the most recent jobs the user added to their session
+
+ :return: list of tuples of the JobId and timestamp
+ """
+ jobIdsTimeLoaded = []
+
+ for jobProxy, _ in self._items.items():
+ try:
+ jobIdsTimeLoaded.append((jobProxy, self.__jobTimeLoaded[jobProxy]))
+ except KeyError:
+ # set timestamp to epoch time if timestamp not found
+ jobIdsTimeLoaded.append((jobProxy, 0))
+
+ # sort list on recent timestamps, only restoring the first n jobs (defined by LOAD_LIMIT)
+ return list(sorted(jobIdsTimeLoaded, key=lambda x: x[1], reverse=True))
 
  def _removeItem(self, item):
  """Removes an item from the TreeWidget without locking

diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py
@@ -22,6 +22,7 @@
 
 from builtins import str
 from builtins import map
+from datetime import datetime
 import re
 import weakref
 
@@ -46,7 +47,8 @@
 PLUGIN_DESCRIPTION = "Monitors a list of jobs"
 PLUGIN_PROVIDES = "MonitorJobsDockWidget"
 REGEX_EMPTY_STRING = re.compile("^$")
-
+JOB_RESTORE_THRESHOLD_DAYS = 3
+JOB_RESTORE_THRESHOLD_LIMIT = 200
 
 class MonitorJobsDockWidget(cuegui.AbstractDockWidget.AbstractDockWidget):
  """Plugin for listing active jobs and managing them."""
@@ -106,14 +108,36 @@ def getJobIds(self):
  return list(map(opencue.id, self.jobMonitor.getJobProxies()))
 
  def restoreJobIds(self, jobIds):
- """Monitors a list of jobs."""
- for jobId in jobIds:
- try:
- self.jobMonitor.addJob(jobId)
- except opencue.EntityNotFoundException:
- logger.warning("Unable to load previously loaded job since "
- "it was moved to the historical "
- "database: %s", jobId)
+ """Restore monitored jobs from previous saved state
+ Only load jobs that have a timestamp less than or equal to the time a job lives on the farm
+ (jobs are moved to historical database)
+
+ :param jobIds: monitored jobs ids and their timestamp from previous working state
+ (loaded from config.ini file)
+ ex: [("Job.f156be87-987a-48b9-b9da-774cd58674a3", 1612482716.170947),...
+ :type jobIds: list[tuples]
+ """
+ today = datetime.datetime.now()
+ limit = JOB_RESTORE_THRESHOLD_LIMIT if len(jobIds) > \
+ JOB_RESTORE_THRESHOLD_LIMIT else len(jobIds)
+ msg = ('Unable to load previously loaded job since it was moved '
+ 'to the historical database: {0}')
+
+ try:
+ for jobId, timestamp in jobIds[:limit]:
+ loggedTime = datetime.datetime.fromtimestamp(timestamp)
+ if (today - loggedTime).days <= JOB_RESTORE_THRESHOLD_DAYS:
+ try:
+ self.jobMonitor.addJob(jobId, timestamp)
+ except opencue.EntityNotFoundException:
+ logger.info(msg, jobId)
+ except ValueError:
+ # load older format
+ for jobId in jobIds[:limit]:
+ try:
+ self.jobMonitor.addJob(jobId)
+ except opencue.EntityNotFoundException:
+ logger.info(msg, jobId)
 
  def pluginRestoreState(self, saved_settings):
  """Called on plugin start with any previously saved state.