Test pipeline fixes and refresh. (#1305)

AcademySoftwareFoundation · Jul 13, 2023 · 6703e07 · 6703e07
1 parent d2057d3
commit 6703e07
Show file tree

Hide file tree

Showing 7 changed files with 85 additions and 58 deletions.
diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml
@@ -7,90 +7,57 @@ on:
  branches: [ master ]
 
 jobs:
- test_python_2019:
- name: Run Python Unit Tests (CY2019)
- runs-on: ubuntu-latest
- container: aswf/ci-opencue:2019
- steps:
- - uses: actions/checkout@v3
- - name: Run Python Tests
- run: ci/run_python_tests.sh
-
- test_cuebot_2019:
- name: Build Cuebot and Run Unit Tests (CY2019)
- runs-on: ubuntu-latest
- container:
- image: aswf/ci-opencue:2019
- steps:
- - uses: actions/checkout@v3
- - name: Build with Gradle
- run: |
- chown -R aswfuser:aswfgroup .
- su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
-
- test_python_2020:
- name: Run Python Unit Tests (CY2020)
+ test_python_2022:
+ name: Run Python Unit Tests (CY2022)
  runs-on: ubuntu-latest
- container: aswf/ci-opencue:2020
+ container: aswf/ci-opencue:2022
  steps:
  - uses: actions/checkout@v3
  - name: Run Python Tests
- run: ci/run_python_tests.sh
+ run: ci/run_python_tests.sh --no-gui
 
- test_cuebot_2020:
- name: Build Cuebot and Run Unit Tests (CY2020)
+ test_cuebot_2022:
+ name: Build Cuebot and Run Unit Tests (CY2022)
  runs-on: ubuntu-latest
  container:
- image: aswf/ci-opencue:2020
+ image: aswf/ci-opencue:2022
  steps:
  - uses: actions/checkout@v3
  - name: Build with Gradle
  run: |
  chown -R aswfuser:aswfgroup .
  su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
 
- test_python_2021:
- name: Run Python Unit Tests (CY2021)
+ test_python_2023:
+ name: Run Python Unit Tests (CY2023)
  runs-on: ubuntu-latest
- container: aswf/ci-opencue:2021
+ container: aswf/ci-opencue:2023
  steps:
  - uses: actions/checkout@v3
  - name: Run Python Tests
  run: ci/run_python_tests.sh
 
- test_cuebot_2021:
- name: Build Cuebot and Run Unit Tests (CY2021)
+ test_cuebot_2023:
+ name: Build Cuebot and Run Unit Tests (CY2023)
  runs-on: ubuntu-latest
  container:
- image: aswf/ci-opencue:2021
+ image: aswf/ci-opencue:2023
  steps:
  - uses: actions/checkout@v3
  - name: Build with Gradle
  run: |
  chown -R aswfuser:aswfgroup .
  su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
 
- test_python_2022:
- name: Run Python Unit Tests (CY2022)
+ test_python2:
+ name: Run Python Unit Tests using Python2
  runs-on: ubuntu-latest
- container: aswf/ci-opencue:2022
+ container: aswf/ci-opencue:2019
  steps:
  - uses: actions/checkout@v3
  - name: Run Python Tests
  run: ci/run_python_tests.sh
 
- test_cuebot_2022:
- name: Build Cuebot and Run Unit Tests (CY2022)
- runs-on: ubuntu-latest
- container:
- image: aswf/ci-opencue:2022
- steps:
- - uses: actions/checkout@v3
- - name: Build with Gradle
- run: |
- chown -R aswfuser:aswfgroup .
- su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
-
  test_pyside6:
  name: Run CueGUI Tests using PySide6
  runs-on: ubuntu-latest
@@ -113,7 +80,7 @@ jobs:
  name: Test Documentation Build
  runs-on: ubuntu-latest
  container:
- image: aswf/ci-opencue:2020
+ image: aswf/ci-opencue:2023
  steps:
  - uses: actions/checkout@v3
  - name: Run Sphinx build

diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh
@@ -1,7 +1,13 @@
 #!/bin/bash
 
+# Script for running OpenCue unit tests with PySide2.
+#
+# This script is written to be run within the OpenCue GitHub Actions environment.
+# See `.github/workflows/testing-pipeline.yml`.
+
 set -e
 
+args=("$@")
 python_version=$(python -V 2>&1)
 echo "Will run tests using ${python_version}"
 
@@ -23,6 +29,6 @@ PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test
 python rqd/setup.py test
 
 # Xvfb no longer supports Python 2.
-if [[ "$python_version" =~ "Python 3" ]]; then
+if [[ "$python_version" =~ "Python 3" && ${args[0]} != "--no-gui" ]]; then
  ci/run_gui_test.sh
 fi
diff --git a/ci/run_python_tests_pyside6.sh b/ci/run_python_tests_pyside6.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Script for running OpenCue unit tests with PySide6.
+#
+# This script is written to be run within the OpenCue GitHub Actions environment.
+# See `.github/workflows/testing-pipeline.yml`.
+
+set -e
+
+python_version=$(python -V 2>&1)
+echo "Will run tests using ${python_version}"
+
+# NOTE: To run this in an almalinux environment, install these packages:
+# yum -y install \
+# dbus-libs \
+# fontconfig \
+# gcc \
+# libxkbcommon-x11 \
+# mesa-libEGL-devel \
+# python-devel \
+# which \
+# xcb-util-keysyms \
+# xcb-util-image \
+# xcb-util-renderutil \
+# xcb-util-wm \
+# Xvfb
+
+# Install Python requirements.
+python3 -m pip install --user -r requirements.txt -r requirements_gui.txt
+# Replace PySide2 with PySide6.
+python3 -m pip uninstall -y PySide2
+python3 -m pip install --user PySide6==6.3.2
+
+# Protos need to have their Python code generated in order for tests to pass.
+python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
+python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
+
+# Fix compiled proto code for Python 3.
+2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
+
+python pycue/setup.py test
+PYTHONPATH=pycue python pyoutline/setup.py test
+PYTHONPATH=pycue python cueadmin/setup.py test
+PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test
+python rqd/setup.py test
+
+ci/run_gui_test.sh
diff --git a/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql b/cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql
@@ -1,6 +1,6 @@
 Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, true, true, true);
 
-Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0)
+Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0);
 
 Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000000', 'test');
 

diff --git a/cuegui/tests/Constants_tests.py b/cuegui/tests/Constants_tests.py
@@ -25,7 +25,7 @@
 
 import mock
 import pyfakefs.fake_filesystem_unittest
-from PySide2 import QtGui
+from qtpy import QtGui
 
 import opencue
 import cuegui.Constants
@@ -40,6 +40,7 @@
 '''
 
 
+# pylint: disable=import-outside-toplevel,redefined-outer-name,reimported
 class ConstantsTests(pyfakefs.fake_filesystem_unittest.TestCase):
  def setUp(self):
  self.setUpPyfakefs()
@@ -53,6 +54,7 @@ def test__should_load_user_config_from_env_var(self):
  self.fs.create_file(config_file_path, contents=CONFIG_YAML)
  os.environ['CUEGUI_CONFIG_FILE'] = config_file_path
 
+ import cuegui.Constants
  result = importlib.reload(cuegui.Constants)
 
  self.assertEqual('98.707.68', result.VERSION)
@@ -65,6 +67,7 @@ def test__should_load_user_config_from_user_profile(self):
  config_file_path = '/home/username/.config/opencue/cuegui.yaml'
  self.fs.create_file(config_file_path, contents=CONFIG_YAML)
 
+ import cuegui.Constants
  result = importlib.reload(cuegui.Constants)
 
  self.assertEqual('98.707.68', result.VERSION)
@@ -73,6 +76,7 @@ def test__should_load_user_config_from_user_profile(self):
 
  @mock.patch('platform.system', new=mock.Mock(return_value='Linux'))
  def test__should_use_default_values(self):
+ import cuegui.Constants
  result = importlib.reload(cuegui.Constants)
 
  self.assertNotEqual('98.707.68', result.VERSION)
@@ -161,6 +165,7 @@ def test__should_use_default_values(self):
 
  @mock.patch('platform.system', new=mock.Mock(return_value='Darwin'))
  def test__should_use_mac_editor(self):
+ import cuegui.Constants
  result = importlib.reload(cuegui.Constants)
 
  self.assertEqual('open -t', result.DEFAULT_EDITOR)
diff --git a/requirements.txt b/requirements.txt
@@ -12,7 +12,8 @@ packaging==20.9
 pathlib==1.0.1;python_version<"3.4"
 protobuf==3.17.3;python_version<"3.0"
 psutil==5.6.7
-pyfakefs==3.6
+pyfakefs==3.6;python_version<"3.7"
+pyfakefs==5.2.3;python_version>="3.7"
 pylint==2.6.0;python_version>="3.7"
 pynput==1.7.6
 PyYAML==5.1

diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py
@@ -1181,8 +1181,10 @@ def pipe_to_file(stdout, stderr, outfile):
  outfile.flush()
  os.fsync(outfile)
 
+ # pylint: disable=import-outside-toplevel
  import select
  import errno
+ # pylint: enable=import-outside-toplevel
 
  fd2file = {}
  fd2output = {}
@@ -1213,8 +1215,7 @@ def print_and_flush_ln(fd, last_timestamp):
  # There are no line breaks
  if len(lines) < 2:
  return curr_line_timestamp
- else:
- next_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S")
+ next_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S")
 
  remainder = lines[-1]
  for line in lines[0:-1]:
@@ -1225,8 +1226,7 @@ def print_and_flush_ln(fd, last_timestamp):
 
  if next_line_timestamp is None:
  return curr_line_timestamp
- else:
- return next_line_timestamp
+ return next_line_timestamp
 
  def translate_newlines(data):
  data = data.decode("utf-8", "ignore")