[neon/ubuntu-core/ago/ui-tests] selenium: Provide scripts to install on host, and sample testsuite
Antoine Gonzalez
null at kde.org
Tue Apr 30 14:52:31 BST 2024
Git commit d69947b18e38e1346cfec396360a2845f27950de by Antoine Gonzalez.
Committed on 30/04/2024 at 13:52.
Pushed by daspood into branch 'ago/ui-tests'.
Provide scripts to install on host, and sample testsuite
D +0 -61 selenium/Dockerfile
M +10 -4 selenium/README.md
A +27 -0 selenium/install_selenium.sh
A +117 -0 selenium/kcalc_24_tests.py
D +0 -72 selenium/run.sh
A +17 -0 selenium/run_selenium.sh
https://invent.kde.org/neon/ubuntu-core/-/commit/d69947b18e38e1346cfec396360a2845f27950de
diff --git a/selenium/Dockerfile b/selenium/Dockerfile
deleted file mode 100644
index b53457c..0000000
--- a/selenium/Dockerfile
+++ /dev/null
@@ -1,61 +0,0 @@
-# Ubuntu 22.04, 23.10, 24.04 do not work
-FROM ubuntu:22.04
-WORKDIR /seleniumtest
-
-# Name of the application contained in the testsnap.snap. We need it to manually connect kf6-core22.
-# `docker build --build-arg SNAP_NAME=kcalc .`
-#ARG SNAP_NAME=kcalc
-
-ENV DEBIAN_FRONTEND=noninteractive
-ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
-ENV container docker
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-
-RUN apt update \
- && apt upgrade -y \
- && apt install -y cmake gcc g++ git ruby dh-python dh-virtualenv debhelper-compat sed \
- python3-pip python3-all-dev python3-pyatspi python3-cairo python3-gi-cairo python3-virtualenv \
- && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 444DABCF3667D0283F894EDDE6D4736255751E5D \
- && echo "deb [ arch=amd64,arm64 ] https://archive.neon.kde.org/user jammy main" | tee /etc/apt/sources.list.d/kde-neon.list \
- && apt update \
- && apt install -y accerciser at-spi2-core gobject-introspection \
- libgirepository1.0-dev plasma-wayland-protocols libcairo2-dev libpipewire-0.3-dev \
- libgirepository-1.0-1 libcairo2 \
- pkg-kde-tools-neon kf6-extra-cmake-modules kf6-kcoreaddons-dev kf6-kwindowsystem-dev kwayland6-dev qt6-base-dev qt6-base-private-dev qt6-wayland-dev libkpipewire-dev \
- kf6-kcoreaddons kf6-kwindowsystem kwayland6 qt6-base qt6-wayland libqt6core6 libqt6dbus6 libqt6waylandclient6 libkpipewire6
-
-# Installing selenium
-RUN cd /tmp \
- && git clone https://invent.kde.org/sdk/selenium-webdriver-at-spi.git \
- && cd selenium-webdriver-at-spi \
- && sed -i 's/ORG_KDE_KWIN_FAKE_INPUT_DESTROY_SINCE_VERSION/ORG_KDE_KWIN_FAKE_INPUT_BUTTON_SINCE_VERSION/g' inputsynth/interaction.cpp \
- && pip3 install -r requirements.txt \
- && echo export PATH="~/.local/bin:$PATH" >> ${HOME}/.bashrc \
- && mkdir build \
- && cd build \
- && cmake -DQT_MAJOR_VERSION=6 -DBUILD_WITH_QT6=ON .. \
- && make \
- && make install
-
-# Copying test files
-COPY ./testsnap.snap /seleniumtest/
-COPY ./testsuite.py /seleniumtest/
-RUN chmod +x testsuite.py
-
-# Attempt to get snapd to run in the final container
-RUN apt install -y fuse snapd squashfuse sudo init \
- && apt clean \
- && dpkg-divert --local --rename --add /sbin/udevadm \
- && ln -s /bin/true /sbin/udevadm \
- && systemctl enable snapd
-VOLUME ["/sys/fs/cgroup"]
-STOPSIGNAL SIGRTMIN+3
-CMD ["/sbin/init"]
-
-# Ideally we would only want to run this, but snapd cannot run without systemd
-#RUN apt install -y snapd
-#RUN snap install kf6-core22
-#RUN snap install --dangerous testsnap.snap
-#RUN snap connect $SNAP_NAME:kf6-core22 kf6-core22:kf6-core22
-#CMD ["/bin/bash", "-c", "selenium-webdriver-at-spi-run testsuite.py"]
diff --git a/selenium/README.md b/selenium/README.md
index 1c688fa..6aa2284 100644
--- a/selenium/README.md
+++ b/selenium/README.md
@@ -1,5 +1,11 @@
-- put the snap package to test in the current directory, renaming it `testsnap.snap`
-- put the python test script to run with Selenium in the current directory, renaming it `testsuite.py`
-- launch `run.sh`, which will build the docker image and run a container in which the tests can be launched
+# Scripts
-Currently: container fails to start, `systemd` struggles with cgroups.
\ No newline at end of file
+- `./install_selenium.sh` will install all required dependencies and build the KDE Selenium wrapper
+- `./run_selenium.sh <snap file> <app name> <testsuite script>` will install the provided snap file, connect kf6-core22 and run the selenium testsuite on it
+- `./kcalc_24_tests.py` is an example testsuite for kcalc, adapted to work with the 24.02 version (breaking changes since the original 23.08 version)
+
+## Important:
+
+- The run script will not remove the test snap after running the tests. This could probably be changed for the CI.
+- The installation script requires up-to-date packages available through `apt`, it was tested to work on KDE Neon, it failed on various "default" ubuntu versions (22.04, 22.10, 23.04, 24.04) even with the neon repository
+- In case of an error with the symbol `ORG_KDE_KWIN_FAKE_INPUT_DESTROY_SINCE_VERSION` during installation, make sure package `plasma-wayland-protocols` is up to date
\ No newline at end of file
diff --git a/selenium/install_selenium.sh b/selenium/install_selenium.sh
new file mode 100755
index 0000000..c71f911
--- /dev/null
+++ b/selenium/install_selenium.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# This script will install selenium-webdriver-at-spi-run on the system
+# The install will only succeed if the host has access to every package, mainly up-to-date kf6/qt6 packages from https://archive.neon.kde.org.
+# Ubuntu 22.04, 22.10, 23.04 and 24.04 do not have access to every package by default, this was tested on KDE Neon.
+
+sudo apt install -y cmake gcc g++ git ruby dh-python dh-virtualenv debhelper-compat \
+ python3-pip python3-all-dev python3-pyatspi python3-cairo python3-gi-cairo python3-virtualenv \
+ accerciser at-spi2-core gobject-introspection \
+ libgirepository1.0-dev plasma-wayland-protocols libcairo2-dev libpipewire-0.3-dev \
+ libgirepository-1.0-1 libcairo2 \
+ pkg-kde-tools-neon kf6-extra-cmake-modules kf6-kcoreaddons-dev kf6-kwindowsystem-dev kwayland6-dev qt6-base-dev qt6-base-private-dev qt6-wayland-dev libkpipewire-dev \
+ kf6-kcoreaddons kf6-kwindowsystem kwayland6 qt6-base qt6-wayland libqt6core6 libqt6dbus6 libqt6waylandclient6 libkpipewire6
+
+cd /tmp \
+ && git clone https://invent.kde.org/sdk/selenium-webdriver-at-spi.git \
+ && cd selenium-webdriver-at-spi \
+ && virtualenv --system-site-packages venv \ # --system-site-packages required for pyatspi to be detected
+ && source venv/bin/activate \
+ && pip3 install -r requirements.txt \
+ && echo export PATH="~/.local/bin:$PATH" >> ${HOME}/.bashrc \
+ && mkdir build \
+ && cd build \
+ && cmake -DQT_MAJOR_VERSION=6 -DBUILD_WITH_QT6=ON .. \
+ && make \
+ && sudo make install # sudo required to copy the files to /usr
+
diff --git a/selenium/kcalc_24_tests.py b/selenium/kcalc_24_tests.py
new file mode 100755
index 0000000..d0b5c14
--- /dev/null
+++ b/selenium/kcalc_24_tests.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2016 Microsoft Corporation. All rights reserved.
+# SPDX-FileCopyrightText: 2021-2022 Harald Sitter <sitter at kde.org>
+
+# Source: https://invent.kde.org/sdk/selenium-webdriver-at-spi/-/blob/master/examples/calculatortest.py?ref_type=heads
+# Errors:
+# - `org.kde.kcalc.desktop` cannot be started (GIO constructor returns null), `/snap/bin/kcalc` needs to be used instead
+# Changes for kcalc 24.02 (from kcalc 23.08):
+# - The input and result are now split in different fields, which means chaining a result with a followup operation is
+# no longer possible, and the result field requires a '=' press to be updated.
+# => `test_initialize()` needs an extra `=` press to display the result
+# => `test_combination()` is disabled as combination no longer works
+
+import unittest
+from appium import webdriver
+from appium.webdriver.common.appiumby import AppiumBy
+from appium.options.common.base import AppiumOptions
+import selenium.common.exceptions
+from selenium.webdriver.support.ui import WebDriverWait
+
+
+class SimpleCalculatorTests(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(self):
+ options = AppiumOptions()
+ # The app capability may be a command line or a desktop file id.
+ options.set_capability("app", "/snap/bin/kcalc") # 'org.kde.kcalc.desktop' does not work
+ # Boilerplate, always the same
+ self.driver = webdriver.Remote(command_executor='http://127.0.0.1:4723', options=options)
+ # Set a timeout for waiting to find elements. If elements cannot be found
+ # in time we'll get a test failure. This should be somewhat long so as to
+ # not fall over when the system is under load, but also not too long that
+ # the test takes forever.
+ self.driver.implicitly_wait = 10
+
+ @classmethod
+ def tearDownClass(self):
+ # Make sure to terminate the driver again, lest it dangles.
+ self.driver.quit()
+
+ def setUp(self):
+ self.driver.find_element(by=AppiumBy.NAME, value="AC").click()
+ wait = WebDriverWait(self.driver, 20)
+ wait.until(lambda x: self.getresults() == '0')
+
+ def getresults(self):
+ displaytext = self.driver.find_element(by='description', value='Result Display').text
+ return displaytext
+
+ def assertResult(self, actual, expected):
+ wait = WebDriverWait(self.driver, 20)
+ try:
+ wait.until(lambda x: self.getresults() == expected)
+ except selenium.common.exceptions.TimeoutException:
+ pass
+ self.assertEqual(self.getresults(), expected)
+
+ def test_initialize(self):
+ self.driver.find_element(by=AppiumBy.NAME, value="AC").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="7").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="=").click()
+ self.assertResult(self.getresults(), "7")
+
+ def test_addition(self):
+ self.driver.find_element(by=AppiumBy.NAME, value="1").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="+").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="7").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="=").click()
+ self.assertResult(self.getresults(), "8")
+
+ @unittest.skip("Results are not re-usable as inputs in kcalc 24.02.1, so this scenario no longer works")
+ def test_combination(self):
+ self.driver.find_element(by=AppiumBy.NAME, value="7").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="×").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="9").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="+").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="1").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="=").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="÷").click() # in kcalc 24, the input is not `64/8` but `/8`
+ self.driver.find_element(by=AppiumBy.NAME, value="8").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="=").click()
+ self.assertResult(self.getresults(), "8")
+
+ def test_division(self):
+ # Using find element by name twice risks the driver finding the
+ # result display text rather than finding the button. To avoid
+ # that, execute the call once and store that as a local value.
+ button8 = self.driver.find_element(by=AppiumBy.NAME, value="8")
+ button8.click()
+ button8.click()
+ self.driver.find_element(by=AppiumBy.NAME, value="÷").click()
+ button1 = self.driver.find_element(by=AppiumBy.NAME, value="1")
+ button1.click()
+ button1.click()
+ self.driver.find_element(by=AppiumBy.NAME, value="=").click()
+ self.assertResult(self.getresults(), "8")
+
+ def test_multiplication(self):
+ self.driver.find_element(by=AppiumBy.NAME, value="9").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="×").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="9").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="=").click()
+ self.assertResult(self.getresults(), "81")
+
+ def test_subtraction(self):
+ self.driver.find_element(by=AppiumBy.NAME, value="9").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="−").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="1").click()
+ self.driver.find_element(by=AppiumBy.NAME, value="=").click()
+ self.assertResult(self.getresults(), "8")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/selenium/run.sh b/selenium/run.sh
deleted file mode 100755
index cee5b19..0000000
--- a/selenium/run.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-# Adapting https://github.com/ogra1/snapd-docker/blob/master/build.sh
-# Changes:
-# - Kept the clean_up function
-# - Kept the docker build and run commands
-# - Removed the use of arguments
-# - Removed the build-related sections of the script as we have a dedicated custom Dockerfile for that
-# - Replaced snapd healthcheck with container healthcheck
-# - Added use-case-specific commands at the end of the script
-
-clean_up() {
- sleep 1
- docker logs -f snaptest > container_logs.txt || true
- docker rm -f snaptest >/dev/null 2>&1 || true
- docker rmi snaptest >/dev/null 2>&1 || true
- docker rmi $(docker images -f "dangling=true" -q) >/dev/null 2>&1 || true
- exit 1
-}
-trap clean_up 1 2 3 4 9 15
-
-# Build the image
-docker build -t snaptest --force-rm=true . || clean_up
-
-# Start the detached container
-docker run \
- --name=snaptest \
- -ti \
- --tmpfs /run \
- --tmpfs /run/lock \
- --tmpfs /tmp \
- --cap-add SYS_ADMIN \
- --device=/dev/fuse \
- --security-opt apparmor:unconfined \
- --security-opt seccomp:unconfined \
- -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
- -v /lib/modules:/lib/modules:ro \
- -d snaptest || clean_up
-
-# FIXME:
-# systemd 249.11-0ubuntu3.12 running in system mode (+PAM +AUDIT +SELINUX +APPARMOR +IMA +SMACK +SECCOMP +GCRYPT +GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 -IDN +IPTC +KMOD +LIBCRYPTSETUP +LIBFDISK +PCRE2 -PWQUALITY -P11KIT -QRENCODE +BZIP2 +LZ4 +XZ +ZLIB +ZSTD -XKBCOMMON +UTMP +SYSVINIT default-hierarchy=unified)
-# Detected virtualization docker.
-# Detected architecture x86-64.
-#
-# Welcome to Ubuntu 22.04.4 LTS!
-#
-# Failed to create /init.scope control group: Read-only file system
-# Failed to allocate manager object: Read-only file system
-# Failed to allocate manager object.
-# Exiting PID 1...
-
-# Wait for snapd to start
-TIMEOUT=30
-SLEEP=0.1
-echo -n "Waiting up to $(($TIMEOUT/10)) seconds for container startup"
-while [ "$( docker container inspect -f '{{.State.Status}}' snaptest )" != "running" ]; do
- echo -n "."
- sleep $SLEEP || clean_up
- if [ "$TIMEOUT" -le "0" ]; then
- echo " Timed out!"
- clean_up
- fi
- TIMEOUT=$(($TIMEOUT-1))
-done
-echo " done"
-
-# Install test snap and kf6
-docker exec snaptest snap install kf6-core22 || clean_up
-docker exec snaptest snap install --dangerous testsnap.snap || clean_up
-docker exec snaptest snap connect $SNAP_NAME:kf6-core22 kf6-core22:kf6-core22 || clean_up
-
-# Run the tests
-docker exec snaptest selenium-webdriver-at-spi-run testsuite.py > testsuite_output.txt
-clean_up
diff --git a/selenium/run_selenium.sh b/selenium/run_selenium.sh
new file mode 100755
index 0000000..59bb9a0
--- /dev/null
+++ b/selenium/run_selenium.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+if [ "$#" -ne 3 ] || ! [ -f "$1" ] || ! [ -f "$3" ]; then
+ echo "Usage: $0 SNAP_FILE SNAP_NAME TESTSUITE_SCRIPT" >&2
+ echo " e.g: $0 ./kcalc_24.02.1_amd64.snap kcalc ./testsuite.py" >&2
+ echo "Note, the installed snap will not be removed automatically" >&2
+ exit 1
+fi
+
+# Install the snap package to test.
+snap install kf6-core22 || exit 1
+sudo snap install --dangerous "$1" || exit 1
+sudo snap connect "$2":kf6-core22 kf6-core22:kf6-core22 || exit 1
+
+# Run the testsuite
+chmod +x "$3"
+selenium-webdriver-at-spi-run "$3"
\ No newline at end of file
More information about the Neon-commits
mailing list