diff --git a/docs/stress-tests.md b/docs/stress-tests.md new file mode 120000 index 0000000..4c72320 --- /dev/null +++ b/docs/stress-tests.md @@ -0,0 +1 @@ +../stress-tests/README.md \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 0daee51..7b6e00c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -63,6 +63,7 @@ nav: - integrations.ca.md - post-install.ca.md - contributing.ca.md +- stress-tests.md - security.ca.md - project-management.md #- upgrade.md diff --git a/stress-tests/.gitignore b/stress-tests/.gitignore new file mode 100644 index 0000000..c9941eb --- /dev/null +++ b/stress-tests/.gitignore @@ -0,0 +1,10 @@ +# Potentially private files +users.csv +docs.csv +dd-stress-test.users.csv +dd-stress-test.docs.csv +# Transient files +dd-stress-test.jmx +apache-jmeter-*/ +results/ +vm-test.log diff --git a/stress-tests/Pipfile b/stress-tests/Pipfile new file mode 100644 index 0000000..440bd33 --- /dev/null +++ b/stress-tests/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +selenium = ">=4.8" +pyyaml = "*" + +[dev-packages] + +[requires] +python_version = "3" diff --git a/stress-tests/Pipfile.lock b/stress-tests/Pipfile.lock new file mode 100644 index 0000000..4620e92 --- /dev/null +++ b/stress-tests/Pipfile.lock @@ -0,0 +1,189 @@ +{ + "_meta": { + "hash": { + "sha256": "7668d1f96f732a37135c5c4d46a18b784dd1e38661f5e8968bb31dcc0cc6ef18" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "async-generator": { + "hashes": [ + "sha256:01c7bf666359b4967d2cda0000cc2e4af16a0ae098cbffcb8472fb9e8ad6585b", + "sha256:6ebb3d106c12920aaae42ccb6f787ef5eefdcdd166ea3d628fa8476abe712144" + ], + "markers": "python_version >= '3.5'", + "version": "==1.10" + }, + "attrs": { + "hashes": [ + "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836", + "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99" + ], + "markers": "python_version >= '3.6'", + "version": "==22.2.0" + }, + "certifi": { + "hashes": [ + "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", + "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" + ], + "markers": "python_version >= '3.6'", + "version": "==2022.12.7" + }, + "exceptiongroup": { + "hashes": [ + "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e", + "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23" + ], + "markers": "python_version < '3.11'", + "version": "==1.1.0" + }, + "h11": { + "hashes": [ + "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", + "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761" + ], + "markers": "python_version >= '3.7'", + "version": "==0.14.0" + }, + "idna": { + "hashes": [ + "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" + ], + "markers": "python_version >= '3.5'", + "version": "==3.4" + }, + "outcome": { + "hashes": [ + "sha256:6f82bd3de45da303cf1f771ecafa1633750a358436a8bb60e06a1ceb745d2672", + "sha256:c4ab89a56575d6d38a05aa16daeaa333109c1f96167aba8901ab18b6b5e0f7f5" + ], + "markers": "python_version >= '3.7'", + "version": "==1.2.0" + }, + "pysocks": { + "hashes": [ + "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299", + "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", + "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0" + ], + "version": "==1.7.1" + }, + "pyyaml": { + "hashes": [ + "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", + "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", + "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", + "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", + "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", + "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", + "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", + "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", + "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", + "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", + "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", + "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", + "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", + "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", + "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", + "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", + "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", + "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", + "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", + "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", + "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", + "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", + "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", + "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", + "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", + "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", + "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", + "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", + "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", + "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", + "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", + "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", + "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", + "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + ], + "index": "pypi", + "version": "==6.0" + }, + "selenium": { + "hashes": [ + "sha256:bd04eb41395605d9b2b65fe587f3fed21431da75512985c52772529e5e210c60", + "sha256:c48372905bffcc3b24bd55ab4683a07ee5e1f30fe918c59558ea5ee44cedf6c3" + ], + "index": "pypi", + "version": "==4.8.2" + }, + "sniffio": { + "hashes": [ + "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101", + "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384" + ], + "markers": "python_version >= '3.7'", + "version": "==1.3.0" + }, + "sortedcontainers": { + "hashes": [ + "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", + "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0" + ], + "version": "==2.4.0" + }, + "trio": { + "hashes": [ + "sha256:ce68f1c5400a47b137c5a4de72c7c901bd4e7a24fbdebfe9b41de8c6c04eaacf", + "sha256:f1dd0780a89bfc880c7c7994519cb53f62aacb2c25ff487001c0052bd721cdf0" + ], + "markers": "python_version >= '3.7'", + "version": "==0.22.0" + }, + "trio-websocket": { + "hashes": [ + "sha256:5b558f6e83cc20a37c3b61202476c5295d1addf57bd65543364e0337e37ed2bc", + "sha256:a3d34de8fac26023eee701ed1e7bf4da9a8326b61a62934ec9e53b64970fd8fe" + ], + "markers": "python_version >= '3.5'", + "version": "==0.9.2" + }, + "urllib3": { + "extras": [ + "socks" + ], + "hashes": [ + "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72", + "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.26.14" + }, + "wsproto": { + "hashes": [ + "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", + "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736" + ], + "markers": "python_version >= '3.7'", + "version": "==1.2.0" + } + }, + "develop": {} +} diff --git a/stress-tests/README.md b/stress-tests/README.md new file mode 100644 index 0000000..6417700 --- /dev/null +++ b/stress-tests/README.md @@ -0,0 +1,54 @@ +# Stress tests + +By executing tests in this normalised fashion it is easier to compare results +between different instances or different patch levels. + +This documents normalised stress-testing and references files under +[`stress-tests`][st]. + +[st]: https://gitlab.com/DD-workspace/DD/-/tree/main/stress-tests + + +## VM tests + +- `vm-test.sh`: generate a text file to compare CPU and other factors across + VM types, providers or instances which may affect DD performance. + +We can compare the resulting lgos just with, e.g. `vim -d`. + + +## DD tests + +Currently these tests perform logins and interact with Nextcloud, but it would +be interesting to expand them to interact with other services. + +### Directory contents + +This directory contains following files: + +- `dd-stress-test.tpl.jmx`: template to generate [JMeter][jm] tests to execute +- `dd-tests.sh`: helper script that generates the actual test plan files and + executes them. See `./dd-tests.sh --help` +- `dd-test-selenium.sh`: this gives us an idea of how a user would perceive + DD to be behaving while under load. Called by `./dd-tests.sh` by default. + + +### Results + +Results will be saved in a `results` directory, where each subdirectory +corresponds to a stress test executed with `dd-test.sh`. + +The naming scheme for those subdirectories is: `DOMAIN_THREADCOUNT_DURATION` +Where `THREADCOUNT` and `DURATION` are the corresponding [JMeter][jm] +parameters. + +Each results directory contains: + +- `log`: the [JMeter][jm] log +- `results`: the [JMeter][jm] results file +- `html/index.html`: the interactive graphs for the data as produced by JMeter +- `selenium/session.html`: the session report as would be perceived by a user. + Note this requires Python3 and selenium and can be disabled by setting the + environment variable: `USE_SELENIUM=NO`. + +[jm]: https://jmeter.apache.org/ diff --git a/stress-tests/dd-stress-test.tpl.jmx b/stress-tests/dd-stress-test.tpl.jmx new file mode 100644 index 0000000..060d0d8 --- /dev/null +++ b/stress-tests/dd-stress-test.tpl.jmx @@ -0,0 +1,822 @@ + + + + + + + false + false + + + + + + + + + + ThreadCount + 10 + = + Number of Threads + + + Duration + 60 + = + in seconds + + + RampUpPeriod + 10 + = + in seconds + + + IdPHost + https://sso.DD_DOMAIN + = + The main URL of your SSO Instance + + + IdPPort + 80 + = + + + IdPContext + idp + = + + + startupDelay + 1 + = + in seconds + + + LoginSP + https://nextcloud.DD_DOMAIN/apps/user_saml/saml/login + = + Domain of registered NC using SAML + + + ProviderId + https://nextcloud.DD_DOMAIN/apps/user_saml/saml/metadata + = + SAML EntityId for NC + + + ACSUrl + https://nextcloud.DD_DOMAIN/apps/user_saml/saml/acs + SP ACS Url to post the final response/assertion back + = + + + NextCloud + nextcloud.DD_DOMAIN + = + + + ClientId + https://nextcloud.DD_DOMAIN/apps/user_saml/saml/metadata + = + + + Environment Variables that will need to be updated + + + + + + + + ./dd-stress-test.users.csv + + User,Password + , + false + true + false + shareMode.all + No Spaces between User, comma, and Password fields!! + false + + + + ./dd-stress-test.docs.csv + + folder,document + , + false + true + false + shareMode.all + No Spaces between Folder, comma, and DocumentID fields!! + false + + + + + true + false + + + + + + + + + https + + + 4 + HttpClient4 + 120000 + 120000 + + + + stopthread + + false + -1 + + ${ThreadCount} + ${RampUpPeriod} + 1500501650000 + 1500501650000 + true + ${Duration} + ${StartupDelay} + SAML Support Login Process + true + true + + + + + Test Plan + NextCloud Tests + NEXTCLOUD Login + + SAML2 SP calls the login page for CAS IdP + + + + + Test Plan + NextCloud Tests + POST - Login User + + Logging into CAS Idp + + + + + Test Plan + NextCloud Tests + POST - Authorization to NEXTCLOUD + + Send response from login to SP for processing + + + + + Test Plan + NextCloud Tests + GET - Document + + Send response from login to SP for processing + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + + + + + + + + + + + ${LoginSP}/login + GET + true + false + true + true + + + + Calling secured SP page, that should then redirect to CAS login page + + + + + + Sec-Fetch-Mode + navigate + + + Sec-Fetch-Site + none + + + Accept-Language + en-US,en;q=0.5 + + + Sec-Fetch-User + ?1 + + + Pragma + no-cache + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Cache-Control + no-cache + + + Accept-Encoding + gzip, deflate, br + + + User-Agent + Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0 + + + Sec-Fetch-Dest + document + + + + + + false + RelayState + <input type="hidden" name="RelayState" value="(.+?)" + $1$ + + 1 + all + + + + false + SAMLResponse + <input type="hidden" name="SAMLResponse" value="(.+?)" + $1$ + + 1 + all + + + + false + Execution + \;execution=(.+?)&amp; + $1$ + + + all + + + + false + SessionCode + session_code=(.*)&amp;ex + $1$ + + + all + + + + false + tab_id + \;tab_id=(.*)\" me + $1$ + + + all + + + + URL + SAMLRequest + SAMLRequest=(.+) + $1$ + + 1 + all + + + + + + + + + + + true + ${User} + = + true + username + false + + + true + ${Password} + = + true + password + false + + + + + + + + ${IdPHost}/auth/realms/master/login-actions/authenticate?session_code=${SessionCode}&execution=${Execution}&client_id=${__urlencode(${ClientId})}&tab_id=${tab_id} + POST + true + false + true + false + + + + POST Login Credentials for SAMLResponse + + + + unescaped + RelayState + <input type="hidden" name="RelayState" value="(.+?)" + $1$ + + 1 + all + + + + unescaped + SAMLResponse + <input type="hidden" name="SAMLResponse" value="(.+?)" + $1$ + + 1 + all + + + + + + + + + + + true + ${RelayState} + = + true + RelayState + + + true + ${SAMLResponse} + = + true + SAMLResponse + + + + + + + + ${ACSUrl} + POST + true + false + true + true + + + + + + + + + + + + + true + OCS-APIREQUEST + true + = + true + + + + ${NextCloud} + + https + + + GET + true + false + true + false + + + + + + + false + requestToken + data-requesttoken="(.+)\" + $1$ + + + all + + + + + + + + false + dir + Documents + = + true + + + + ${NextCloud} + + https + + /apps/files + GET + true + false + true + true + + + + + + + + + Accept + application/json, text/plain, */* + + + requesttoken + ${requestToken} + + + + + + + true + + + + false + <?xml version="1.0"?> +<d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns" xmlns:ocs="http://open-collaboration-services.org/ns"> + <d:prop> + <d:getlastmodified /> + <d:getetag /> + <d:getcontenttype /> + <d:resourcetype /> + <oc:fileid /> + <oc:permissions /> + <oc:size /> + <d:getcontentlength /> + <d:quota-available-bytes /> + <nc:has-preview /> + <nc:mount-type /> + <nc:is-encrypted /> + <ocs:share-permissions /> + <nc:share-attributes /> + <oc:tags /> + <oc:favorite /> + <oc:owner-id /> + <oc:owner-display-name /> + <oc:share-types /> + <oc:comments-unread /> + </d:prop> +</d:propfind> + = + + + + ${NextCloud} + 443 + https + UTF-8 + /remote.php/dav/files/${User} + PROPFIND + true + false + true + false + + + + + + + + + Sec-Fetch-Mode + cors + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.5 + + + Origin + https://${NextCloud} + + + Accept + text/plain + + + Depth + 0 + + + X-Requested-With + XMLHttpRequest + + + Content-Type + text/plain;charset=UTF-8 + + + Accept-Encoding + gzip, deflate, br + + + User-Agent + Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0 + + + Sec-Fetch-Dest + empty + + + requesttoken + ${requestToken} + + + + + + + + + + false + file + https://${NextCloud}/remote.php/dav/files/${User}/${document} + = + true + + + false + canDownload + 1 + = + true + + + + ${NextCloud} + 443 + https + + /apps/files_pdfviewer/ + GET + true + false + true + false + + + + + + + + + Sec-Fetch-Mode + navigate + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.5 + + + Upgrade-Insecure-Requests + 1 + + + Accept-Encoding + gzip, deflate, br + + + User-Agent + Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0 + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8 + + + Sec-Fetch-Dest + iframe + + + requesttoken + ${requestToken} + + + + + + + + + + ${NextCloud} + 443 + https + utf-8 + /remote.php/dav/files/${User}/${document} + GET + true + false + true + false + + + + + + + + + Sec-Fetch-Mode + cors + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.5 + + + Accept-Encoding + gzip, deflate, br + + + User-Agent + Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0 + + + Accept + */* + + + Sec-Fetch-Dest + empty + + + requesttoken + ${requestToken} + + + + + + + + + diff --git a/stress-tests/dd-test-selenium.py b/stress-tests/dd-test-selenium.py new file mode 100644 index 0000000..2dc94af --- /dev/null +++ b/stress-tests/dd-test-selenium.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 + +import argparse +import io +import pathlib +import random +import traceback +from datetime import datetime + +import yaml + +from selenium import webdriver + +# We need selenium 4 +from selenium.webdriver.common.by import By +from selenium.webdriver.support.wait import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.chrome.options import Options as ChromeOptions + +from typing import Any, Dict, Iterable, List + + +class DDSession: + driver: webdriver.Chrome + + def __init__( + self, + instance: str, + users_file: str, + out_dir: pathlib.Path, + stepTimeoutSeconds: int = 20, + ): + self.instance = instance + self.users_file = [ + [i.strip() for i in l.split(sep=",", maxsplit=1)] + for l in users_file.split("\n") + if l + ] + self.out_dir = out_dir + self.stepTimeoutSeconds = stepTimeoutSeconds + + if not self.out_dir.is_dir(): + self.out_dir.mkdir() + + self.start_time = datetime.utcnow() + self.last_time = self.start_time + self.screenshot_counter = 0 + self.data: Dict[str, Any] = dict( + instance=self.instance, start_time=self.start_time, steps=[] + ) + self.reset() + + def persist(self) -> None: + yaml.dump(self.data, (self.out_dir / "session.yml").open("w")) + (self.out_dir / "session.html").open("w").write(self.summarise()["html"]) + + def summarise(self) -> Dict[str, Any]: + return DDSession.summarise_data(self.data) + + @staticmethod + def summarise_data(data) -> Dict[str, Any]: + import copy + from operator import getitem + from itertools import groupby + + def summarise_item(g: List[Dict[str, Any]]) -> Dict[str, Any]: + if "substeps" in g[0]: + items = [sum((st["step_t"] for st in s["substeps"])) for s in g] + else: + items = [st["step_t"] for st in g] + return { + "max": max(items), + "count": len(items), + "average": sum(items) / len(items), + } + + def summarise(d: Iterable[Dict[str, Any]]) -> Dict[str, Any]: + dict_data = dict() + for k, g in groupby( + sorted(d, key=lambda x: getitem(x, "step")), + key=lambda x: getitem(x, "step"), + ): + dict_data[k] = summarise_item(list(g)) + return dict_data + + summary: Dict[str, Any] = dict() + summary["overview"] = summarise(data["steps"]) + + d = copy.deepcopy(data["steps"]) + substeps = [ + dict(st, step=f"{step['step']}: {st['substep']}") + for step in data["steps"] + for st in step["substeps"] + ] + summary["stepbystep"] = summarise(substeps) + + html = """ + + +""" + html += f""" +

{data['instance']}

+

{data['start_time']}

+
Summary +
{yaml.dump(summary)}
+
+
+""" + for s in data["steps"]: + html += f"""

{s['step']}: {s['t']} s (+ {sum((st['step_t'] for st in s['substeps']))} s)

""" + for st in s["substeps"]: + html += f"""

{s['step']} - {st['substep']}: {st['t']} s (+ {st['step_t']} s)

""" + html += f"""""" + html += "" + summary["html"] = html + return summary + + @property + def executed_seconds(self) -> float: + return (datetime.utcnow() - self.start_time).total_seconds() + + @property + def nextcloud_url(self) -> str: + return f"https://nextcloud.{self.instance}" + + def reset(self, new_driver: bool = False) -> None: + # If needed: + # https://stackoverflow.com/a/72922584 + if getattr(self, "driver", None) is None or new_driver: + if getattr(self, "driver", None) is not None: + self.driver.quit() + options = ChromeOptions() + for arg in [ + "--headless", + "--no-sandbox", + "--disable-infobars", + "--disable-extensions", + "--disable-dev-shm-usage", + "window-size=1400,600", + ]: + options.add_argument(arg) + self.driver = webdriver.Chrome(options=options) + msg = "Restart" + else: + self.driver.delete_all_cookies() + msg = "Cleaned cache" + self.screenshot("Browser", msg) + + def perform_login(self) -> None: + # Wait until Keycloak is shown + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: d.find_elements(By.ID, "kc-form-login") + ) + + self.screenshot("Login", "Request") + + # Fill in form data for login + (username, password) = random.choice(self.users_file) + usernameField = self.driver.find_element(By.ID, "username") + usernameField.send_keys(username) + passwordField = self.driver.find_element(By.ID, "password") + passwordField.send_keys(password) + loginButton = self.driver.find_element(By.ID, "kc-login") + self.screenshot("Login", "Form filled") + + loginButton.click() + + self.screenshot("Login", "Form sent") + # TODO: wait for redirection middle step and session.screenshot it + + def load_nextcloud_files(self) -> None: + # Start loading + self.driver.get(self.nextcloud_url) + + # Wait until Nextcloud with Megamenu is shown or a login is requested + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: ( + d.find_elements(By.ID, "kc-form-login") + + d.find_elements(By.ID, "menu-apps-icon") + ) + ) + + # Detect whether or not a login is needed + login_required = self.driver.find_elements(By.ID, "kc-form-login") + if login_required: + # Perform login + self.perform_login() + + self.screenshot("Nextcloud", "Loading") + + # Wait until Nextcloud files app has loaded + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: d.find_elements( + By.CSS_SELECTOR, "#app-content-files td.selection" + ) + ) + self.screenshot("Nextcloud", "Loaded") + + def open_file_in_onlyoffice(self, file_base_name: str) -> None: + # Get and click docx file + # This assumes we are on the nextcloud files app with a FILE_BASE file + window_count = len(self.driver.window_handles) + docxFileLabel = self.driver.find_element(By.PARTIAL_LINK_TEXT, file_base_name) + docxFileLabel.click() + + # Switch to OnlyOffice window + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + EC.number_of_windows_to_be(window_count + 1) + ) + child = self.driver.window_handles[window_count] + self.driver.switch_to.window(child) + + self.screenshot("OnlyOffice", "Opening") + + # Wait for OnlyOffice to start loading + oofCSS = "div.app-onlyoffice #app iframe" + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: d.find_elements(By.CSS_SELECTOR, oofCSS) + ) + self.screenshot("OnlyOffice", "Loading 1") + + # Switch to its iframe + oof = WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: EC.frame_to_be_available_and_switch_to_it( + d.find_element(By.CSS_SELECTOR, oofCSS) + )(d) + ) + self.screenshot("OnlyOffice", "Loading 2") + + oofLoaded = lambda d: EC.element_to_be_clickable( + d.find_element(By.ID, "id-toolbar-btn-save") + )(d) + + # Get the first loading screen + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: oofLoaded(d) + or d.find_elements(By.CSS_SELECTOR, "#loading-mask div.loader-page") + ) + self.screenshot("OnlyOffice", "Loading 3") + + # Wait for OnlyOffice second loading phase + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: oofLoaded(d) + or d.find_elements(By.CSS_SELECTOR, "div.asc-loadmask-title") + ) + self.screenshot("OnlyOffice", "Loading 4") + + # Wait a for final loading (save button clickable) + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds * 3).until(oofLoaded) + self.screenshot("OnlyOffice", "Loaded") + + # Close OnlyOffice window + self.driver.close() + # And change back to the main window + self.driver.switch_to.window(self.driver.window_handles[window_count - 1]) + self.screenshot("OnlyOffice", "Closed") + + def screenshot(self, step: str, substep_txt: str) -> None: + # Get data + scr_now = datetime.utcnow() + scr_s = (scr_now - self.start_time).total_seconds() + step_t = (scr_now - self.last_time).total_seconds() + scr_id = self.screenshot_counter + + # Upgrade values + self.last_time = scr_now + self.screenshot_counter += 1 + + # Constants + scr_fn = f"{scr_id:04d}.png" + scr_path = self.out_dir / scr_fn + # Get screenie + scr_img = self.driver.get_screenshot_as_png() + # Write it + open(scr_path, "wb").write(scr_img) + # Add to report + substep: Dict[str, Any] = dict( + id=scr_id, substep=substep_txt, step_t=step_t, t=scr_s, png=scr_fn + ) + if self.data["steps"] and self.data["steps"][-1]["step"] == step: + self.data["steps"][-1]["substeps"].append(substep) + else: + self.data["steps"].append(dict(step=step, t=scr_s, substeps=[substep])) + + +def randomBool() -> bool: + return random.choice([True, False]) + + +def main_test( + instance: str, + users_file: io.TextIOWrapper, + out_dir: pathlib.Path, + duration: int, + stepTimeoutSeconds: int = 20, +) -> DDSession: + session = DDSession( + instance=instance, + users_file=users_file.read(), + out_dir=out_dir, + stepTimeoutSeconds=20, + ) + + first_run = True + while session.executed_seconds < duration: + # 50% chance of requiring login reseting cookies + # First run always requires login, skip reset + if not first_run and randomBool(): + # Possibly driver too + # 25% chance of cleaning all cache and reopening the browser + session.reset(new_driver=randomBool()) + first_run = False + + try: + # Load nextcloud files + session.load_nextcloud_files() + # Open file in OnlyOffice + session.open_file_in_onlyoffice("template_1") + except Exception as ex: + session.screenshot("Exception", str(ex)) + print(traceback.format_exc()) + session.persist() + return session + + +def main_summary(filename: io.TextIOWrapper) -> None: + data = yaml.safe_load(filename) + summary = DDSession.summarise_data(data) + html_fn = pathlib.Path(filename.name).with_suffix(".html") + open(html_fn, "w").write(summary["html"]) + + +if __name__ == "__main__": + import pathlib + + parser = argparse.ArgumentParser( + prog="DD Selenium tester", description="Run basic UI tests on DD" + ) + subparsers = parser.add_subparsers(required=True) + test_parser = subparsers.add_parser("test") + + test_parser.add_argument("instance") + test_parser.add_argument( + "-u", + "--users-file", + default="dd-stress-test.users.csv", + type=argparse.FileType("r"), + ) + test_parser.add_argument("-d", "--duration", default=300, type=int) + test_parser.add_argument("-o", "--out-dir", default="results", type=pathlib.Path) + + summary_parser = subparsers.add_parser("summary") + summary_parser.add_argument( + "filename", default="session.yml", type=argparse.FileType("r") + ) + + ns = parser.parse_args() + if "instance" in ns: + main_test(**vars(ns)) + else: + main_summary(**vars(ns)) diff --git a/stress-tests/dd-test.sh b/stress-tests/dd-test.sh new file mode 100755 index 0000000..e5d1985 --- /dev/null +++ b/stress-tests/dd-test.sh @@ -0,0 +1,180 @@ +#!/bin/sh -eu + +# Process inputs +DD_DOMAIN="${1:-}" +tc="${2:-}" +duration="${3:-60}" +USE_SELENIUM="${USE_SELENIUM:-YES}" +SCRIPT_PATH="$(realpath "${0}")" +SCRIPT_NAME="$(basename "${0}")" + +JMETER_DEFAULT="./apache-jmeter-5.5/bin/jmeter" + +full_tests() { + # Runtime: 7 tests * 5 mins / test = 35 mins + # Cool-off periods: 30s * 6 = 3 mins + # Total: 38 mins + cooloff="30" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 10 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 20 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 30 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 60 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 100 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 300 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 600 300 +} + +help_users_file() { + cat <<-EOF + The format of the users.csv file must be: + USERNAME1,PASSWORD1 + USERNAME2,PASSWORD2 + ... + + Take care not to have any spaces between fields. + EOF +} + +help_jmeter() { + cat <<-EOF + Note this scripts depends on JMeter with some plugins enabled. + You can set the JMETER environment variable to its binary path. + If this variable is unset, ${JMETER_DEFAULT} + will be used, from this script's location. + See: + https://jmeter.apache.org/download_jmeter.cgi + https://jmeter-plugins.org/install/Install/ + EOF +} + +help() { + cat <<-EOF + Examples: + ./${SCRIPT_NAME} DD_DOMAIN THREAD_COUNT [DURATION] + or: + ./${SCRIPT_NAME} --full-tests DD_DOMAIN + + EOF + + help_jmeter + + cat <<-EOF + + When using --full-tests, a pre-selected combination of + THREAD_COUNT and DURATION will be used against DD_DOMAIN. + + Where DD_DOMAIN is the base domain, e.g. if your DD instance's + Nextcloud can be accessed at nextcloud.example.org, the parameter + should be "example.org". + + THREAD_COUNT refers to the amount of users that will be simulated. + + DURATION is the total test time time in seconds. Defaults to 60. + + Note that you MUST have a users.csv file in the current directory. + + By default this script runs tests with selenium and documents the + session as would be perceived by a user. + You can disable this behaviour by setting the environment variable + USE_SELENIUM=NO. + EOF + help_users_file +} + +if [ "${DD_DOMAIN:-}" = "--full-tests" ]; then + shift # Consume operation argument + # Re-set global variable + DD_DOMAIN="${1:-}" + # Execute full suite + full_tests + # And exit + exit 0 +elif [ "${1:-}" = "--help" ]; then + help + exit 0 +elif [ -z "${DD_DOMAIN:-}" ] || [ -z "${tc:-}" ]; then + help >> /dev/stderr + exit 1 +fi + +USERS_FILE="$(pwd)/users.csv" +DOCS_FILE="$(pwd)/docs.csv" + +# Change current path +cd "$(dirname "${SCRIPT_PATH}")" +out_dir="$(pwd)/results/${DD_DOMAIN}_${tc}_${duration}" + +if [ -f "${USERS_FILE}" ]; then + cat "${USERS_FILE}" > dd-stress-test.users.csv +else + printf "ERROR: missing file\t%s\n\n" "${USERS_FILE}" >> /dev/stderr + help_users_file >> /dev/stderr + exit 2 +fi + +if [ -f "${DOCS_FILE}" ]; then + cat "${DOCS_FILE}" > dd-stress-test.docs.csv +else + cat > dd-stress-test.docs.csv <<-EOF + /,Readme.md + /,template.docx + /,template_1.docx + EOF +fi + + +JMETER="${JMETER:-${JMETER_DEFAULT}}" +# Ensure JMeter is available / bootstrap it +if [ ! -f "${JMETER}" ]; then + echo "INFO: Could not find JMeter, attempting to download it" >> /dev/stderr + curl -L 'https://dlcdn.apache.org//jmeter/binaries/apache-jmeter-5.5.tgz' | tar -xz + JMETER="${JMETER_DEFAULT}" +fi +if [ ! -f "${JMETER}" ]; then + printf "ERROR: missing JMeter\t%s\n\n" "${JMETER}" >> /dev/stderr +fi +# Ensure JMeter plugins are available / bootstrap them +JMETER_PLUGINS="$(dirname "${JMETER}")/../lib/ext/jmeter-plugins-manager-1.8.jar" +if [ ! -f "${JMETER_PLUGINS}" ]; then + echo "INFO: Could not find JMeter plugins, attempting to download them" >> /dev/stderr + curl -L 'https://jmeter-plugins.org/get/' > "${JMETER_PLUGINS}" +fi +if [ ! -f "${JMETER_PLUGINS}" ]; then + printf "ERROR: missing JMeter plugins\t%s\n\n" "${JMETER_PLUGINS}" >> /dev/stderr +fi +if [ ! -f "${JMETER}" ] || [ ! -f "${JMETER_PLUGINS}" ]; then + help_jmeter >> /dev/stderr + exit 3 +fi + +# Clean up out dir +rm -rf "${out_dir}" +mkdir -p "${out_dir}" + +# Adapt template +sed -E \ + -e "s%([^>]*)>(.*)<\!-- TC.*$%\\1>${tc} <\!-- TC -->%" \ + -e "s%([^>]*)>(.*)<\!-- DURATION.*$%\\1>${duration} <\!-- DURATION -->%" \ + -e "s/DD_DOMAIN/${DD_DOMAIN}/g" \ + dd-stress-test.tpl.jmx > dd-stress-test.jmx + +# Call Selenium test process in parallel +if [ "${USE_SELENIUM}" = "YES" ]; then + printf "\n\nRunning parallel Selenium-based tests:\t%s\tover %s seconds\n\n" "${DD_DOMAIN}" "${duration}" + python3 dd-test-selenium.py test --duration "${duration}" --out-dir "${out_dir}/selenium" "${DD_DOMAIN}" 2>&1 > "${out_dir}/selenium.log" & +fi + +# Execute test +printf "\n\nAbout to test:\t%s\twith %s 'users' over %s seconds\n\n" \ + "${DD_DOMAIN}" "${tc}" "${duration}" +env HEAP="-Xms2g -Xmx2g -XX:MaxMetaspaceSize=2g" "${JMETER}" -n -t dd-stress-test.jmx -l "${out_dir}/results" -e -o "${out_dir}/html" +mv jmeter.log "${out_dir}/log" + +# Notify results +printf "\n\nYou can find the results at:\t%s\n\n" "${out_dir}" diff --git a/stress-tests/requirements.txt b/stress-tests/requirements.txt new file mode 100644 index 0000000..28cd789 --- /dev/null +++ b/stress-tests/requirements.txt @@ -0,0 +1,20 @@ +async-generator==1.10 +attrs==22.2.0 +certifi==2022.12.7 +exceptiongroup==1.1.0 +h11==0.14.0 +idna==3.4 +outcome==1.2.0 +pip==22.3.1 +PySocks==1.7.1 +PyYAML==6.0 +selenium==4.8.2 +setuptools==67.0.0 +sniffio==1.3.0 +sortedcontainers==2.4.0 +sqlite3==0.0.0 +trio==0.22.0 +trio-websocket==0.9.2 +urllib3==1.26.14 +wheel==0.38.4 +wsproto==1.2.0 diff --git a/stress-tests/vm-test.sh b/stress-tests/vm-test.sh new file mode 100644 index 0000000..6d093ab --- /dev/null +++ b/stress-tests/vm-test.sh @@ -0,0 +1,26 @@ +#!/bin/sh -eu + +LOG_FILE="${LOG_FILE:-vm-test.log}" + +SYSBENCH="$(command -v sysbench2)" + +# Save stderr as well +exec 2>&1 + +echo "$(date)" | tee "${LOG_FILE}" + +if [ -z "${SYSBENCH}" ]; then + echo "Skipping: sysbench tests (try: apt install sysbench)" | \ + tee -a "${LOG_FILE}" +else + printf "\n\nfileio\n\n" | tee -a "${LOG_FILE}" + "${SYSBENCH}" fileio prepare --file-test-mode=rndrw --threads=4 --time=60 | tee -a "${LOG_FILE}" + "${SYSBENCH}" fileio run --file-test-mode=rndrw --threads=4 --time=60 | tee -a "${LOG_FILE}" + printf "\n\ncpu\n\n" | tee -a "${LOG_FILE}" + "${SYSBENCH}" cpu run --threads=4 --time=60 | tee -a "${LOG_FILE}" + printf "\n\nmemory\n\n" | tee -a "${LOG_FILE}" + "${SYSBENCH}" memory run --threads=4 --time=60 | tee -a "${LOG_FILE}" +fi +# Perform basic OpenSSL tests too +printf "\n\nCPU OpenSSL\n\n" | tee -a "${LOG_FILE}" +openssl speed --seconds 60 sha256 | tee -a "${LOG_FILE}"