From 567bfd770d8c09f03db8416f1e8e6c6fa8590f42 Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 27 Feb 2023 19:38:41 +0100 Subject: [PATCH] [stress-tests] Documentation + necessary files to perform testing By executing tests in this normalised fashion it is easier to compare results between different instances or different patch levels. Roughly speaking there are two kinds of tests: - VM tests: which summarise general performance of the instance, without taking DD into account - DD tests: which simulates many logins and interactions with DD, while recording the session as a user would pereceive it from a browser By using these we should be able to consistently compare and improve performance. The original dd-stress-test.tpl.jmx tests file was prepared by Teradisk with hardcoded instance, threadcount and duration values. Testing should now be performed with `vm-test.sh` and `dd-test.sh` respectively, and the template file should stay generic. --- docs/stress-tests.md | 1 + mkdocs.yml | 1 + stress-tests/.gitignore | 10 + stress-tests/Pipfile | 13 + stress-tests/Pipfile.lock | 189 +++++++ stress-tests/README.md | 54 ++ stress-tests/dd-stress-test.tpl.jmx | 822 ++++++++++++++++++++++++++++ stress-tests/dd-test-selenium.py | 361 ++++++++++++ stress-tests/dd-test.sh | 180 ++++++ stress-tests/requirements.txt | 20 + stress-tests/vm-test.sh | 26 + 11 files changed, 1677 insertions(+) create mode 120000 docs/stress-tests.md create mode 100644 stress-tests/.gitignore create mode 100644 stress-tests/Pipfile create mode 100644 stress-tests/Pipfile.lock create mode 100644 stress-tests/README.md create mode 100644 stress-tests/dd-stress-test.tpl.jmx create mode 100644 stress-tests/dd-test-selenium.py create mode 100755 stress-tests/dd-test.sh create mode 100644 stress-tests/requirements.txt create mode 100644 stress-tests/vm-test.sh diff --git a/docs/stress-tests.md b/docs/stress-tests.md new file mode 120000 index 0000000..4c72320 --- /dev/null +++ b/docs/stress-tests.md @@ -0,0 +1 @@ +../stress-tests/README.md \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 0daee51..7b6e00c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -63,6 +63,7 @@ nav: - integrations.ca.md - post-install.ca.md - contributing.ca.md +- stress-tests.md - security.ca.md - project-management.md #- upgrade.md diff --git a/stress-tests/.gitignore b/stress-tests/.gitignore new file mode 100644 index 0000000..c9941eb --- /dev/null +++ b/stress-tests/.gitignore @@ -0,0 +1,10 @@ +# Potentially private files +users.csv +docs.csv +dd-stress-test.users.csv +dd-stress-test.docs.csv +# Transient files +dd-stress-test.jmx +apache-jmeter-*/ +results/ +vm-test.log diff --git a/stress-tests/Pipfile b/stress-tests/Pipfile new file mode 100644 index 0000000..440bd33 --- /dev/null +++ b/stress-tests/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +selenium = ">=4.8" +pyyaml = "*" + +[dev-packages] + +[requires] +python_version = "3" diff --git a/stress-tests/Pipfile.lock b/stress-tests/Pipfile.lock new file mode 100644 index 0000000..4620e92 --- /dev/null +++ b/stress-tests/Pipfile.lock @@ -0,0 +1,189 @@ +{ + "_meta": { + "hash": { + "sha256": "7668d1f96f732a37135c5c4d46a18b784dd1e38661f5e8968bb31dcc0cc6ef18" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "async-generator": { + "hashes": [ + "sha256:01c7bf666359b4967d2cda0000cc2e4af16a0ae098cbffcb8472fb9e8ad6585b", + "sha256:6ebb3d106c12920aaae42ccb6f787ef5eefdcdd166ea3d628fa8476abe712144" + ], + "markers": "python_version >= '3.5'", + "version": "==1.10" + }, + "attrs": { + "hashes": [ + "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836", + "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99" + ], + "markers": "python_version >= '3.6'", + "version": "==22.2.0" + }, + "certifi": { + "hashes": [ + "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", + "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" + ], + "markers": "python_version >= '3.6'", + "version": "==2022.12.7" + }, + "exceptiongroup": { + "hashes": [ + "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e", + "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23" + ], + "markers": "python_version < '3.11'", + "version": "==1.1.0" + }, + "h11": { + "hashes": [ + "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", + "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761" + ], + "markers": "python_version >= '3.7'", + "version": "==0.14.0" + }, + "idna": { + "hashes": [ + "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" + ], + "markers": "python_version >= '3.5'", + "version": "==3.4" + }, + "outcome": { + "hashes": [ + "sha256:6f82bd3de45da303cf1f771ecafa1633750a358436a8bb60e06a1ceb745d2672", + "sha256:c4ab89a56575d6d38a05aa16daeaa333109c1f96167aba8901ab18b6b5e0f7f5" + ], + "markers": "python_version >= '3.7'", + "version": "==1.2.0" + }, + "pysocks": { + "hashes": [ + "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299", + "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", + "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0" + ], + "version": "==1.7.1" + }, + "pyyaml": { + "hashes": [ + "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", + "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", + "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", + "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", + "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", + "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", + "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", + "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", + "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", + "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", + "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", + "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", + "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", + "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", + "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", + "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", + "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", + "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", + "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", + "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", + "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", + "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", + "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", + "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", + "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", + "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", + "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", + "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", + "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", + "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", + "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", + "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", + "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", + "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + ], + "index": "pypi", + "version": "==6.0" + }, + "selenium": { + "hashes": [ + "sha256:bd04eb41395605d9b2b65fe587f3fed21431da75512985c52772529e5e210c60", + "sha256:c48372905bffcc3b24bd55ab4683a07ee5e1f30fe918c59558ea5ee44cedf6c3" + ], + "index": "pypi", + "version": "==4.8.2" + }, + "sniffio": { + "hashes": [ + "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101", + "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384" + ], + "markers": "python_version >= '3.7'", + "version": "==1.3.0" + }, + "sortedcontainers": { + "hashes": [ + "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", + "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0" + ], + "version": "==2.4.0" + }, + "trio": { + "hashes": [ + "sha256:ce68f1c5400a47b137c5a4de72c7c901bd4e7a24fbdebfe9b41de8c6c04eaacf", + "sha256:f1dd0780a89bfc880c7c7994519cb53f62aacb2c25ff487001c0052bd721cdf0" + ], + "markers": "python_version >= '3.7'", + "version": "==0.22.0" + }, + "trio-websocket": { + "hashes": [ + "sha256:5b558f6e83cc20a37c3b61202476c5295d1addf57bd65543364e0337e37ed2bc", + "sha256:a3d34de8fac26023eee701ed1e7bf4da9a8326b61a62934ec9e53b64970fd8fe" + ], + "markers": "python_version >= '3.5'", + "version": "==0.9.2" + }, + "urllib3": { + "extras": [ + "socks" + ], + "hashes": [ + "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72", + "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.26.14" + }, + "wsproto": { + "hashes": [ + "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", + "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736" + ], + "markers": "python_version >= '3.7'", + "version": "==1.2.0" + } + }, + "develop": {} +} diff --git a/stress-tests/README.md b/stress-tests/README.md new file mode 100644 index 0000000..6417700 --- /dev/null +++ b/stress-tests/README.md @@ -0,0 +1,54 @@ +# Stress tests + +By executing tests in this normalised fashion it is easier to compare results +between different instances or different patch levels. + +This documents normalised stress-testing and references files under +[`stress-tests`][st]. + +[st]: https://gitlab.com/DD-workspace/DD/-/tree/main/stress-tests + + +## VM tests + +- `vm-test.sh`: generate a text file to compare CPU and other factors across + VM types, providers or instances which may affect DD performance. + +We can compare the resulting lgos just with, e.g. `vim -d`. + + +## DD tests + +Currently these tests perform logins and interact with Nextcloud, but it would +be interesting to expand them to interact with other services. + +### Directory contents + +This directory contains following files: + +- `dd-stress-test.tpl.jmx`: template to generate [JMeter][jm] tests to execute +- `dd-tests.sh`: helper script that generates the actual test plan files and + executes them. See `./dd-tests.sh --help` +- `dd-test-selenium.sh`: this gives us an idea of how a user would perceive + DD to be behaving while under load. Called by `./dd-tests.sh` by default. + + +### Results + +Results will be saved in a `results` directory, where each subdirectory +corresponds to a stress test executed with `dd-test.sh`. + +The naming scheme for those subdirectories is: `DOMAIN_THREADCOUNT_DURATION` +Where `THREADCOUNT` and `DURATION` are the corresponding [JMeter][jm] +parameters. + +Each results directory contains: + +- `log`: the [JMeter][jm] log +- `results`: the [JMeter][jm] results file +- `html/index.html`: the interactive graphs for the data as produced by JMeter +- `selenium/session.html`: the session report as would be perceived by a user. + Note this requires Python3 and selenium and can be disabled by setting the + environment variable: `USE_SELENIUM=NO`. + +[jm]: https://jmeter.apache.org/ diff --git a/stress-tests/dd-stress-test.tpl.jmx b/stress-tests/dd-stress-test.tpl.jmx new file mode 100644 index 0000000..060d0d8 --- /dev/null +++ b/stress-tests/dd-stress-test.tpl.jmx @@ -0,0 +1,822 @@ + + + + + + + false + false + + + + + + + + + + ThreadCount + 10 + = + Number of Threads + + + Duration + 60 + = + in seconds + + + RampUpPeriod + 10 + = + in seconds + + + IdPHost + https://sso.DD_DOMAIN + = + The main URL of your SSO Instance + + + IdPPort + 80 + = + + + IdPContext + idp + = + + + startupDelay + 1 + = + in seconds + + + LoginSP + https://nextcloud.DD_DOMAIN/apps/user_saml/saml/login + = + Domain of registered NC using SAML + + + ProviderId + https://nextcloud.DD_DOMAIN/apps/user_saml/saml/metadata + = + SAML EntityId for NC + + + ACSUrl + https://nextcloud.DD_DOMAIN/apps/user_saml/saml/acs + SP ACS Url to post the final response/assertion back + = + + + NextCloud + nextcloud.DD_DOMAIN + = + + + ClientId + https://nextcloud.DD_DOMAIN/apps/user_saml/saml/metadata + = + + + Environment Variables that will need to be updated + + + + + + + + ./dd-stress-test.users.csv + + User,Password + , + false + true + false + shareMode.all + No Spaces between User, comma, and Password fields!! + false + + + + ./dd-stress-test.docs.csv + + folder,document + , + false + true + false + shareMode.all + No Spaces between Folder, comma, and DocumentID fields!! + false + + + + + true + false + + + + + + + + + https + + + 4 + HttpClient4 + 120000 + 120000 + + + + stopthread + + false + -1 + + ${ThreadCount} + ${RampUpPeriod} + 1500501650000 + 1500501650000 + true + ${Duration} + ${StartupDelay} + SAML Support Login Process + true + true + + + + + Test Plan + NextCloud Tests + NEXTCLOUD Login + + SAML2 SP calls the login page for CAS IdP + + + + + Test Plan + NextCloud Tests + POST - Login User + + Logging into CAS Idp + + + + + Test Plan + NextCloud Tests + POST - Authorization to NEXTCLOUD + + Send response from login to SP for processing + + + + + Test Plan + NextCloud Tests + GET - Document + + Send response from login to SP for processing + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + + + + + + + + + + + ${LoginSP}/login + GET + true + false + true + true + + + + Calling secured SP page, that should then redirect to CAS login page + + + + + + Sec-Fetch-Mode + navigate + + + Sec-Fetch-Site + none + + + Accept-Language + en-US,en;q=0.5 + + + Sec-Fetch-User + ?1 + + + Pragma + no-cache + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8 + + + Upgrade-Insecure-Requests + 1 + + + Cache-Control + no-cache + + + Accept-Encoding + gzip, deflate, br + + + User-Agent + Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0 + + + Sec-Fetch-Dest + document + + + + + + false + RelayState + <input type="hidden" name="RelayState" value="(.+?)" + $1$ + + 1 + all + + + + false + SAMLResponse + <input type="hidden" name="SAMLResponse" value="(.+?)" + $1$ + + 1 + all + + + + false + Execution + \;execution=(.+?)&amp; + $1$ + + + all + + + + false + SessionCode + session_code=(.*)&amp;ex + $1$ + + + all + + + + false + tab_id + \;tab_id=(.*)\" me + $1$ + + + all + + + + URL + SAMLRequest + SAMLRequest=(.+) + $1$ + + 1 + all + + + + + + + + + + + true + ${User} + = + true + username + false + + + true + ${Password} + = + true + password + false + + + + + + + + ${IdPHost}/auth/realms/master/login-actions/authenticate?session_code=${SessionCode}&execution=${Execution}&client_id=${__urlencode(${ClientId})}&tab_id=${tab_id} + POST + true + false + true + false + + + + POST Login Credentials for SAMLResponse + + + + unescaped + RelayState + <input type="hidden" name="RelayState" value="(.+?)" + $1$ + + 1 + all + + + + unescaped + SAMLResponse + <input type="hidden" name="SAMLResponse" value="(.+?)" + $1$ + + 1 + all + + + + + + + + + + + true + ${RelayState} + = + true + RelayState + + + true + ${SAMLResponse} + = + true + SAMLResponse + + + + + + + + ${ACSUrl} + POST + true + false + true + true + + + + + + + + + + + + + true + OCS-APIREQUEST + true + = + true + + + + ${NextCloud} + + https + + + GET + true + false + true + false + + + + + + + false + requestToken + data-requesttoken="(.+)\" + $1$ + + + all + + + + + + + + false + dir + Documents + = + true + + + + ${NextCloud} + + https + + /apps/files + GET + true + false + true + true + + + + + + + + + Accept + application/json, text/plain, */* + + + requesttoken + ${requestToken} + + + + + + + true + + + + false + <?xml version="1.0"?> +<d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns" xmlns:ocs="http://open-collaboration-services.org/ns"> + <d:prop> + <d:getlastmodified /> + <d:getetag /> + <d:getcontenttype /> + <d:resourcetype /> + <oc:fileid /> + <oc:permissions /> + <oc:size /> + <d:getcontentlength /> + <d:quota-available-bytes /> + <nc:has-preview /> + <nc:mount-type /> + <nc:is-encrypted /> + <ocs:share-permissions /> + <nc:share-attributes /> + <oc:tags /> + <oc:favorite /> + <oc:owner-id /> + <oc:owner-display-name /> + <oc:share-types /> + <oc:comments-unread /> + </d:prop> +</d:propfind> + = + + + + ${NextCloud} + 443 + https + UTF-8 + /remote.php/dav/files/${User} + PROPFIND + true + false + true + false + + + + + + + + + Sec-Fetch-Mode + cors + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.5 + + + Origin + https://${NextCloud} + + + Accept + text/plain + + + Depth + 0 + + + X-Requested-With + XMLHttpRequest + + + Content-Type + text/plain;charset=UTF-8 + + + Accept-Encoding + gzip, deflate, br + + + User-Agent + Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0 + + + Sec-Fetch-Dest + empty + + + requesttoken + ${requestToken} + + + + + + + + + + false + file + https://${NextCloud}/remote.php/dav/files/${User}/${document} + = + true + + + false + canDownload + 1 + = + true + + + + ${NextCloud} + 443 + https + + /apps/files_pdfviewer/ + GET + true + false + true + false + + + + + + + + + Sec-Fetch-Mode + navigate + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.5 + + + Upgrade-Insecure-Requests + 1 + + + Accept-Encoding + gzip, deflate, br + + + User-Agent + Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0 + + + Accept + text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8 + + + Sec-Fetch-Dest + iframe + + + requesttoken + ${requestToken} + + + + + + + + + + ${NextCloud} + 443 + https + utf-8 + /remote.php/dav/files/${User}/${document} + GET + true + false + true + false + + + + + + + + + Sec-Fetch-Mode + cors + + + Sec-Fetch-Site + same-origin + + + Accept-Language + en-US,en;q=0.5 + + + Accept-Encoding + gzip, deflate, br + + + User-Agent + Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0 + + + Accept + */* + + + Sec-Fetch-Dest + empty + + + requesttoken + ${requestToken} + + + + + + + + + diff --git a/stress-tests/dd-test-selenium.py b/stress-tests/dd-test-selenium.py new file mode 100644 index 0000000..2dc94af --- /dev/null +++ b/stress-tests/dd-test-selenium.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 + +import argparse +import io +import pathlib +import random +import traceback +from datetime import datetime + +import yaml + +from selenium import webdriver + +# We need selenium 4 +from selenium.webdriver.common.by import By +from selenium.webdriver.support.wait import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.chrome.options import Options as ChromeOptions + +from typing import Any, Dict, Iterable, List + + +class DDSession: + driver: webdriver.Chrome + + def __init__( + self, + instance: str, + users_file: str, + out_dir: pathlib.Path, + stepTimeoutSeconds: int = 20, + ): + self.instance = instance + self.users_file = [ + [i.strip() for i in l.split(sep=",", maxsplit=1)] + for l in users_file.split("\n") + if l + ] + self.out_dir = out_dir + self.stepTimeoutSeconds = stepTimeoutSeconds + + if not self.out_dir.is_dir(): + self.out_dir.mkdir() + + self.start_time = datetime.utcnow() + self.last_time = self.start_time + self.screenshot_counter = 0 + self.data: Dict[str, Any] = dict( + instance=self.instance, start_time=self.start_time, steps=[] + ) + self.reset() + + def persist(self) -> None: + yaml.dump(self.data, (self.out_dir / "session.yml").open("w")) + (self.out_dir / "session.html").open("w").write(self.summarise()["html"]) + + def summarise(self) -> Dict[str, Any]: + return DDSession.summarise_data(self.data) + + @staticmethod + def summarise_data(data) -> Dict[str, Any]: + import copy + from operator import getitem + from itertools import groupby + + def summarise_item(g: List[Dict[str, Any]]) -> Dict[str, Any]: + if "substeps" in g[0]: + items = [sum((st["step_t"] for st in s["substeps"])) for s in g] + else: + items = [st["step_t"] for st in g] + return { + "max": max(items), + "count": len(items), + "average": sum(items) / len(items), + } + + def summarise(d: Iterable[Dict[str, Any]]) -> Dict[str, Any]: + dict_data = dict() + for k, g in groupby( + sorted(d, key=lambda x: getitem(x, "step")), + key=lambda x: getitem(x, "step"), + ): + dict_data[k] = summarise_item(list(g)) + return dict_data + + summary: Dict[str, Any] = dict() + summary["overview"] = summarise(data["steps"]) + + d = copy.deepcopy(data["steps"]) + substeps = [ + dict(st, step=f"{step['step']}: {st['substep']}") + for step in data["steps"] + for st in step["substeps"] + ] + summary["stepbystep"] = summarise(substeps) + + html = """ + + +""" + html += f""" +

{data['instance']}

+

{data['start_time']}

+
Summary +
{yaml.dump(summary)}
+
+
+""" + for s in data["steps"]: + html += f"""

{s['step']}: {s['t']} s (+ {sum((st['step_t'] for st in s['substeps']))} s)

""" + for st in s["substeps"]: + html += f"""

{s['step']} - {st['substep']}: {st['t']} s (+ {st['step_t']} s)

""" + html += f"""""" + html += "" + summary["html"] = html + return summary + + @property + def executed_seconds(self) -> float: + return (datetime.utcnow() - self.start_time).total_seconds() + + @property + def nextcloud_url(self) -> str: + return f"https://nextcloud.{self.instance}" + + def reset(self, new_driver: bool = False) -> None: + # If needed: + # https://stackoverflow.com/a/72922584 + if getattr(self, "driver", None) is None or new_driver: + if getattr(self, "driver", None) is not None: + self.driver.quit() + options = ChromeOptions() + for arg in [ + "--headless", + "--no-sandbox", + "--disable-infobars", + "--disable-extensions", + "--disable-dev-shm-usage", + "window-size=1400,600", + ]: + options.add_argument(arg) + self.driver = webdriver.Chrome(options=options) + msg = "Restart" + else: + self.driver.delete_all_cookies() + msg = "Cleaned cache" + self.screenshot("Browser", msg) + + def perform_login(self) -> None: + # Wait until Keycloak is shown + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: d.find_elements(By.ID, "kc-form-login") + ) + + self.screenshot("Login", "Request") + + # Fill in form data for login + (username, password) = random.choice(self.users_file) + usernameField = self.driver.find_element(By.ID, "username") + usernameField.send_keys(username) + passwordField = self.driver.find_element(By.ID, "password") + passwordField.send_keys(password) + loginButton = self.driver.find_element(By.ID, "kc-login") + self.screenshot("Login", "Form filled") + + loginButton.click() + + self.screenshot("Login", "Form sent") + # TODO: wait for redirection middle step and session.screenshot it + + def load_nextcloud_files(self) -> None: + # Start loading + self.driver.get(self.nextcloud_url) + + # Wait until Nextcloud with Megamenu is shown or a login is requested + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: ( + d.find_elements(By.ID, "kc-form-login") + + d.find_elements(By.ID, "menu-apps-icon") + ) + ) + + # Detect whether or not a login is needed + login_required = self.driver.find_elements(By.ID, "kc-form-login") + if login_required: + # Perform login + self.perform_login() + + self.screenshot("Nextcloud", "Loading") + + # Wait until Nextcloud files app has loaded + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: d.find_elements( + By.CSS_SELECTOR, "#app-content-files td.selection" + ) + ) + self.screenshot("Nextcloud", "Loaded") + + def open_file_in_onlyoffice(self, file_base_name: str) -> None: + # Get and click docx file + # This assumes we are on the nextcloud files app with a FILE_BASE file + window_count = len(self.driver.window_handles) + docxFileLabel = self.driver.find_element(By.PARTIAL_LINK_TEXT, file_base_name) + docxFileLabel.click() + + # Switch to OnlyOffice window + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + EC.number_of_windows_to_be(window_count + 1) + ) + child = self.driver.window_handles[window_count] + self.driver.switch_to.window(child) + + self.screenshot("OnlyOffice", "Opening") + + # Wait for OnlyOffice to start loading + oofCSS = "div.app-onlyoffice #app iframe" + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: d.find_elements(By.CSS_SELECTOR, oofCSS) + ) + self.screenshot("OnlyOffice", "Loading 1") + + # Switch to its iframe + oof = WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: EC.frame_to_be_available_and_switch_to_it( + d.find_element(By.CSS_SELECTOR, oofCSS) + )(d) + ) + self.screenshot("OnlyOffice", "Loading 2") + + oofLoaded = lambda d: EC.element_to_be_clickable( + d.find_element(By.ID, "id-toolbar-btn-save") + )(d) + + # Get the first loading screen + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: oofLoaded(d) + or d.find_elements(By.CSS_SELECTOR, "#loading-mask div.loader-page") + ) + self.screenshot("OnlyOffice", "Loading 3") + + # Wait for OnlyOffice second loading phase + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds).until( + lambda d: oofLoaded(d) + or d.find_elements(By.CSS_SELECTOR, "div.asc-loadmask-title") + ) + self.screenshot("OnlyOffice", "Loading 4") + + # Wait a for final loading (save button clickable) + WebDriverWait(self.driver, timeout=self.stepTimeoutSeconds * 3).until(oofLoaded) + self.screenshot("OnlyOffice", "Loaded") + + # Close OnlyOffice window + self.driver.close() + # And change back to the main window + self.driver.switch_to.window(self.driver.window_handles[window_count - 1]) + self.screenshot("OnlyOffice", "Closed") + + def screenshot(self, step: str, substep_txt: str) -> None: + # Get data + scr_now = datetime.utcnow() + scr_s = (scr_now - self.start_time).total_seconds() + step_t = (scr_now - self.last_time).total_seconds() + scr_id = self.screenshot_counter + + # Upgrade values + self.last_time = scr_now + self.screenshot_counter += 1 + + # Constants + scr_fn = f"{scr_id:04d}.png" + scr_path = self.out_dir / scr_fn + # Get screenie + scr_img = self.driver.get_screenshot_as_png() + # Write it + open(scr_path, "wb").write(scr_img) + # Add to report + substep: Dict[str, Any] = dict( + id=scr_id, substep=substep_txt, step_t=step_t, t=scr_s, png=scr_fn + ) + if self.data["steps"] and self.data["steps"][-1]["step"] == step: + self.data["steps"][-1]["substeps"].append(substep) + else: + self.data["steps"].append(dict(step=step, t=scr_s, substeps=[substep])) + + +def randomBool() -> bool: + return random.choice([True, False]) + + +def main_test( + instance: str, + users_file: io.TextIOWrapper, + out_dir: pathlib.Path, + duration: int, + stepTimeoutSeconds: int = 20, +) -> DDSession: + session = DDSession( + instance=instance, + users_file=users_file.read(), + out_dir=out_dir, + stepTimeoutSeconds=20, + ) + + first_run = True + while session.executed_seconds < duration: + # 50% chance of requiring login reseting cookies + # First run always requires login, skip reset + if not first_run and randomBool(): + # Possibly driver too + # 25% chance of cleaning all cache and reopening the browser + session.reset(new_driver=randomBool()) + first_run = False + + try: + # Load nextcloud files + session.load_nextcloud_files() + # Open file in OnlyOffice + session.open_file_in_onlyoffice("template_1") + except Exception as ex: + session.screenshot("Exception", str(ex)) + print(traceback.format_exc()) + session.persist() + return session + + +def main_summary(filename: io.TextIOWrapper) -> None: + data = yaml.safe_load(filename) + summary = DDSession.summarise_data(data) + html_fn = pathlib.Path(filename.name).with_suffix(".html") + open(html_fn, "w").write(summary["html"]) + + +if __name__ == "__main__": + import pathlib + + parser = argparse.ArgumentParser( + prog="DD Selenium tester", description="Run basic UI tests on DD" + ) + subparsers = parser.add_subparsers(required=True) + test_parser = subparsers.add_parser("test") + + test_parser.add_argument("instance") + test_parser.add_argument( + "-u", + "--users-file", + default="dd-stress-test.users.csv", + type=argparse.FileType("r"), + ) + test_parser.add_argument("-d", "--duration", default=300, type=int) + test_parser.add_argument("-o", "--out-dir", default="results", type=pathlib.Path) + + summary_parser = subparsers.add_parser("summary") + summary_parser.add_argument( + "filename", default="session.yml", type=argparse.FileType("r") + ) + + ns = parser.parse_args() + if "instance" in ns: + main_test(**vars(ns)) + else: + main_summary(**vars(ns)) diff --git a/stress-tests/dd-test.sh b/stress-tests/dd-test.sh new file mode 100755 index 0000000..e5d1985 --- /dev/null +++ b/stress-tests/dd-test.sh @@ -0,0 +1,180 @@ +#!/bin/sh -eu + +# Process inputs +DD_DOMAIN="${1:-}" +tc="${2:-}" +duration="${3:-60}" +USE_SELENIUM="${USE_SELENIUM:-YES}" +SCRIPT_PATH="$(realpath "${0}")" +SCRIPT_NAME="$(basename "${0}")" + +JMETER_DEFAULT="./apache-jmeter-5.5/bin/jmeter" + +full_tests() { + # Runtime: 7 tests * 5 mins / test = 35 mins + # Cool-off periods: 30s * 6 = 3 mins + # Total: 38 mins + cooloff="30" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 10 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 20 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 30 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 60 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 100 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 300 300 + sleep "${cooloff}" + "${SCRIPT_PATH}" "${DD_DOMAIN}" 600 300 +} + +help_users_file() { + cat <<-EOF + The format of the users.csv file must be: + USERNAME1,PASSWORD1 + USERNAME2,PASSWORD2 + ... + + Take care not to have any spaces between fields. + EOF +} + +help_jmeter() { + cat <<-EOF + Note this scripts depends on JMeter with some plugins enabled. + You can set the JMETER environment variable to its binary path. + If this variable is unset, ${JMETER_DEFAULT} + will be used, from this script's location. + See: + https://jmeter.apache.org/download_jmeter.cgi + https://jmeter-plugins.org/install/Install/ + EOF +} + +help() { + cat <<-EOF + Examples: + ./${SCRIPT_NAME} DD_DOMAIN THREAD_COUNT [DURATION] + or: + ./${SCRIPT_NAME} --full-tests DD_DOMAIN + + EOF + + help_jmeter + + cat <<-EOF + + When using --full-tests, a pre-selected combination of + THREAD_COUNT and DURATION will be used against DD_DOMAIN. + + Where DD_DOMAIN is the base domain, e.g. if your DD instance's + Nextcloud can be accessed at nextcloud.example.org, the parameter + should be "example.org". + + THREAD_COUNT refers to the amount of users that will be simulated. + + DURATION is the total test time time in seconds. Defaults to 60. + + Note that you MUST have a users.csv file in the current directory. + + By default this script runs tests with selenium and documents the + session as would be perceived by a user. + You can disable this behaviour by setting the environment variable + USE_SELENIUM=NO. + EOF + help_users_file +} + +if [ "${DD_DOMAIN:-}" = "--full-tests" ]; then + shift # Consume operation argument + # Re-set global variable + DD_DOMAIN="${1:-}" + # Execute full suite + full_tests + # And exit + exit 0 +elif [ "${1:-}" = "--help" ]; then + help + exit 0 +elif [ -z "${DD_DOMAIN:-}" ] || [ -z "${tc:-}" ]; then + help >> /dev/stderr + exit 1 +fi + +USERS_FILE="$(pwd)/users.csv" +DOCS_FILE="$(pwd)/docs.csv" + +# Change current path +cd "$(dirname "${SCRIPT_PATH}")" +out_dir="$(pwd)/results/${DD_DOMAIN}_${tc}_${duration}" + +if [ -f "${USERS_FILE}" ]; then + cat "${USERS_FILE}" > dd-stress-test.users.csv +else + printf "ERROR: missing file\t%s\n\n" "${USERS_FILE}" >> /dev/stderr + help_users_file >> /dev/stderr + exit 2 +fi + +if [ -f "${DOCS_FILE}" ]; then + cat "${DOCS_FILE}" > dd-stress-test.docs.csv +else + cat > dd-stress-test.docs.csv <<-EOF + /,Readme.md + /,template.docx + /,template_1.docx + EOF +fi + + +JMETER="${JMETER:-${JMETER_DEFAULT}}" +# Ensure JMeter is available / bootstrap it +if [ ! -f "${JMETER}" ]; then + echo "INFO: Could not find JMeter, attempting to download it" >> /dev/stderr + curl -L 'https://dlcdn.apache.org//jmeter/binaries/apache-jmeter-5.5.tgz' | tar -xz + JMETER="${JMETER_DEFAULT}" +fi +if [ ! -f "${JMETER}" ]; then + printf "ERROR: missing JMeter\t%s\n\n" "${JMETER}" >> /dev/stderr +fi +# Ensure JMeter plugins are available / bootstrap them +JMETER_PLUGINS="$(dirname "${JMETER}")/../lib/ext/jmeter-plugins-manager-1.8.jar" +if [ ! -f "${JMETER_PLUGINS}" ]; then + echo "INFO: Could not find JMeter plugins, attempting to download them" >> /dev/stderr + curl -L 'https://jmeter-plugins.org/get/' > "${JMETER_PLUGINS}" +fi +if [ ! -f "${JMETER_PLUGINS}" ]; then + printf "ERROR: missing JMeter plugins\t%s\n\n" "${JMETER_PLUGINS}" >> /dev/stderr +fi +if [ ! -f "${JMETER}" ] || [ ! -f "${JMETER_PLUGINS}" ]; then + help_jmeter >> /dev/stderr + exit 3 +fi + +# Clean up out dir +rm -rf "${out_dir}" +mkdir -p "${out_dir}" + +# Adapt template +sed -E \ + -e "s%([^>]*)>(.*)<\!-- TC.*$%\\1>${tc} <\!-- TC -->%" \ + -e "s%([^>]*)>(.*)<\!-- DURATION.*$%\\1>${duration} <\!-- DURATION -->%" \ + -e "s/DD_DOMAIN/${DD_DOMAIN}/g" \ + dd-stress-test.tpl.jmx > dd-stress-test.jmx + +# Call Selenium test process in parallel +if [ "${USE_SELENIUM}" = "YES" ]; then + printf "\n\nRunning parallel Selenium-based tests:\t%s\tover %s seconds\n\n" "${DD_DOMAIN}" "${duration}" + python3 dd-test-selenium.py test --duration "${duration}" --out-dir "${out_dir}/selenium" "${DD_DOMAIN}" 2>&1 > "${out_dir}/selenium.log" & +fi + +# Execute test +printf "\n\nAbout to test:\t%s\twith %s 'users' over %s seconds\n\n" \ + "${DD_DOMAIN}" "${tc}" "${duration}" +env HEAP="-Xms2g -Xmx2g -XX:MaxMetaspaceSize=2g" "${JMETER}" -n -t dd-stress-test.jmx -l "${out_dir}/results" -e -o "${out_dir}/html" +mv jmeter.log "${out_dir}/log" + +# Notify results +printf "\n\nYou can find the results at:\t%s\n\n" "${out_dir}" diff --git a/stress-tests/requirements.txt b/stress-tests/requirements.txt new file mode 100644 index 0000000..28cd789 --- /dev/null +++ b/stress-tests/requirements.txt @@ -0,0 +1,20 @@ +async-generator==1.10 +attrs==22.2.0 +certifi==2022.12.7 +exceptiongroup==1.1.0 +h11==0.14.0 +idna==3.4 +outcome==1.2.0 +pip==22.3.1 +PySocks==1.7.1 +PyYAML==6.0 +selenium==4.8.2 +setuptools==67.0.0 +sniffio==1.3.0 +sortedcontainers==2.4.0 +sqlite3==0.0.0 +trio==0.22.0 +trio-websocket==0.9.2 +urllib3==1.26.14 +wheel==0.38.4 +wsproto==1.2.0 diff --git a/stress-tests/vm-test.sh b/stress-tests/vm-test.sh new file mode 100644 index 0000000..6d093ab --- /dev/null +++ b/stress-tests/vm-test.sh @@ -0,0 +1,26 @@ +#!/bin/sh -eu + +LOG_FILE="${LOG_FILE:-vm-test.log}" + +SYSBENCH="$(command -v sysbench2)" + +# Save stderr as well +exec 2>&1 + +echo "$(date)" | tee "${LOG_FILE}" + +if [ -z "${SYSBENCH}" ]; then + echo "Skipping: sysbench tests (try: apt install sysbench)" | \ + tee -a "${LOG_FILE}" +else + printf "\n\nfileio\n\n" | tee -a "${LOG_FILE}" + "${SYSBENCH}" fileio prepare --file-test-mode=rndrw --threads=4 --time=60 | tee -a "${LOG_FILE}" + "${SYSBENCH}" fileio run --file-test-mode=rndrw --threads=4 --time=60 | tee -a "${LOG_FILE}" + printf "\n\ncpu\n\n" | tee -a "${LOG_FILE}" + "${SYSBENCH}" cpu run --threads=4 --time=60 | tee -a "${LOG_FILE}" + printf "\n\nmemory\n\n" | tee -a "${LOG_FILE}" + "${SYSBENCH}" memory run --threads=4 --time=60 | tee -a "${LOG_FILE}" +fi +# Perform basic OpenSSL tests too +printf "\n\nCPU OpenSSL\n\n" | tee -a "${LOG_FILE}" +openssl speed --seconds 60 sha256 | tee -a "${LOG_FILE}"