alioth/nixpkgs
3
0
Fork 0
forked from mirrors/nixpkgs
Code Releases Activity

Merge pull request #146905 from Synthetica9/failure_mode

Browse source 
nixos/test-driver: add polling_condition
This commit is contained in:
Jacek Galowicz 2022-01-08 18:28:25 +01:00 committed by GitHub
parent 41216b003f 793a2f50f1
commit 048fd95f10
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 673 additions and 419 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
nixos/doc/manual/development/writing-nixos-tests.section.md
View file

@ -88,6 +88,8 @@ starting them in parallel:
start_all()
```
## Machine objects {#ssec-machine-objects}
The following methods are available on machine objects:
`start`
@ -313,3 +315,52 @@ repository):
# fmt: on
'';
```
## Failing tests early {#ssec-failing-tests-early}
To fail tests early when certain invariables are no longer met (instead of waiting for the build to time out), the decorator `polling_condition` is provided. For example, if we are testing a program `foo` that should not quit after being started, we might write the following:
```py
@polling_condition
def foo_running():
machine.succeed("pgrep -x foo")
machine.succeed("foo --start")
machine.wait_until_succeeds("pgrep -x foo")
with foo_running:
... # Put `foo` through its paces
```
`polling_condition` takes the following (optional) arguments:
`seconds_interval`
:
specifies how often the condition should be polled:
```py
@polling_condition(seconds_interval=10)
def foo_running():
machine.succeed("pgrep -x foo")
```
`description`
:
is used in the log when the condition is checked. If this is not provided, the description is pulled from the docstring of the function. These two are therefore equivalent:
```py
@polling_condition
def foo_running():
"check that foo is running"
machine.succeed("pgrep -x foo")
```
```py
@polling_condition(description="check that foo is running")
def foo_running():
machine.succeed("pgrep -x foo")
```

858
nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml
View file

@ -117,407 +117,413 @@ if not "Linux" in machine.succeed("uname"):
<programlisting language="python">
start_all()
</programlisting>
<para>
The following methods are available on machine objects:
</para>
<variablelist>
<varlistentry>
<term>
<literal>start</literal>
</term>
<listitem>
<para>
Start the virtual machine. This method is asynchronous — it
does not wait for the machine to finish booting.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>shutdown</literal>
</term>
<listitem>
<para>
Shut down the machine, waiting for the VM to exit.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>crash</literal>
</term>
<listitem>
<para>
Simulate a sudden power failure, by telling the VM to exit
immediately.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>block</literal>
</term>
<listitem>
<para>
Simulate unplugging the Ethernet cable that connects the
machine to the other machines.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>unblock</literal>
</term>
<listitem>
<para>
Undo the effect of <literal>block</literal>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>screenshot</literal>
</term>
<listitem>
<para>
Take a picture of the display of the virtual machine, in PNG
format. The screenshot is linked from the HTML log.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>get_screen_text_variants</literal>
</term>
<listitem>
<para>
Return a list of different interpretations of what is
currently visible on the machine's screen using optical
character recognition. The number and order of the
interpretations is not specified and is subject to change, but
if no exception is raised at least one will be returned.
</para>
<note>
<section xml:id="ssec-machine-objects">
<title>Machine objects</title>
<para>
The following methods are available on machine objects:
</para>
<variablelist>
<varlistentry>
<term>
<literal>start</literal>
</term>
<listitem>
<para>
This requires passing <literal>enableOCR</literal> to the
test attribute set.
Start the virtual machine. This method is asynchronous — it
does not wait for the machine to finish booting.
</para>
</note>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>get_screen_text</literal>
</term>
<listitem>
<para>
Return a textual representation of what is currently visible
on the machine's screen using optical character recognition.
</para>
<note>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>shutdown</literal>
</term>
<listitem>
<para>
This requires passing <literal>enableOCR</literal> to the
test attribute set.
Shut down the machine, waiting for the VM to exit.
</para>
</note>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>send_monitor_command</literal>
</term>
<listitem>
<para>
Send a command to the QEMU monitor. This is rarely used, but
allows doing stuff such as attaching virtual USB disks to a
running machine.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>send_key</literal>
</term>
<listitem>
<para>
Simulate pressing keys on the virtual keyboard, e.g.,
<literal>send_key(&quot;ctrl-alt-delete&quot;)</literal>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>send_chars</literal>
</term>
<listitem>
<para>
Simulate typing a sequence of characters on the virtual
keyboard, e.g.,
<literal>send_chars(&quot;foobar\n&quot;)</literal> will type
the string <literal>foobar</literal> followed by the Enter
key.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>execute</literal>
</term>
<listitem>
<para>
Execute a shell command, returning a list
<literal>(status, stdout)</literal>. If the command detaches,
it must close stdout, as <literal>execute</literal> will wait
for this to consume all output reliably. This can be achieved
by redirecting stdout to stderr <literal>&gt;&amp;2</literal>,
to <literal>/dev/console</literal>,
<literal>/dev/null</literal> or a file. Examples of detaching
commands are <literal>sleep 365d &amp;</literal>, where the
shell forks a new process that can write to stdout and
<literal>xclip -i</literal>, where the
<literal>xclip</literal> command itself forks without closing
stdout. Takes an optional parameter
<literal>check_return</literal> that defaults to
<literal>True</literal>. Setting this parameter to
<literal>False</literal> will not check for the return code
and return -1 instead. This can be used for commands that shut
down the VM and would therefore break the pipe that would be
used for retrieving the return code.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>succeed</literal>
</term>
<listitem>
<para>
Execute a shell command, raising an exception if the exit
status is not zero, otherwise returning the standard output.
Commands are run with <literal>set -euo pipefail</literal>
set:
</para>
<itemizedlist>
<listitem>
<para>
If several commands are separated by <literal>;</literal>
and one fails, the command as a whole will fail.
</para>
</listitem>
<listitem>
<para>
For pipelines, the last non-zero exit status will be
returned (if there is one, zero will be returned
otherwise).
</para>
</listitem>
<listitem>
<para>
Dereferencing unset variables fail the command.
</para>
</listitem>
<listitem>
<para>
It will wait for stdout to be closed. See
<literal>execute</literal> for the implications.
</para>
</listitem>
</itemizedlist>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>fail</literal>
</term>
<listitem>
<para>
Like <literal>succeed</literal>, but raising an exception if
the command returns a zero status.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_until_succeeds</literal>
</term>
<listitem>
<para>
Repeat a shell command with 1-second intervals until it
succeeds.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_until_fails</literal>
</term>
<listitem>
<para>
Repeat a shell command with 1-second intervals until it fails.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_unit</literal>
</term>
<listitem>
<para>
Wait until the specified systemd unit has reached the
<quote>active</quote> state.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_file</literal>
</term>
<listitem>
<para>
Wait until the specified file exists.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_open_port</literal>
</term>
<listitem>
<para>
Wait until a process is listening on the given TCP port (on
<literal>localhost</literal>, at least).
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_closed_port</literal>
</term>
<listitem>
<para>
Wait until nobody is listening on the given TCP port.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_x</literal>
</term>
<listitem>
<para>
Wait until the X11 server is accepting connections.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_text</literal>
</term>
<listitem>
<para>
Wait until the supplied regular expressions matches the
textual contents of the screen by using optical character
recognition (see <literal>get_screen_text</literal> and
<literal>get_screen_text_variants</literal>).
</para>
<note>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>crash</literal>
</term>
<listitem>
<para>
This requires passing <literal>enableOCR</literal> to the
test attribute set.
Simulate a sudden power failure, by telling the VM to exit
immediately.
</para>
</note>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_console_text</literal>
</term>
<listitem>
<para>
Wait until the supplied regular expressions match a line of
the serial console output. This method is useful when OCR is
not possibile or accurate enough.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_window</literal>
</term>
<listitem>
<para>
Wait until an X11 window has appeared whose name matches the
given regular expression, e.g.,
<literal>wait_for_window(&quot;Terminal&quot;)</literal>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>copy_from_host</literal>
</term>
<listitem>
<para>
Copies a file from host to machine, e.g.,
<literal>copy_from_host(&quot;myfile&quot;, &quot;/etc/my/important/file&quot;)</literal>.
</para>
<para>
The first argument is the file on the host. The file needs to
be accessible while building the nix derivation. The second
argument is the location of the file on the machine.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>systemctl</literal>
</term>
<listitem>
<para>
Runs <literal>systemctl</literal> commands with optional
support for <literal>systemctl --user</literal>
</para>
<programlisting language="python">
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>block</literal>
</term>
<listitem>
<para>
Simulate unplugging the Ethernet cable that connects the
machine to the other machines.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>unblock</literal>
</term>
<listitem>
<para>
Undo the effect of <literal>block</literal>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>screenshot</literal>
</term>
<listitem>
<para>
Take a picture of the display of the virtual machine, in PNG
format. The screenshot is linked from the HTML log.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>get_screen_text_variants</literal>
</term>
<listitem>
<para>
Return a list of different interpretations of what is
currently visible on the machine's screen using optical
character recognition. The number and order of the
interpretations is not specified and is subject to change,
but if no exception is raised at least one will be returned.
</para>
<note>
<para>
This requires passing <literal>enableOCR</literal> to the
test attribute set.
</para>
</note>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>get_screen_text</literal>
</term>
<listitem>
<para>
Return a textual representation of what is currently visible
on the machine's screen using optical character recognition.
</para>
<note>
<para>
This requires passing <literal>enableOCR</literal> to the
test attribute set.
</para>
</note>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>send_monitor_command</literal>
</term>
<listitem>
<para>
Send a command to the QEMU monitor. This is rarely used, but
allows doing stuff such as attaching virtual USB disks to a
running machine.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>send_key</literal>
</term>
<listitem>
<para>
Simulate pressing keys on the virtual keyboard, e.g.,
<literal>send_key(&quot;ctrl-alt-delete&quot;)</literal>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>send_chars</literal>
</term>
<listitem>
<para>
Simulate typing a sequence of characters on the virtual
keyboard, e.g.,
<literal>send_chars(&quot;foobar\n&quot;)</literal> will
type the string <literal>foobar</literal> followed by the
Enter key.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>execute</literal>
</term>
<listitem>
<para>
Execute a shell command, returning a list
<literal>(status, stdout)</literal>. If the command
detaches, it must close stdout, as
<literal>execute</literal> will wait for this to consume all
output reliably. This can be achieved by redirecting stdout
to stderr <literal>&gt;&amp;2</literal>, to
<literal>/dev/console</literal>,
<literal>/dev/null</literal> or a file. Examples of
detaching commands are <literal>sleep 365d &amp;</literal>,
where the shell forks a new process that can write to stdout
and <literal>xclip -i</literal>, where the
<literal>xclip</literal> command itself forks without
closing stdout. Takes an optional parameter
<literal>check_return</literal> that defaults to
<literal>True</literal>. Setting this parameter to
<literal>False</literal> will not check for the return code
and return -1 instead. This can be used for commands that
shut down the VM and would therefore break the pipe that
would be used for retrieving the return code.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>succeed</literal>
</term>
<listitem>
<para>
Execute a shell command, raising an exception if the exit
status is not zero, otherwise returning the standard output.
Commands are run with <literal>set -euo pipefail</literal>
set:
</para>
<itemizedlist>
<listitem>
<para>
If several commands are separated by
<literal>;</literal> and one fails, the command as a
whole will fail.
</para>
</listitem>
<listitem>
<para>
For pipelines, the last non-zero exit status will be
returned (if there is one, zero will be returned
otherwise).
</para>
</listitem>
<listitem>
<para>
Dereferencing unset variables fail the command.
</para>
</listitem>
<listitem>
<para>
It will wait for stdout to be closed. See
<literal>execute</literal> for the implications.
</para>
</listitem>
</itemizedlist>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>fail</literal>
</term>
<listitem>
<para>
Like <literal>succeed</literal>, but raising an exception if
the command returns a zero status.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_until_succeeds</literal>
</term>
<listitem>
<para>
Repeat a shell command with 1-second intervals until it
succeeds.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_until_fails</literal>
</term>
<listitem>
<para>
Repeat a shell command with 1-second intervals until it
fails.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_unit</literal>
</term>
<listitem>
<para>
Wait until the specified systemd unit has reached the
<quote>active</quote> state.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_file</literal>
</term>
<listitem>
<para>
Wait until the specified file exists.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_open_port</literal>
</term>
<listitem>
<para>
Wait until a process is listening on the given TCP port (on
<literal>localhost</literal>, at least).
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_closed_port</literal>
</term>
<listitem>
<para>
Wait until nobody is listening on the given TCP port.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_x</literal>
</term>
<listitem>
<para>
Wait until the X11 server is accepting connections.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_text</literal>
</term>
<listitem>
<para>
Wait until the supplied regular expressions matches the
textual contents of the screen by using optical character
recognition (see <literal>get_screen_text</literal> and
<literal>get_screen_text_variants</literal>).
</para>
<note>
<para>
This requires passing <literal>enableOCR</literal> to the
test attribute set.
</para>
</note>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_console_text</literal>
</term>
<listitem>
<para>
Wait until the supplied regular expressions match a line of
the serial console output. This method is useful when OCR is
not possibile or accurate enough.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>wait_for_window</literal>
</term>
<listitem>
<para>
Wait until an X11 window has appeared whose name matches the
given regular expression, e.g.,
<literal>wait_for_window(&quot;Terminal&quot;)</literal>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>copy_from_host</literal>
</term>
<listitem>
<para>
Copies a file from host to machine, e.g.,
<literal>copy_from_host(&quot;myfile&quot;, &quot;/etc/my/important/file&quot;)</literal>.
</para>
<para>
The first argument is the file on the host. The file needs
to be accessible while building the nix derivation. The
second argument is the location of the file on the machine.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>systemctl</literal>
</term>
<listitem>
<para>
Runs <literal>systemctl</literal> commands with optional
support for <literal>systemctl --user</literal>
</para>
<programlisting language="python">
machine.systemctl(&quot;list-jobs --no-pager&quot;) # runs `systemctl list-jobs --no-pager`
machine.systemctl(&quot;list-jobs --no-pager&quot;, &quot;any-user&quot;) # spawns a shell for `any-user` and runs `systemctl --user list-jobs --no-pager`
</programlisting>
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>shell_interact</literal>
</term>
<listitem>
<para>
Allows you to directly interact with the guest shell. This
should only be used during test development, not in production
tests. Killing the interactive session with
<literal>Ctrl-d</literal> or <literal>Ctrl-c</literal> also
ends the guest session.
</para>
</listitem>
</varlistentry>
</variablelist>
<para>
To test user units declared by
<literal>systemd.user.services</literal> the optional
<literal>user</literal> argument can be used:
</para>
<programlisting language="python">
</listitem>
</varlistentry>
<varlistentry>
<term>
<literal>shell_interact</literal>
</term>
<listitem>
<para>
Allows you to directly interact with the guest shell. This
should only be used during test development, not in
production tests. Killing the interactive session with
<literal>Ctrl-d</literal> or <literal>Ctrl-c</literal> also
ends the guest session.
</para>
</listitem>
</varlistentry>
</variablelist>
<para>
To test user units declared by
<literal>systemd.user.services</literal> the optional
<literal>user</literal> argument can be used:
</para>
<programlisting language="python">
machine.start()
machine.wait_for_x()
machine.wait_for_unit(&quot;xautolock.service&quot;, &quot;x-session-user&quot;)
</programlisting>
<para>
This applies to <literal>systemctl</literal>,
<literal>get_unit_info</literal>, <literal>wait_for_unit</literal>,
<literal>start_job</literal> and <literal>stop_job</literal>.
</para>
<para>
For faster dev cycles it's also possible to disable the code-linters
(this shouldn't be commited though):
</para>
<programlisting language="bash">
<para>
This applies to <literal>systemctl</literal>,
<literal>get_unit_info</literal>,
<literal>wait_for_unit</literal>, <literal>start_job</literal> and
<literal>stop_job</literal>.
</para>
<para>
For faster dev cycles it's also possible to disable the
code-linters (this shouldn't be commited though):
</para>
<programlisting language="bash">
import ./make-test-python.nix {
skipLint = true;
machine =
@ -531,13 +537,13 @@ import ./make-test-python.nix {
'';
}
</programlisting>
<para>
This will produce a Nix warning at evaluation time. To fully disable
the linter, wrap the test script in comment directives to disable
the Black linter directly (again, don't commit this within the
Nixpkgs repository):
</para>
<programlisting language="bash">
<para>
This will produce a Nix warning at evaluation time. To fully
disable the linter, wrap the test script in comment directives to
disable the Black linter directly (again, don't commit this within
the Nixpkgs repository):
</para>
<programlisting language="bash">
testScript =
''
# fmt: off
@ -545,4 +551,66 @@ import ./make-test-python.nix {
# fmt: on
'';
</programlisting>
</section>
<section xml:id="ssec-failing-tests-early">
<title>Failing tests early</title>
<para>
To fail tests early when certain invariables are no longer met
(instead of waiting for the build to time out), the decorator
<literal>polling_condition</literal> is provided. For example, if
we are testing a program <literal>foo</literal> that should not
quit after being started, we might write the following:
</para>
<programlisting language="python">
@polling_condition
def foo_running():
machine.succeed(&quot;pgrep -x foo&quot;)
machine.succeed(&quot;foo --start&quot;)
machine.wait_until_succeeds(&quot;pgrep -x foo&quot;)
with foo_running:
... # Put `foo` through its paces
</programlisting>
<para>
<literal>polling_condition</literal> takes the following
(optional) arguments:
</para>
<para>
<literal>seconds_interval</literal>
</para>
<para>
: specifies how often the condition should be polled:
</para>
<programlisting>
```py
@polling_condition(seconds_interval=10)
def foo_running():
machine.succeed(&quot;pgrep -x foo&quot;)
```
</programlisting>
<para>
<literal>description</literal>
</para>
<para>
: is used in the log when the condition is checked. If this is not
provided, the description is pulled from the docstring of the
function. These two are therefore equivalent:
</para>
<programlisting>
```py
@polling_condition
def foo_running():
&quot;check that foo is running&quot;
machine.succeed(&quot;pgrep -x foo&quot;)
```
```py
@polling_condition(description=&quot;check that foo is running&quot;)
def foo_running():
machine.succeed(&quot;pgrep -x foo&quot;)
```
</programlisting>
</section>
</section>

4
nixos/lib/test-driver/default.nix
View file

@ -14,7 +14,7 @@
python3Packages.buildPythonApplication rec {
pname = "nixos-test-driver";
version = "1.0";
version = "1.1";
src = ./.;
propagatedBuildInputs = [ coreutils netpbm python3Packages.colorama python3Packages.ptpython qemu_pkg socat vde2 ]
@ -26,7 +26,7 @@ python3Packages.buildPythonApplication rec {
mypy --disallow-untyped-defs \
--no-implicit-optional \
--ignore-missing-imports ${src}/test_driver
pylint --errors-only ${src}/test_driver
pylint --errors-only --enable=unused-import ${src}/test_driver
black --check --diff ${src}/test_driver
'';
}

2
nixos/lib/test-driver/setup.py
View file

@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name="nixos-test-driver",
version='1.0',
version='1.1',
packages=find_packages(),
entry_points={
"console_scripts": [

41
nixos/lib/test-driver/test_driver/driver.py
View file

@ -1,12 +1,13 @@
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Dict, Iterator, List
from typing import Any, Dict, Iterator, List, Union, Optional, Callable, ContextManager
import os
import tempfile
from test_driver.logger import rootlog
from test_driver.machine import Machine, NixStartScript, retry
from test_driver.vlan import VLan
from test_driver.polling_condition import PollingCondition
class Driver:
@ -16,6 +17,7 @@ class Driver:
tests: str
vlans: List[VLan]
machines: List[Machine]
polling_conditions: List[PollingCondition]
def __init__(
self,
@ -36,12 +38,15 @@ class Driver:
for s in scripts:
yield NixStartScript(s)
self.polling_conditions = []
self.machines = [
Machine(
start_command=cmd,
keep_vm_state=keep_vm_state,
name=cmd.machine_name,
tmp_dir=tmp_dir,
callbacks=[self.check_polling_conditions],
)
for cmd in cmd(start_scripts)
]
@ -84,6 +89,7 @@ class Driver:
retry=retry,
serial_stdout_off=self.serial_stdout_off,
serial_stdout_on=self.serial_stdout_on,
polling_condition=self.polling_condition,
Machine=Machine, # for typing
)
machine_symbols = {m.name: m for m in self.machines}
@ -159,3 +165,36 @@ class Driver:
def serial_stdout_off(self) -> None:
rootlog._print_serial_logs = False
def check_polling_conditions(self) -> None:
for condition in self.polling_conditions:
condition.maybe_raise()
def polling_condition(
self,
fun_: Optional[Callable] = None,
*,
seconds_interval: float = 2.0,
description: Optional[str] = None,
) -> Union[Callable[[Callable], ContextManager], ContextManager]:
driver = self
class Poll:
def __init__(self, fun: Callable):
self.condition = PollingCondition(
fun,
seconds_interval,
description,
)
def __enter__(self) -> None:
driver.polling_conditions.append(self.condition)
def __exit__(self, a, b, c) -> None: # type: ignore
res = driver.polling_conditions.pop()
assert res is self.condition
if fun_ is None:
return Poll
else:
return Poll(fun_)

9
nixos/lib/test-driver/test_driver/machine.py
View file

@ -318,6 +318,7 @@ class Machine:
# Store last serial console lines for use
# of wait_for_console_text
last_lines: Queue = Queue()
callbacks: List[Callable]
def __repr__(self) -> str:
return f"<Machine '{self.name}'>"
@ -329,12 +330,14 @@ class Machine:
name: str = "machine",
keep_vm_state: bool = False,
allow_reboot: bool = False,
callbacks: Optional[List[Callable]] = None,
) -> None:
self.tmp_dir = tmp_dir
self.keep_vm_state = keep_vm_state
self.allow_reboot = allow_reboot
self.name = name
self.start_command = start_command
self.callbacks = callbacks if callbacks is not None else []
# set up directories
self.shared_dir = self.tmp_dir / "shared-xchg"
@ -406,6 +409,7 @@ class Machine:
return answer
def send_monitor_command(self, command: str) -> str:
self.run_callbacks()
with self.nested("sending monitor command: {}".format(command)):
message = ("{}\n".format(command)).encode()
assert self.monitor is not None
@ -509,6 +513,7 @@ class Machine:
def execute(
self, command: str, check_return: bool = True, timeout: Optional[int] = 900
) -> Tuple[int, str]:
self.run_callbacks()
self.connect()
if timeout is not None:
@ -969,3 +974,7 @@ class Machine:
self.shell.close()
self.monitor.close()
self.serial_thread.join()
def run_callbacks(self) -> None:
for callback in self.callbacks:
callback()

77
nixos/lib/test-driver/test_driver/polling_condition.py Normal file
View file

@ -0,0 +1,77 @@
from typing import Callable, Optional
import time
from .logger import rootlog
class PollingConditionFailed(Exception):
pass
class PollingCondition:
condition: Callable[[], bool]
seconds_interval: float
description: Optional[str]
last_called: float
entered: bool
def __init__(
self,
condition: Callable[[], Optional[bool]],
seconds_interval: float = 2.0,
description: Optional[str] = None,
):
self.condition = condition # type: ignore
self.seconds_interval = seconds_interval
if description is None:
if condition.__doc__:
self.description = condition.__doc__
else:
self.description = condition.__name__
else:
self.description = str(description)
self.last_called = float("-inf")
self.entered = False
def check(self) -> bool:
if self.entered or not self.overdue:
return True
with self, rootlog.nested(self.nested_message):
rootlog.info(f"Time since last: {time.monotonic() - self.last_called:.2f}s")
try:
res = self.condition() # type: ignore
except Exception:
res = False
res = res is None or res
rootlog.info(self.status_message(res))
return res
def maybe_raise(self) -> None:
if not self.check():
raise PollingConditionFailed(self.status_message(False))
def status_message(self, status: bool) -> str:
return f"Polling condition {'succeeded' if status else 'failed'}: {self.description}"
@property
def nested_message(self) -> str:
nested_message = ["Checking polling condition"]
if self.description is not None:
nested_message.append(repr(self.description))
return " ".join(nested_message)
@property
def overdue(self) -> bool:
return self.last_called + self.seconds_interval < time.monotonic()
def __enter__(self) -> None:
self.entered = True
def __exit__(self, exc_type, exc_value, traceback) -> None: # type: ignore
self.entered = False
self.last_called = time.monotonic()

50
nixos/tests/vscodium.nix
View file

@ -34,36 +34,46 @@ let
};
enableOCR = true;
testScript = ''
@polling_condition
def codium_running():
machine.succeed('pgrep -x codium')
start_all()
machine.wait_for_unit('graphical.target')
machine.wait_until_succeeds('pgrep -x codium')
# Wait until vscodium is visible. "File" is in the menu bar.
machine.wait_for_text('File')
machine.screenshot('start_screen')
with codium_running:
# Wait until vscodium is visible. "File" is in the menu bar.
machine.wait_for_text('Get Started')
machine.screenshot('start_screen')
test_string = 'testfile'
test_string = 'testfile'
# Create a new file
machine.send_key('ctrl-n')
machine.wait_for_text('Untitled')
machine.screenshot('empty_editor')
# Create a new file
machine.send_key('ctrl-n')
machine.wait_for_text('Untitled')
machine.screenshot('empty_editor')
# Type a string
machine.send_chars(test_string)
machine.wait_for_text(test_string)
machine.screenshot('editor')
# Type a string
machine.send_chars(test_string)
machine.wait_for_text(test_string)
machine.screenshot('editor')
# Save the file
machine.send_key('ctrl-s')
machine.wait_for_text('Save')
machine.screenshot('save_window')
machine.send_key('ret')
# Save the file
machine.send_key('ctrl-s')
machine.wait_for_text('Save')
machine.screenshot('save_window')
machine.send_key('ret')
# (the default filename is the first line of the file)
machine.wait_for_file(f'/home/alice/{test_string}')
# (the default filename is the first line of the file)
machine.wait_for_file(f'/home/alice/{test_string}')
machine.send_key('ctrl-q')
machine.wait_until_fails('pgrep -x codium')
'';
});
in builtins.mapAttrs (k: v: mkTest k v { }) tests
in
builtins.mapAttrs (k: v: mkTest k v { }) tests