diff --git a/neon/lib/basetest_neon.pm b/neon/lib/basetest_neon.pm index 4260473..f312803 100644 --- a/neon/lib/basetest_neon.pm +++ b/neon/lib/basetest_neon.pm @@ -1,254 +1,254 @@ # Copyright (C) 2017-2018 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . package basetest_neon; use base 'basetest'; use JSON qw(decode_json); use strict; use testapi; sub new { my ($class, $args) = @_; my $self = $class->SUPER::new($args); # TODO: this maybe should be global as within a test series we still only # use the same VM and disk, so setup actually only needs to happen once # in an entire os-autoinst run. $self->{boot_setup_ran} = 0; return $self; } sub post_fail_hook { # Special handler for plymouth. # We continue to have problems with systems rebooting. According to the # journal systemd reached the shutdown state but the screen is stuck on # plymouth. There's a range of options of what is going wrong ranging from # kvm/qemu bug to systemd bug to openqa bug. Without additional data its # hard to say what's going on. To get that data we'll attempt to get data # out of the kernel. Should that turn out unresponsive as well we at least # know it's not a problem in the guest. # This ends in a system crash. We'll not be able to retrieve anything useful # anyway as the system is in a garbage state if plymouth gets stuck. if (check_screen('plymouth', 4)) { send_key 'alt-sysrq-r'; # raw kbd mode send_key 'alt-sysrq-9'; # log level send_key 'alt-sysrq-k'; # kill programs on VT (plymouth I hope) sleep 1; save_screenshot; send_key 'alt-sysrq-l'; # backtrace cpus sleep 1; save_screenshot; send_key 'alt-sysrq-m'; # memory info sleep 1; save_screenshot; send_key 'alt-sysrq-t'; # all task info sleep 1; save_screenshot; send_key 'alt-sysrq-w'; # list blocked tasks sleep 1; save_screenshot; send_key 'alt-sysrq-c'; # crash the system sleep 1; save_screenshot; } if (check_screen('drkonqi-notification', 4)) { assert_and_click('drkonqi-notification'); record_soft_failure 'not implemented drkonqi opening'; } if (check_screen('drkonqi-dialog', 4)) { assert_and_click('drkonqi-dialog'); # FIXME: these are hacks because we don't have a needle to assert # for how long the gdb tracing may take. sleep 12; send_key 'alt-f4'; # FIXME: like above but we don't know when coredumpd is done sleep 12; } select_console 'log-console'; # The next uploads are largely failok since we want to get as many logs # as possible evne if some are missing. upload_logs '/home/'.$testapi::username.'/.cache/xsession-errors', failok => 1; upload_logs '/home/'.$testapi::username.'/.cache/sddm/xsession-errors', failok => 1; upload_logs '/home/'.$testapi::username.'/.xsession-errors', failok => 1; upload_logs '/home/'.$testapi::username.'/.local/share/sddm/wayland-session.log', failok => 1; script_run 'journalctl --no-pager -b 0 > /tmp/journal.txt'; upload_logs '/tmp/journal.txt', failok => 1; script_run 'coredumpctl info > /tmp/dumps.txt'; upload_logs '/tmp/dumps.txt', failok => 1; return 1; } sub login { my ($self) = @_; # Short wait, we should be close to sddm if we this gets called. assert_screen 'sddm', 120; $self->maybe_login; } sub maybe_login { # Short wait, we should be close to sddm if we this gets called. if (check_screen 'sddm', 16) { type_password $testapi::password; send_key 'ret'; - wait_still_screen; + sleep 4; } } sub logout { testapi::x11_start_program('Logout'); assert_and_click ('ksmserver-logout'); } sub _upgrade { if (!get_var('OPENQA_APT_UPGRADE')) { return } if (get_var('OPENQA_INSTALLATION_OFFLINE')) { die 'You cannot upgrade on an offline test!!!' } assert_script_sudo 'apt update', 2 * 60; my $pkgs = get_var('OPENQA_APT_UPGRADE'); if ($pkgs eq "all") { $pkgs = "dist-upgrade"; } else { $pkgs = "install " . $pkgs; } assert_script_sudo 'DEBIAN_FRONTEND=noninteractive apt -y ' . $pkgs, 30 * 60; } sub boot_to_dm { my ($self, %args) = @_; $args{run_setup} //= 1; # Grub in user edition is broken as of Jan 2018 and doesn't match our needle # because it is shittyly themed. As we do not entirely care about this in # application tests we'll simply ignore it by checking for either grub or # sddm. Due to auto time out we'll eventually end up at sddm even if # we do not explicitly hit 'ret'. assert_screen [qw(grub sddm)], 60 * 3; if (match_has_tag('grub')) { send_key 'ret'; assert_screen 'sddm', 60 * 2; } # else sddm, nothing to do if ($args{run_setup} && !$self->{boot_setup_ran}) { select_console 'log-console'; { # Drop random file which wasn't meant to be there in xenial # TODO: this can e dropped after 2018-09-10 or so (once all disks # rotated in the fix) # https://packaging.neon.kde.org/neon/settings.git/commit/?h=Neon/unstable_xenial&id=dc28d791e5e4432174dca9a02eccecebccf024b0 if (get_var('OPENQA_SERIES') eq 'xenial') { script_sudo 'rm /etc/apt/preferences.d/99-neon-qca'; } assert_script_run 'wget ' . data_url('basetest_setup.rb'), 60; assert_script_sudo 'ruby basetest_setup.rb', 60; $self->_upgrade; } select_console 'x11'; $self->{boot_setup_ran} = 1; } # Move mouse to make sure sddm isn't idle before we return. # Make sure to hide it afterwards, lest it triggers hovers. mouse_set(1, 1); mouse_hide; } # Waits for system to boot and log in (no assertion about desktop). sub boot { my ($self, $args) = @_; $self->boot_to_dm; $self->login; } # Switches to log-console and reboots the system via 'reboot' sub reboot { select_console 'log-console'; script_sudo 'reboot', 0; reset_consoles; } sub enable_snapd { my ($self, %args) = @_; $args{auto_install_snap} //= 0; select_console 'log-console'; assert_script_sudo 'systemctl enable --now snapd.service'; # When turning on snapd it may schedule a whole bunch of stuff to update. # Wait until all automatic changes are done. To establish that talk to # the API and get in-progress changes. my @changes = (); do { sleep 8; my $changes_json = script_output 'curl --unix-socket /run/snapd.socket http:/v2/changes?select=in-progress'; @changes = @{decode_json($changes_json)->{result}}; } while (@changes); my $runtime = 'kde-frameworks-5'; my $snap = get_var('OPENQA_SNAP_NAME'); my $snap_channel = get_var('OPENQA_SNAP_CHANNEL'); my $runtime_channel = get_var('OPENQA_SNAP_RUNTIME_CHANNEL'); if ($runtime_channel eq "" || !defined $runtime_channel) { $runtime_channel = get_var('OPENQA_SNAP_CHANNEL'); } if ($runtime_channel eq "" || !defined $runtime_channel) { $runtime_channel = "stable"; } assert_script_sudo "snap switch --$runtime_channel $runtime"; assert_script_sudo 'snap refresh', 30 * 60; script_run "snap info $runtime > /tmp/runtime.info"; upload_logs '/tmp/runtime.info'; assert_script_run "curl --unix-socket /run/snapd.socket http:/v2/snaps/$runtime > /tmp/runtime.json"; upload_logs '/tmp/runtime.json'; if ($args{auto_install_snap}) { assert_script_sudo "snap install --$snap_channel $snap", 15 * 60; script_run "snap info $snap > /tmp/snap.info"; upload_logs '/tmp/snap.info'; assert_script_run "curl --unix-socket /run/snapd.socket http:/v2/snaps/$snap > /tmp/snap.json"; upload_logs '/tmp/snap.json'; } assert_script_run 'curl --unix-socket /run/snapd.socket http:/v2/interfaces > /tmp/interfaces.json'; upload_logs '/tmp/interfaces.json'; select_console 'x11'; } sub enable_snapd_and_install_snap { my ($self, $args) = @_; return $self->enable_snapd(auto_install_snap => 1); } 1; diff --git a/neon/lib/distribution_neon.pm b/neon/lib/distribution_neon.pm index 001cfe7..f9b0377 100644 --- a/neon/lib/distribution_neon.pm +++ b/neon/lib/distribution_neon.pm @@ -1,176 +1,176 @@ package distribution_neon; use base 'distribution'; use testapi qw(send_key %cmd assert_screen check_screen check_var get_var match_has_tag set_var type_password type_string wait_serial mouse_hide send_key_until_needlematch record_soft_failure wait_still_screen wait_screen_change diag); sub init() { my ($self) = @_; $self->SUPER::init(); $self->init_consoles(); $self->{console_sudo_cache} = %{()}; } sub x11_start_program($$$) { my ($self, $program, $timeout, $options) = @_; # enable valid option as default $options->{valid} //= 1; send_key "alt-f2"; mouse_hide(1); check_screen('desktop-runner', $timeout); type_string $program; - wait_still_screen; + sleep 2; send_key "ret"; } sub ensure_installed { my ($self, $pkgs, %args) = @_; my $pkglist = ref $pkgs eq 'ARRAY' ? join ' ', @$pkgs : $pkgs; $args{timeout} //= 90; testapi::x11_start_program('konsole'); assert_screen('konsole'); testapi::assert_script_sudo("chown $testapi::username /dev/$testapi::serialdev"); testapi::assert_script_sudo("chmod 666 /dev/$testapi::serialdev"); # make sure packagekit service is available testapi::assert_script_sudo('systemctl is-active -q packagekit || (systemctl unmask -q packagekit ; systemctl start -q packagekit)'); $self->script_run( "for i in {1..$retries} ; do pkcon -y install $pkglist && break ; done ; RET=\$?; echo \"\n pkcon finished\n\"; echo \"pkcon-\${RET}-\" > /dev/$testapi::serialdev", 0 ); if (check_screen('polkit-install', $args{timeout})) { type_password; send_key('ret', 1); } wait_serial('pkcon-0-', $args{timeout}) || die "pkcon failed"; send_key('alt-f4'); } # initialize the consoles needed during our tests sub init_consoles { my ($self) = @_; $self->add_console('root-virtio-terminal', 'virtio-terminal', {}); # NB: ubuntu only sets up tty1 to 7 by default. $self->add_console('log-console', 'tty-console', {tty => 6}); # in bionic ubuntu switched to tty1 for default. we adjusted our sddm # accordingly. $self->add_console('x11', 'tty-console', {tty => 1}); # oem-config runs on tty1, later it will drop into tty7 for the final # x11. $self->add_console('oem-config', 'tty-console', {tty => 1}); use Data::Dumper; print Dumper($self->{consoles}); return; } sub script_sudo($$) { # Clear the TTY first, otherwise we may needle match a previous sudo # password query and get confused. Clearing first make sure the TTY is empty # and we'll either get a new password query or none (still in cache). type_string "clear\n"; # NB: this is an adjusted code copy from os-autoinst distribution, we're # caching sudo results as by default sudo has a cache anyway. my ($self, $prog, $wait) = @_; # Iff the current console is a tty and it's been less than 4 minutes since # the last auth we don't expect an auth and skip the auth needle. # The time stamps are reset in activate_console which is only called after # consoles get reset and switched to again, so this should be fairly ok. # !tty never are cached. e.g. on x11 you could have multiple konsoles but # since the sudo cache is per-shell we don't know if there is a cache. use Scalar::Util 'blessed'; my $class = blessed($self->{consoles}{$testapi::selected_console}); my $is_tty = ($class =~ m/ttyConsole/); my $last_auth = $self->{console_sudo_cache}{$testapi::selected_console}; my $need_auth = (!$is_tty || !$last_auth || (time() - $last_auth >= 4 * 60)); # Debug for now. Can be removed when someone stumbles upon this again. print "sudo cache [tty: $is_tty, last_auth: $last_auth, need auth: $need_auth]:\n"; $wait //= 10; my $str; if ($wait > 0) { $str = testapi::hashed_string("SS$prog$wait"); $prog = "$prog; echo $str > /dev/$testapi::serialdev"; } testapi::type_string "sudo $prog\n"; if ($need_auth) { if (testapi::check_screen "sudo-passwordprompt", 2, no_wait => 1) { testapi::type_password; testapi::send_key "ret"; $self->{console_sudo_cache}{$testapi::selected_console} = time(); } } if ($str) { return testapi::wait_serial($str, $wait); } return; } sub activate_console { my ($self, $console) = @_; diag "activating $console"; $self->{console_sudo_cache}{$console} = 0; if ($console eq 'log-console') { assert_screen 'tty6-selected'; type_string $testapi::username; send_key 'ret'; assert_screen 'tty-password'; type_password $testapi::password; send_key 'ret'; assert_screen [qw(tty-logged-in tty-login-incorrect)]; if (match_has_tag('tty-login-incorrect')) { # Let's try again if the login failed. If it fails again give up. # It can happen that due to IO some of the password gets lost. # Not much to be done about that other than retry and hope for the # best. type_string $testapi::username; send_key 'ret'; assert_screen 'tty-password'; type_password $testapi::password; send_key 'ret'; assert_screen 'tty-logged-in'; } # Mostly just a workaround. os-autoinst wants to write to /dev/ttyS0 but # on ubuntu that doesn't fly unless chowned first. testapi::assert_script_sudo("chown $testapi::username /dev/$testapi::serialdev"); testapi::assert_script_sudo("chmod 666 /dev/$testapi::serialdev"); } return; } # Make sure consoles are ready and in a well known state. This prevents # switching between consoles quickly from ending up on a console which isn't # yet ready for use (e.g. typing on TTY before ready and losing chars). sub console_selected { my ($self, $console, %args) = @_; # FIXME: should make sure the session is unlocked if ($console eq 'x11') { # Do not wait on X11 specifically. Desktop state is wildely divergent. # Instead wait a static amount. This is a bit shit. But meh. # We could maybe needle the panel specifically? But then sddm has no # panel. I am really not sure how to best handle this. sleep 2; return; } assert_screen($console, no_wait => 1); } 1; diff --git a/neon/tests/plasma/plasma_alternatives.pm b/neon/tests/plasma/plasma_alternatives.pm index 1e0079a..0990b9e 100644 --- a/neon/tests/plasma/plasma_alternatives.pm +++ b/neon/tests/plasma/plasma_alternatives.pm @@ -1,93 +1,93 @@ # Copyright (C) 2018 Bhavisha Dhruve # Copyright (C) 2016-2018 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . use base "basetest_neon"; use strict; use testapi; sub switch_to { my ($type) = @_; do { # Close open popup if there is any. send_key 'esc'; send_key 'esc'; send_key 'esc'; # Starts the Alternative Menu assert_and_click 'plasma-launcher', button => 'right'; # Selects the menu type. assert_and_click 'kickoff-alternatives'; assert_screen "kickoff-alternatives-$type"; # Repeat this entire dance if the popup has corrupted graphics. # This happens every so often and renders the popup incorrectly. If # we were to click at this point we'd be selecting an off-by-one item. # instead of the intended one. # Cause unknown. } while (match_has_tag 'kickoff-alternatives-corrupted'); # Select type. assert_and_click "kickoff-alternatives-$type"; # Apply the switch. assert_and_click 'plasma-alternatives-switch'; } sub run { my ($self) = @_; assert_screen 'folder-desktop'; # Switch to menu (kicker) switch_to 'menu'; assert_screen 'folder-desktop'; # Check if kicker opens instead of kickoff assert_and_click 'plasma-launcher'; assert_screen 'plasma-kicker'; send_key 'esc'; # Starting a new session $self->logout; # Back in the session $self->login; assert_screen 'folder-desktop'; # Roll back to launcher (kickoff) switch_to 'launcher'; assert_screen 'folder-desktop'; assert_and_click 'plasma-launcher'; assert_screen 'kickoff-popup'; - wait_still_screen; + sleep 2; send_key 'esc'; assert_screen 'folder-desktop'; } sub test_flags { # without anything - rollback to 'lastgood' snapshot if failed # 'fatal' - whole test suite is in danger if this fails # 'milestone' - after this test succeeds, update 'lastgood' # 'important' - if this fails, set the overall state to 'fail' return { important => 1 }; } 1; diff --git a/neon/tests/plasma/plasma_favorite.pm b/neon/tests/plasma/plasma_favorite.pm index b90a932..a07ee2f 100644 --- a/neon/tests/plasma/plasma_favorite.pm +++ b/neon/tests/plasma/plasma_favorite.pm @@ -1,80 +1,80 @@ # Copyright (C) 2018 Bhavisha Dhruve # Copyright (C) 2016-2018 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . use base "basetest_neon"; use strict; use testapi; sub run { my ($self) = @_; assert_screen 'folder-desktop'; # Starts the Application Launcher assert_and_click 'plasma-launcher'; - wait_still_screen; + sleep(2); # Switches to the Application Tab assert_screen 'kickoff-favorite'; assert_and_click 'kickoff-application'; assert_and_click 'kickoff-office'; # Adds Okular in the favorites tab assert_and_click 'kickoff-okular', button => 'right'; assert_and_click 'kickoff-add-to-favorite'; assert_screen 'kickoff-favorite-okular', 60; send_key 'esc'; - wait_still_screen; + sleep(2); assert_and_click 'plasma-launcher'; send_key 'esc'; # Logging out from the session $self->logout; # Back in the session $self->login; assert_screen 'folder-desktop', 60; # Removes Okular from the favorites tab assert_and_click 'plasma-launcher'; - wait_still_screen; + sleep(2); # Move the mouse far far away in an attempt to not hit # https://bugs.kde.org/show_bug.cgi?id=407517 # which may also be a timing issue for us here as technically we shouldn't # be able to have the mouse already above the okular entry. mouse_set(0, 0); assert_and_click 'kickoff-favorite-okular', button => 'right'; assert_and_click 'kickoff-remove-from-favorite'; assert_screen ['kickoff-favorite-okular', 'kickoff-favorite'], 60; if (match_has_tag('kickoff-favorite-okular')) { die 'Okular should not be visible on the favorite tab' } # Close the kickoff otherwise next test will fail assert_and_click 'kickoff-dismiss'; } sub test_flags { # without anything - rollback to 'lastgood' snapshot if failed # 'fatal' - whole test suite is in danger if this fails # 'milestone' - after this test succeeds, update 'lastgood' # 'important' - if this fails, set the overall state to 'fail' return { important => 1 }; } 1; diff --git a/neon/tests/plasma/plasma_lockscreen.pm b/neon/tests/plasma/plasma_lockscreen.pm index 9d0d803..d21c1c0 100644 --- a/neon/tests/plasma/plasma_lockscreen.pm +++ b/neon/tests/plasma/plasma_lockscreen.pm @@ -1,138 +1,138 @@ # Copyright (C) 2016-2017 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . use base "basetest_neon"; use strict; use testapi; sub lock_screen { hold_key('ctrl'); hold_key('alt'); hold_key('l'); release_key('l'); release_key('alt'); release_key('ctrl'); assert_screen('plasma-locked'); } sub run { # NB: do not go into any consoles after this, they show up in the user # switch dialog! # Before we start the lock screen test make sure we aren't logged in on # our terminal. # Otherwise the tty6 session would show up in the switch and make results # unreliable. select_console 'log-console'; { script_run 'exit', 0; # Make sure logout actually happened. We have had cases where tty6 # magically appeared in the switch dialog, supposedly because exit failed. # NOTE: Unfortunately we cannot assert anything here. On 18.04+ we'd # automatically switch to the remaining active VT (x11), so the state of # vt6 is left a mystery. # Whether the sleep actually helps with anything is unknown. But one can # hope. sleep 4; reset_consoles; } select_console 'x11'; x11_start_program 'kcmshell5 screenlocker' ; assert_screen 'kcm-screenlocker'; assert_and_click 'kcm-screenlocker-appearance'; if (!check_screen('kcm-screenlocker-appearance-type-is-color', 4)) { # TODO: drop once all images have been rotated (~mid Sept 2018) record_soft_failure 'Testing an old disk image without static lockscreen'; assert_screen 'kcm-screenlocker-appearance-type'; assert_and_click 'kcm-screenlocker-appearance-type'; assert_and_click 'kcm-screenlocker-appearance-type-color'; } # Should the deafault ever become undesirable: #1d99f3 is the lovely color. assert_and_click 'kcm-ok'; lock_screen; # simple unlock type_password $testapi::password; send_key 'ret'; assert_screen 'folder-desktop'; lock_screen; assert_screen('plasma-locked-idle'); mouse_set(1, 1); mouse_hide; # NB: do not use esc. Esc is a toggle key. If it is not idle anymore esc # will make it idle again! send_key 'ctrl'; # make double sure it's unidled send_key 'shift'; # make triple sure it's unidled send_key 'f1'; # make quadruple sure it's unidled # virtual keyboard assert_and_click 'plasma-locked-keyboard-icon'; assert_screen 'plasma-locked-keyboard'; assert_and_click 'plasma-locked-keyboard-q'; assert_and_click 'plasma-locked-keyboard-q'; # qq in password field assert_screen 'plasma-locked-keyboard-qq', no_wait => 1; send_key 'backspace'; send_key 'backspace'; assert_and_click 'plasma-locked-keyboard-icon-active'; assert_screen 'plasma-locked'; # switch user assert_and_click 'plasma-locked-switch-icon'; assert_and_click 'plasma-locked-switch'; assert_screen 'sddm'; type_password $testapi::password; send_key 'ret'; # ugh, sddm has no way to get us back, start a new session? - wait_still_screen; + sleep 2; assert_and_click 'plasma-launcher', timeout => 60; # 60 seconds since we don't assert desktop assert_and_click 'kickoff-leave'; assert_and_click 'kickoff-leave-logout'; assert_and_click 'ksmserver-logout'; - wait_still_screen; + sleep 2; # we are back in our regular session, unlock and be happy # done unless (get_var("QEMUVGA") eq 'qxl') { # This is confirmed to not be a problem on qxl. Should we switch to # other VGAs in the future we'll know if they are exhibiting the # problem too as we only skip qxl. record_soft_failure 'kscreenlocker comes back with vkbd bug 387270'; assert_and_click 'plasma-locked-keyboard-icon-active'; } type_password $testapi::password; send_key 'ret'; assert_screen 'folder-desktop', 60; } sub test_flags { # without anything - rollback to 'lastgood' snapshot if failed # 'fatal' - whole test suite is in danger if this fails # 'milestone' - after this test succeeds, update 'lastgood' # 'important' - if this fails, set the overall state to 'fail' return { important => 1 }; } 1;