diff --git a/bin/run.rb b/bin/run.rb index b731a46..96caae1 100755 --- a/bin/run.rb +++ b/bin/run.rb @@ -1,208 +1,211 @@ #!/usr/bin/env ruby # frozen_string_literal: true # # Copyright (C) 2017 Harald Sitter # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) version 3, or any # later version accepted by the membership of KDE e.V. (or its # successor approved by the membership of KDE e.V.), which shall # act as a proxy defined in Section 6 of version 3 of the license. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . require 'etc' require 'fileutils' require 'json' require_relative '../lib/junit' require_relative '../lib/paths' ISOTOVIDEO = if File.exist?('/opt/os-autoinst/isotovideo') && !ENV['OPENQA_OS_AUTOINST_IN_TREE'] '/opt/os-autoinst/isotovideo' else File.expand_path('os-autoinst/isotovideo') end # FIXME: we really want 20G to not accidently risk out of disking the server # if a test has a data leak. OTOH we need larger setups for some tests. # might be worth investing into a solution that can dynamically upscale a # 20G base image (would need larger overlay + resizing the partition table) DISK_SIZE_GB = '30'.freeze ENV['PERL5LIB'] = PERL5LIB # Default to xenial unless otherwise specified. ENV['OPENQA_SERIES'] = 'xenial' unless ENV['OPENQA_SERIES'] puts 'kvm-ok?' system 'kvm-ok' system 'ls -lah /dev/kvm' # os-autoinst internally hosts a mojo server to shove assets between host and # guest, this controls the debuggyness there. # MOJO_LOG_LEVEL=debug # not a typo 鑊! # FIXME: hack while we run everything in the same job we need to only clean the # wok on the initial installation test. otherwise we lose data. if ENV['INSTALLATION'] FileUtils.rm_r('wok') if File.exist?('wok') end Dir.mkdir('wok') unless File.exist?('wok') Dir.chdir('wok') FileUtils.rm_rf('../metadata', verbose: true) FileUtils.mkdir('../metadata', verbose: true) # Cloud scaled node, use all cores, else only half of them to not impair # other functionality on the node. cpus = Etc.nprocessors cpus = (cpus / 2.0).ceil unless File.exist?('/tooling/is_scaling_node') defaultvga = 'qxl' config = { BACKEND: 'qemu', CDMODEL: 'virtio-scsi-pci', DESKTOP: 'kde', DISTRI: 'debian', PRJDIR: '/workspace', CASEDIR: '/workspace/neon', PRODUCTDIR: '/workspace/neon', # cirrus: old std, doesn't do wayland # qxl: used for spice as well. as special guest driver. works with wayland. # doesn't clear/redraw screen on VT switch properly, # causing rendering artifacts prevent screen matches # std: new standard. has 800x600 resolution for some reason # virtio/virgil: broke uefi display init somehow. not actually built with # 3d accel on debian/ubuntu. needs passing of options to actually enable # accel -display sdl,gl=on` # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=813658 QEMUVGA: ENV.fetch('QEMUVGA', defaultvga), TESTDEBUG: false, MAKETESTSNAPSHOTS: false, QEMUCPUS: cpus, QEMURAM: 2048, HDDSIZEGB_1: DISK_SIZE_GB, # G is appended by os-autoinst UEFI_BIOS: '/usr/share/OVMF/OVMF_CODE.fd', UEFI: 1, QEMU_COMPRESS_QCOW2: true, TYPE: ENV.fetch('TYPE') } ENV.each { |k, v| config[k.to_sym] = v if k.start_with?('OPENQA_') } # The 16.04 ovmf doesn't work with virtio/virgil3d VGA and fails to init the # display. Use a binary copy of the bionic build # ovmf_0~20171205.a9212288-1_all.deb # The secondary "OVMF-pure-efi" is from kraxel.org # edk2.git-ovmf-x64-0-20180807.244.gde005223b7.noarch.rpm # There's also fancy builds at https://www.kraxel.org/repos/jenkins/edk2/ # which contain more pertinent stuff. bionic_ovmf = File.expand_path("#{__dir__}/../OVMF/OVMF_CODE.fd") config[:UEFI_BIOS] = bionic_ovmf if File.exist?(bionic_ovmf) # Switch to bios mode when requested. config.delete(:UEFI) if ENV['OPENQA_BIOS'] config[:TESTS_TO_RUN] = ENV['TESTS_TO_RUN'] config[:PLASMA_DESKTOP] = ENV['PLASMA_DESKTOP'] if ENV['INSTALLATION'] config[:INSTALLATION] = ENV['INSTALLATION'] config[:INSTALLATION_OEM] = ENV['INSTALLATION_OEM'] config[:ISO] = '/workspace/neon.iso' - # explicitly boot from ISO. on reboots we'll expect the ISO to have been - # ejected. - config[:BOOTFROM] = 'cdrom' + if ENV['OPENQA_PARTITIONING'] + # explicitly boot from ISO. for ubiquity we need to reboot, ordinarily + # qemu would then boot from the HDD if it has an ESP, we never want to + # boot from HDD in partitioning tests though. + config[:BOOTFROM] = 'cdrom' + end if ENV['OPENQA_SECUREBOOT'] # https://fedoraproject.org/wiki/Using_UEFI_with_QEMU#Testing_Secureboot_in_a_VM # https://rpmfind.net/linux/rpm2html/search.php?query=edk2-ovmf secureboot = File.expand_path("#{__dir__}/../OVMF/SecureBoot.iso") config[:ISO_1] = secureboot config[:SECUREBOOT] = true end else config[:BOOT_HDD_IMAGE] = true config[:KEEPHDDS] = true # Re-use existing raid/, comes from install test. os_auto_inst_dir = File.join('/srv/os-autoinst/', ENV.fetch('OPENQA_SERIES'), ENV.fetch('TYPE')) os_auto_inst_raid = "#{os_auto_inst_dir}/wok/raid" if File.exist?(os_auto_inst_raid) # Do not explode on recylced build dirs which might still have the origin # symlink linger. FileUtils.rm_f('../raid') FileUtils.ln_s(os_auto_inst_raid, '../raid') # Copy base image metadata if File.exist?("#{os_auto_inst_dir}/metadata/") FileUtils.cp_r("#{os_auto_inst_dir}/metadata/.", '../metadata/', verbose: true) end end # This is separate from the os-autinst recycling as you can manually simulate # it by simplying moving a suitable raid in place. This is for localhost # usage. On CI systems we alway should hit the os-autoinst path and symlink # the raid. existing_raid = File.realpath('../raid') if File.exist?(existing_raid) warn "Overlaying existing #{existing_raid}" FileUtils.rm_r('raid') if File.exist?('raid') FileUtils.mkpath('raid') unless system("qemu-img create -f qcow2 -o backing_file=#{existing_raid}/1 raid/1 #{DISK_SIZE_GB}G") raise "Failed to create overlay for #{existing_raid}" end system('qemu-img info raid/1') end config[:QEMU_DISABLE_SNAPSHOTS] = true config[:MAKETESTSNAPSHOTS] = false end # Set our wrapper as qemu. It transparently injects hugepages options # to the call when possible, to increase performance. And other stuff. ENV['QEMU'] = File.join(__dir__, 'kvm_arg_injector') if ENV['PLASMA_MOBILE'] config[:ISO] = '/workspace/neon-pm.iso' config[:BOOT_HDD_IMAGE] = false config[:KEEPHDDS] = false end warn "Going to use #{cpus} Cores" warn "Going to use KVM: #{!config.include?(:QEMU_NO_KVM)}" warn "Running from #{ISOTOVIDEO}" File.write('vars.json', JSON.generate(config)) File.write('live_log', '') system({ 'QEMU_AUDIO_DRV' => 'none' }, ISOTOVIDEO, '-d') || raise Dir.chdir('..') Dir.glob('wok/ulogs/metadata-*') do |file| target = File.basename(file) target = target.split('-', 2)[-1] FileUtils.mv(file, File.join('metadata', target), verbose: true) end # Generate a slideshow and ignore return value. If this fails chances are junit # will too, if junit doesn't fail we'd not care that slideshow failed. This is # more of a bonus feature. system("#{__dir__}/slideshow.rb", 'wok/slide.html') JUnit.from_openqa('wok/testresults') diff --git a/neon/main.pm b/neon/main.pm index 21c4060..1026bd2 100644 --- a/neon/main.pm +++ b/neon/main.pm @@ -1,174 +1,179 @@ # Copyright (C) 2016-2018 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . use warnings; use testapi; use autotest; use File::Basename; use List::Util qw[min]; BEGIN { unshift @INC, dirname(__FILE__) . '/../../lib'; } $testapi::username = 'user'; $testapi::password = 'password'; testapi::set_var('OEM_USERNAME', 'oem'); testapi::set_var('OEM_PASSWORD', 'oem'); # Special var to check if run in the cloud. This enables tests to only run # certain set up bits when run in the cloud rather than a local docker # container. if (!testapi::get_var("OPENQA_IN_CLOUD")) { testapi::set_var('OPENQA_IN_CLOUD', defined $ENV{'NODE_NAME'}); } my $dist = testapi::get_var("CASEDIR") . '/lib/distribution_neon.pm'; require $dist; testapi::set_distribution(distribution_neon->new()); sub unregister_needle_tags { my ($tag) = @_; my @a = @{needle::tags($tag)}; for my $n (@a) { $n->unregister($tag); } } sub cleanup_needles { if (!testapi::get_var('SECUREBOOT')) { unregister_needle_tags('ENV-SECUREBOOT'); } else { unregister_needle_tags('ENV-NO-SECUREBOOT'); } if (testapi::get_var('UEFI')) { unregister_needle_tags('ENV-BIOS'); } else { # BIOS mode unregister_needle_tags('ENV-UEFI'); } unless (testapi::get_var('OPENQA_INSTALLATION_OFFLINE')) { unregister_needle_tags('ENV-OFFLINE'); } unless (testapi::get_var('OPENQA_INSTALLATION_NONENGLISH')) { unregister_needle_tags('ENV-NONENGLISH'); } # Drop needles tagged with a different TYPE. # This is a bit inflexible right now but the best to be done at short # notice. my $good_tag = sprintf('ENV-TYPE-%s', testapi::get_var('TYPE')); for my $tag (keys %needle::tags) { if ($tag !~ /ENV-TYPE-/) { next; } if ($tag eq $good_tag) { next; } # We've found a disqualified tag. Drop all needles that have it. # UNLESS that needle has a qualifier tag (i.e. ENV-TYPE-$TYPE). # qualification > disqualification my @needles = @{needle::tags($tag)}; for my $needle (@needles) { if ($needle->has_tag($good_tag)) { next; } $needle->unregister($tag); } } # FIXME: workaround for bionic # to get bionic tests quickly off the ground we lower all match limits to # 70% . this should give reasonable leeway with most font differences. # additionally TTY is also bugging around and sometimes using wrong colors # we'll want to gradually increase this to 100% and sort out failures # as they pop up. if (testapi::get_var('OPENQA_SERIES') eq 'bionic') { # needle::needles is so overlapping names are lost. # Go through tags instead for my $needles (values %needle::tags) { for my $needle (@{$needles}) { my @areas = $needle->{area}; for my $area (@{$needle->{area}}) { $area->{match} //= 95; # os-autoinst doesn't default this. $area->{match} = min($area->{match}, 70); } } } } # TODO: implement exclusion of newer needles on older systems # Now that we dropped all unsuitable needles. We should restirct the match. # For all needles with our good tag we'll drop all needles that have the # other tags but not our good tag # e.g. n1 [ENV-TYPE-stable, dolphin] # n2 [dolphin] # -> we unregister n2 as it is less suitable than n1 } $needle::cleanuphandler = \&cleanup_needles; if (testapi::get_var("INSTALLATION") && testapi::get_var('OPENQA_PARTITIONING')) { if (testapi::get_var("TYPE")} eq 'devedition-gitunstable') { autotest::loadtest('tests/install/calamares_partitioning.pm'); } else { - die 'cannot partition !unstable'; + # For ubiquity the partitioning test is the regular install test but + # it has built-in conditionals for OPENQA_PARTITIONING. This is because + # with ubiquity we need to run complete installs to conduct our tests, + # so a separate test would be a huge copy pasta. Cala OTOH has + # a super simplified much faster test which easily stands up on its own. + autotest::loadtest('tests/install_ubiquity.pm'); } } elsif (testapi::get_var("INSTALLATION")) { my %test = ( 'devedition-gitunstable' => "tests/install_calamares.pm", '' => "tests/install_ubiquity.pm" ); if (testapi::get_var("INSTALLATION_OEM")) { autotest::loadtest('tests/install/ubiquity_oem.pm'); } else { autotest::loadtest($test{testapi::get_var("TYPE")} || $test{''}); } autotest::loadtest('tests/install/first_start.pm'); } elsif (testapi::get_var('OPENQA_SNAP_NAME')) { print("Running a snap test...\n"); my $snap_name = testapi::get_var('OPENQA_SNAP_NAME'); my $script = "tests/snap/$snap_name.pm"; if (-f join('/', testapi::get_var('CASEDIR'), $script)) { print("Found specific test for snap $script\n"); autotest::loadtest($script); } else { print("Using generic test for snap $snap_name\n"); autotest::loadtest("tests/snap/generic.pm"); } } elsif (testapi::get_var("TESTS_TO_RUN")) { my @testpaths = split /:/, testapi::get_var("TESTS_TO_RUN"); for my $testpath (@testpaths) { autotest::loadtest $testpath; } } elsif (testapi::get_var("PLASMA_DESKTOP")) { autotest::loadtest('tests/plasma/plasma_folder.pm'); autotest::loadtest('tests/plasma/plasma_favorite.pm'); autotest::loadtest('tests/plasma/plasma_alternatives.pm'); # Do not run tests after the lockscreen. We log into a second session # to get away from a switch-user sddm (which has no other means to do so # via the GUI). This may have any number of unfortunate side effects in the # main session (e.g. cache corruption?). autotest::loadtest('tests/plasma/plasma_lockscreen.pm'); } else { testapi::diag 'ERROR FAILURE BAD ERROR no clue what to run!'; exit 1; } 1; diff --git a/neon/tests/install_ubiquity.pm b/neon/tests/install_ubiquity.pm index c9e82e4..8bf4001 100644 --- a/neon/tests/install_ubiquity.pm +++ b/neon/tests/install_ubiquity.pm @@ -1,200 +1,202 @@ # Copyright (C) 2016-2017 Harald Sitter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of # the License or (at your option) version 3 or any later version # accepted by the membership of KDE e.V. (or its successor approved # by the membership of KDE e.V.), which shall act as a proxy # defined in Section 14 of version 3 of the license. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . use base "livetest_neon"; use strict; use testapi; my $user = $testapi::username; my $password = $testapi::password; sub assert_keyboard_page { assert_screen 'installer-keyboard', 16; # On bionic the keyboard is by default !english when installing !english, # this is a problem because we need english keyboard maps to do input # via openqa. So, once we've asserted the default keyboard page, change it # to use english instead. if (match_has_tag('installer-keyboard-espanol') && get_var('OPENQA_SERIES') ne 'xenial') { # Open the combobox assert_and_click 'installer-keyboard'; # Jump close to english (ingles). type_string 'in'; # At the time of writing there is a # crasher when selecting certain english variants, so we can't just # type this out. We'll move arrow down from 'in' until we have found # en_US. my $counter = 20; while (!check_screen('installer-keyboard-select-en-us', 1)) { if (!$counter--) { last; } send_key 'down'; sleep 1; } assert_and_click 'installer-keyboard-select-en-us'; } # Make sure we've now ended up with the standard en-us keyboard setup. assert_screen 'installer-keyboard-en-us', 2; assert_and_click 'installer-next'; } # Prepares live session for install. This expects a newly booted system. sub prepare { my ($self) = shift; select_console 'log-console'; { assert_script_run 'wget ' . data_url('geoip_service.rb'), 16; script_sudo 'systemd-run ruby `pwd`/geoip_service.rb', 16; } select_console 'x11'; $self->maybe_switch_offline; } # Runs an install. # @param disk_empty whether the disk is empty (when not empty it will be wiped) sub install { my ($self, %args) = @_; $args{disk_empty} //= 1; # Installer assert_and_click "installer-icon"; assert_screen "installer-welcome", 60; if (get_var('OPENQA_INSTALLATION_NONENGLISH')) { assert_and_click 'installer-welcome-click'; send_key 'down'; send_key 'ret'; assert_screen 'installer-welcome-espanol'; } assert_and_click "installer-next"; # bionic version of ubiquity moved the keyboard configuration as first step if (testapi::get_var('OPENQA_SERIES') ne 'xenial') { assert_keyboard_page; } assert_screen "installer-prepare", 16; assert_and_click "installer-next"; if ($args{disk_empty}) { assert_screen "installer-disk", 16; assert_and_click "installer-install-now"; } else { assert_and_click "installer-disk-wipe"; assert_screen "installer-disk-wipe-selected", 16; assert_and_click "installer-install-now"; } assert_and_click "installer-disk-confirm", 'left', 16; # Timezone has 75% fuzzyness as timezone is geoip'd so its fairly divergent. # Also, starting here only the top section of the window gets matched as # the bottom part with the buttons now has a progressbar and status # text which is non-deterministic. # NB: we give way more leeway on the new needle appearing as disk IO can # cause quite a bit of slowdown and ubiquity's transition policy is # fairly weird when moving away from the disk page. assert_screen "installer-timezone", 60; assert_and_click "installer-next"; # bionic version of ubiquity moved the keyboard configuration as first step # while in xenial version the keyboard config is after timezone setup if (testapi::get_var('OPENQA_SERIES') eq 'xenial') { assert_keyboard_page; } assert_screen "installer-user", 16; type_string $user; # user in user field, name field (needle doesn't include hostname in match) assert_screen "installer-user-user", 16; send_key "tab", 1; # username field send_key "tab", 1; # 1st password field type_string $password; send_key "tab", 1; # 2nd password field type_string $password; # all fields filled (not matching hostname field) assert_screen "installer-user-complete", 16; assert_and_click "installer-next"; assert_screen "installer-show", 15; # Let install finish assert_screen "installer-restart", 640; } sub run { my ($self) = shift; # Divert installation data to live data. $testapi::username = 'neon'; $testapi::password = ''; $self->boot; $self->prepare; $self->install; - # Reset the system and redo the entire installation to ensure partitioning - # works on pre-existing partition tables. This is broken in bionic as of - # the user edition ISO from 2018-09-15. + if (get_var('OPENQA_PARTITIONING')) { + # Reset the system and redo the entire installation to ensure partitioning + # works on pre-existing partition tables. This is broken in bionic as of + # the user edition ISO from 2018-09-15. - power 'reset'; - reset_consoles; + power 'reset'; + reset_consoles; - $self->boot; - $self->prepare; - $self->install(disk_empty => 0); + $self->boot; + $self->prepare; + $self->install(disk_empty => 0); + } select_console 'log-console'; { # Make sure networking is on (we disable it during installation). $self->online; $self->upload_ubiquity_logs; } select_console 'x11'; assert_and_click "installer-restart-now"; $self->live_reboot; # Set installation data. $testapi::username = $user; $testapi::password = $password; } sub upload_ubiquity_logs { # Uploads end up in wok/ulogs/ assert_script_sudo 'tar cfJ /tmp/installer.tar.xz /var/log/installer'; upload_logs '/tmp/installer.tar.xz', failok => 1; } sub post_fail_hook { my ($self) = shift; $self->SUPER::post_fail_hook; $self->upload_ubiquity_logs; } sub test_flags { # without anything - rollback to 'lastgood' snapshot if failed # 'fatal' - whole test suite is in danger if this fails # 'milestone' - after this test succeeds, update 'lastgood' # 'important' - if this fails, set the overall state to 'fail' return { important => 1, fatal => 1 }; } 1;