From dd02a2949792c24fe6108b725257c33079365cd8 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 18 Oct 2022 17:51:02 +0300 Subject: [PATCH] add integration/acceptance test with Subplot These need to be run with the virtual smartcard emulation in the Docker container specified in .gitlab-ci.yml for tests. The tests are a little simplistic, as it turned out that making changes to the smart card results in flaky tests. Thus only parts of opgpcard that don't change the card are tested. Sponsored-by: NLnet Foundation; NGI Assure --- tools/build.rs | 27 ++++++++ tools/cargo-test-in-docker | 23 +++++++ tools/src/bin/opgpcard/cli.rs | 5 +- tools/subplot/opgpcard.md | 114 ++++++++++++++++++++++++++++++++ tools/subplot/opgpcard.rs | 99 +++++++++++++++++++++++++++ tools/subplot/opgpcard.subplot | 15 +++++ tools/subplot/opgpcard.yaml | 12 ++++ tools/subplot/test-in-docker.sh | 29 ++++++++ tools/tests/opgpcard.rs | 6 ++ 9 files changed, 327 insertions(+), 3 deletions(-) create mode 100644 tools/build.rs create mode 100755 tools/cargo-test-in-docker create mode 100644 tools/subplot/opgpcard.md create mode 100644 tools/subplot/opgpcard.rs create mode 100644 tools/subplot/opgpcard.subplot create mode 100644 tools/subplot/opgpcard.yaml create mode 100755 tools/subplot/test-in-docker.sh create mode 100644 tools/tests/opgpcard.rs diff --git a/tools/build.rs b/tools/build.rs new file mode 100644 index 0000000..608f687 --- /dev/null +++ b/tools/build.rs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2022 Lars Wirzenius +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::fs::File; +use std::path::Path; + +fn main() { + // Only generate test code from the subplot, if a virtual smart + // card is available. This is a kludge until Subplot can do + // conditional scenarios + // (https://gitlab.com/subplot/subplot/-/issues/20). + let flagfile = Path::new("virtual-card-available"); + if flagfile.exists() { + subplot_build::codegen("subplot/opgpcard.subplot") + .expect("failed to generate code with Subplot"); + } else { + // If we're not generating code from the subplot, we should at + // least create an empty file so that the tests/opgpcard.rs + // file can include it. Otherwise the build will fail. + let out_dir = std::env::var("OUT_DIR").unwrap(); + let include = Path::new(&out_dir).join("opgpcard.rs"); + eprintln!("build.rs: include={}", include.display()); + if !include.exists() { + File::create(include).unwrap(); + } + } +} diff --git a/tools/cargo-test-in-docker b/tools/cargo-test-in-docker new file mode 100755 index 0000000..945294f --- /dev/null +++ b/tools/cargo-test-in-docker @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Run this to run opgpcard (tools directory) test suite inside a +# Docker container with a virtual smartcard running. The test suite +# contains tests that run the opgpcard binary and rely on a virtual +# smart card to be available. +# +# SPDX-FileCopyrightText: 2022 Lars Wirzenius +# SPDX-License-Identifier: MIT OR Apache-2.0 + +set -euo pipefail + +docker run --rm -it \ + -v root:/root \ + -v cargo:/cargo \ + -v $(pwd):/src \ + registry.gitlab.com/openpgp-card/virtual-cards/smartpgp-builddeps sh -c ' +sed -i "s/timeout=20/timeout=60/" /home/jcardsim/run-card.sh && +/etc/init.d/pcscd start && +su - -c "sh /home/jcardsim/run-card.sh >/dev/null" jcardsim && +cd /src/tools && +if ! [ -e virtual-card-available ]; then rm -f tests/opgpcard.rs; fi && +CARGO_TARGET_DIR=/cargo/ cargo test' diff --git a/tools/src/bin/opgpcard/cli.rs b/tools/src/bin/opgpcard/cli.rs index 0b1334e..cf594ea 100644 --- a/tools/src/bin/opgpcard/cli.rs +++ b/tools/src/bin/opgpcard/cli.rs @@ -6,9 +6,8 @@ use std::path::PathBuf; use crate::{OutputFormat, OutputVersion}; -pub const DEFAULT_OUTPUT_VERSION: &str = "1.0.0"; -pub const OUTPUT_VERSIONS: &[OutputVersion] = - &[OutputVersion::new(0, 0, 0), OutputVersion::new(1, 0, 0)]; +pub const DEFAULT_OUTPUT_VERSION: &str = "0.9.0"; +pub const OUTPUT_VERSIONS: &[OutputVersion] = &[OutputVersion::new(0, 9, 0)]; #[derive(Parser, Debug)] #[clap( diff --git a/tools/subplot/opgpcard.md b/tools/subplot/opgpcard.md new file mode 100644 index 0000000..9ed378b --- /dev/null +++ b/tools/subplot/opgpcard.md @@ -0,0 +1,114 @@ + + +# Introduction + +This document describes the requirements and acceptance criteria for +the `opgpcard` tool, and also how to verify that they are met. This +document is meant to be read and understood by all stakeholders, and +processed by the [Subplot](https://subplot.tech/) tool, which also +generates code to automatically perform the verification. + +## Note about running the tests described here + +The verification scenarios in this document assume the availability of +a virtual smart card. Specifically one described in +. The +`openpgp-card/tools` crate is set up to generate tests only if the +file `tools/virtual-card-available` exists, and the `openpgp-card` +repository `.gitlab-ci.yml` file is set up to create that file when +the repository is tested in GitLab CI. + +This means that if you run `cargo test`, no test code is normally +generated from this document. To run the tests locally, outside of +GitLab CI, use the script `tools/cargo-test-in-docker`. + +# Acceptance criteria + +These scenarios mainly test the JSON output format of the tool. That +format is meant for consumption by other tools, and it is thus more +important that it stays stable. The text output that is meant for +human consumption may change at will, so it's not worth testing. + +## Smoke test + +_Requirement: The tool can report its version._ + +Justification: This is useful mainly to make sure the tool can be run +at all. As such, it acts as a simple [smoke +test](https://en.wikipedia.org/wiki/Smoke_testing_(software)). +However, if this fails, then nothing else has a chance to work. + +Note that this is not in JSON format, as it is output by the `clap` +library, and `opgpcard` doesn't affect what it looks like. + +~~~scenario +given an installed opgpcard +when I run opgpcard --version +then stdout matches regex ^opgpcard \d+\.\d+\.\d+$ +~~~ + +## List cards: `opgpcard list` + +_Requirement: The tool lists available cards._ + +This is not at all a thorough test, but it exercises the simple happy +paths of the subcommand. + +~~~scenario +given an installed opgpcard +when I run opgpcard --output-format=json list +then stdout, as JSON, matches embedded file list.json +~~~ + +~~~{#list.json .file .json} +{ + "idents": ["AFAF:00001234"] +} +~~~ + +## Card status: `opgpcard status` + +_Requirement: The tool shows status of available cards._ + +This is not at all a thorough test, but it exercises the simple happy +paths of the subcommand. + +~~~scenario +given an installed opgpcard +when I run opgpcard --output-format=json status +then stdout, as JSON, matches embedded file status.json +~~~ + +~~~{#status.json .file .json} +{ + "card_version": "2.0", + "ident": "AFAF:00001234" +} +~~~ + +## Card information: `opgpcard info` + +_Requirement: The tool shows information about available cards._ + +This is not at all a thorough test, but it exercises the simple happy +paths of the subcommand. + +~~~scenario +given an installed opgpcard +when I run opgpcard --output-format=json info +then stdout, as JSON, matches embedded file info.json +~~~ + +~~~{#info.json .file .json} +{ + "card_version": "2.0", + "application_id": "D276000124 01 01 0200 AFAF 00001234 0000", + "manufacturer_id": "AFAF", + "manufacturer_name": "Unknown", + "card_service_data": "", + "ident": "AFAF:00001234" +} +~~~ diff --git a/tools/subplot/opgpcard.rs b/tools/subplot/opgpcard.rs new file mode 100644 index 0000000..0297eb4 --- /dev/null +++ b/tools/subplot/opgpcard.rs @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: 2022 Lars Wirzenius +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use subplotlib::file::SubplotDataFile; +use subplotlib::steplibrary::runcmd::Runcmd; + +use serde_json::Value; +use std::path::Path; + +#[derive(Debug, Default)] +struct SubplotContext {} + +impl ContextElement for SubplotContext {} + +#[step] +#[context(SubplotContext)] +#[context(Runcmd)] +fn install_opgpcard(context: &ScenarioContext) { + let target_exe = env!("CARGO_BIN_EXE_opgpcard"); + let target_path = Path::new(target_exe); + let target_path = target_path.parent().ok_or("No parent?")?; + context.with_mut( + |context: &mut Runcmd| { + context.prepend_to_path(target_path); + Ok(()) + }, + false, + )?; +} + +#[step] +#[context(Runcmd)] +fn stdout_matches_json_file(context: &ScenarioContext, file: SubplotDataFile) { + let expected: Value = serde_json::from_slice(file.data())?; + println!("expecting JSON: {:#?}", expected); + + let stdout = context.with(|runcmd: &Runcmd| Ok(runcmd.stdout_as_string()), false)?; + let actual: Value = serde_json::from_str(&stdout)?; + println!("stdout JSON: {:#?}", actual); + + println!("fuzzy checking JSON values"); + assert!(json_eq(&actual, &expected)); +} + +// Fuzzy match JSON values. For objects, anything in expected must be +// in actual, but it's OK for there to be extra things. +fn json_eq(actual: &Value, expected: &Value) -> bool { + match actual { + Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => { + println!("simple value"); + println!("actual ={:?}", actual); + println!("expected={:?}", expected); + let eq = actual == expected; + println!("simple value eq={}", eq); + return eq; + } + Value::Array(a_values) => { + if let Value::Array(e_values) = expected { + println!("both actual and equal are arrays"); + for (a_value, e_value) in a_values.iter().zip(e_values.iter()) { + println!("comparing corresponding array elements"); + if !json_eq(a_value, e_value) { + println!("array elements differ"); + return false; + } + } + println!("arrays match"); + return true; + } else { + println!("actual is array, expected is not"); + return false; + } + } + Value::Object(a_obj) => { + if let Value::Object(e_obj) = expected { + println!("both actual and equal are objects"); + for key in e_obj.keys() { + println!("checking key {}", key); + if !a_obj.contains_key(key) { + println!("key {} is missing from actual", key); + return false; + } + let a_value = a_obj.get(key).unwrap(); + let e_value = e_obj.get(key).unwrap(); + let eq = json_eq(a_value, e_value); + println!("values for {} eq={}", key, eq); + if !eq { + return false; + } + } + println!("objects match"); + return true; + } else { + println!("actual is object, expected is not"); + return false; + } + } + } +} diff --git a/tools/subplot/opgpcard.subplot b/tools/subplot/opgpcard.subplot new file mode 100644 index 0000000..855d350 --- /dev/null +++ b/tools/subplot/opgpcard.subplot @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2022 Lars Wirzenius +# SPDX-License-Identifier: MIT OR Apache-2.0 + +title: "opgpcard acceptance tests" +markdowns: + - opgpcard.md +bindings: + - opgpcard.yaml + - lib/files.yaml + - lib/runcmd.yaml +impls: + rust: + - opgpcard.rs +classes: + - json diff --git a/tools/subplot/opgpcard.yaml b/tools/subplot/opgpcard.yaml new file mode 100644 index 0000000..1bdc70b --- /dev/null +++ b/tools/subplot/opgpcard.yaml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2022 Lars Wirzenius +# SPDX-License-Identifier: MIT OR Apache-2.0 + +- given: an installed opgpcard + impl: + rust: + function: install_opgpcard + +- then: "stdout, as JSON, matches embedded file {file:file}" + impl: + rust: + function: stdout_matches_json_file diff --git a/tools/subplot/test-in-docker.sh b/tools/subplot/test-in-docker.sh new file mode 100755 index 0000000..13bf2e1 --- /dev/null +++ b/tools/subplot/test-in-docker.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Run "cargo test" inside a Docker container with virtual cards +# installed and running. This will allow at least rudimentary testing +# of actual card functionality of opgpcard. +# +# SPDX-FileCopyrightText: 2022 Lars Wirzenius +# SPDX-License-Identifier: MIT OR Apache-2.0 + +set -euo pipefail + +image="registry.gitlab.com/openpgp-card/virtual-cards/smartpgp-builddeps" + +src="$(cd .. && pwd)" + +if ! [ -e Cargo.toml ] || ! grep '^name.*openpgp-card-tools' Cargo.toml; then + echo "MUST run this in the root of the openpgp-card-tool crate" 1>&2 + exit 1 +fi + +cargo build +docker run --rm -t \ + --volume "cargo:/cargo" \ + --volume "dotcargo:/root/.cargo" \ + --volume "$src:/opt" "$image" \ + bash -c ' +/etc/init.d/pcscd start && \ +su - -c "sh /home/jcardsim/run-card.sh >/dev/null" jcardsim && \ +cd /opt/tools && env CARGO_TARGET_DIR=/cargo cargo test' diff --git a/tools/tests/opgpcard.rs b/tools/tests/opgpcard.rs new file mode 100644 index 0000000..72f3554 --- /dev/null +++ b/tools/tests/opgpcard.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: 2022 Lars Wirzenius +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#![allow(clippy::needless_return)] + +include!(concat!(env!("OUT_DIR"), "/opgpcard.rs"));