biosimulators_test_suite package#

Subpackages#

Submodules#

biosimulators_test_suite.config module#

Configuration

Author:

Jonathan Karr <karr@mssm.edu>

Date:

2021-01-30

Copyright:

2021, Center for Reproducible Biomedical Modeling

License:

MIT

class biosimulators_test_suite.config.Config(pull_docker_image=None, docker_hub_username=None, docker_hub_token=None, biosimulators_auth_endpoint=None, biosimulators_audience=None, biosimulators_api_client_id=None, biosimulators_api_client_secret=None, biosimulators_prod_api_endpoint=None, biosimulators_dev_api_endpoint=None, biosimulators_curator_gh_ids=None, biosimulators_default_specifications_version=None, biosimulators_default_image_version=None, biosimulators_docker_registry_url=None, biosimulators_docker_registry_username=None, biosimulators_docker_registry_token=None, biosimulators_docker_image_url_pattern=None, runbiosimulations_auth_endpoint=None, runbiosimulations_audience=None, runbiosimulations_api_client_id=None, runbiosimulations_api_client_secret=None, runbiosimulations_api_endpoint=None, test_case_timeout=None, user_to_exec_in_simulator_containers=None, singularity_image_dirname=None)[source]#

Bases: object

Configuration

pull_docker_image[source]#

whether to pull the Docker image for the simulator (default: True)

Type:

bool

docker_hub_username[source]#

username for Docker Hub

Type:

str

docker_hub_token[source]#

Token for Docker Hub

Type:

str

biosimulators_auth_endpoint[source]#

Authentification endpoint for the BioSimulators API

Type:

str

biosimulators_audience[source]#

Open API audience for the BioSimulators API

Type:

str

biosimulators_api_client_id[source]#

Client id of the BioSimulators API

Type:

str

biosimulators_api_client_secret[source]#

Client secret of the BioSimulators API

Type:

str

biosimulators_prod_api_endpoint[source]#

Base URL for the BioSimulators API

Type:

str

biosimulators_dev_api_endpoint[source]#

Base URL for the BioSimulators API

Type:

str

biosimulators_curator_gh_ids[source]#

GitHub user ids of the BioSimulators curators

Type:

list of str

biosimulators_default_specifications_version[source]#

Default version of the BioSimulators simulation specifications

Type:

str

biosimulators_default_image_version[source]#

Default version of the BioSimulators Docker image format

Type:

str

biosimulators_docker_registry_url[source]#

URL of the Docker registry for simulators

Type:

str

biosimulators_docker_registry_username[source]#

username for Docker registry for simulators

Type:

str

biosimulators_docker_registry_token[source]#

token for Docker registry for simulators

Type:

str

biosimulators_docker_image_url_pattern[source]#

URL pattern for Docker images for simulators

Type:

str

runbiosimulations_auth_endpoint[source]#

Authentification endpoint for the runBioSimulations API

Type:

str

runbiosimulations_audience[source]#

Open API audience for the runBioSimulations API

Type:

str

runbiosimulations_api_client_id[source]#

Client id of the runBioSimulations API

Type:

str

runbiosimulations_api_client_secret[source]#

Client secret of the runBioSimulations API

Type:

str

runbiosimulations_api_endpoint[source]#

Base URL for the runBioSimulations API

Type:

str

test_case_timeout[source]#

time out for test cases in seconds

Type:

int

user_to_exec_in_simulator_containers[source]#

user id or name to execute calls inside simulator containers

  • Use _CURRENT_USER_ to indicate that the Docker container should execute commands as the current user (os.getuid())

  • Use the format <name|uid>[:<group|gid>] to indicate any other user/group that the Docker container should use to execute commands

Type:

str or None

singularity_image_dirname[source]#

directory to save Singularity images

Type:

str

biosimulators_test_suite.data_model module#

Data model for test cases for validating simulators

Author:

Jonathan Karr <karr@mssm.edu>

Date:

2020-12-21

Copyright:

2020, Center for Reproducible Biomedical Modeling

License:

MIT

class biosimulators_test_suite.data_model.AlertType(value)[source]#

Bases: str, Enum

Type of alert upon the failure of a test case

exception = 'exception'[source]#
warning = 'warning'[source]#
class biosimulators_test_suite.data_model.ExpectedSedDataSet(id=None, label=None)[source]#

Bases: object

An expected SED report

Attributes

id (str): id label (str): label

class biosimulators_test_suite.data_model.ExpectedSedPlot(id=None)[source]#

Bases: object

An expected SED report

Attributes

id (str): id

class biosimulators_test_suite.data_model.ExpectedSedReport(id=None, data_sets=None, points=None, values=None)[source]#

Bases: object

An expected SED report

Attributes

id (str): id data_sets (list of ExpectedSedDataSet): labels of expected data sets points (tuple of int): number of expected points of values (dict of str to dict of list): expected values of data sets or elements of data sets

class biosimulators_test_suite.data_model.OutputMedium(value)[source]#

Bases: str, Enum

Output medium

console = 'console'[source]#
gh_issue = 'gh_issue'[source]#
class biosimulators_test_suite.data_model.SedTaskRequirements(model_format=None, model_format_features=None, simulation_algorithm=None)[source]#

Bases: object

Required model format for simulation algorithm for each task in a SED document

model_format[source]#

EDAM id for the format of the model involved in the task

Type:

str

model_format_features[source]#

model format features required to execute the task such as SBML packages

Type:

set of str

simulation_algorithm[source]#

KiSAO id for the simulation algorithm involved in the task

Type:

str

class biosimulators_test_suite.data_model.TestCase(id=None, name=None, description=None, output_medium=OutputMedium.console)[source]#

Bases: ABC

A test case for validating a simulator

id[source]#

id

Type:

str

name[source]#

name

Type:

str

description[source]#

description

Type:

str

output_medium[source]#

medium the description should be formatted for

Type:

OutputMedium

abstract eval(specifications, working_dirname, synthetic_archives_dir=None, dry_run=False, cli=None)[source]#

Evaluate a simulator’s performance on a test case

Parameters:

  • specifications (dict) – specifications of the simulator to validate

  • working_dirname (str) – directory for temporary files for evaluating test case

  • synthetic_archives_dir (str, optional) – Directory to save the synthetic COMBINE/OMEX archives generated by the test cases

  • dry_run (bool, optional) – if True, do not use the simulator to execute COMBINE/OMEX archives.

  • cli (str, optional) – command-line interface to use to execute the tests involving the simulation of COMBINE/OMEX archives rather than a Docker image

Raises:

  • SkippedTestCaseException – if the test case is not applicable to the simulator

  • Exception – if the simulator did not pass the test case

get_simulator_docker_image(specifications, pull=None)[source]#

Get the Docker image for a simulator, pulling if necessary

Parameters:

specifications (dict) – specifications of the simulator to validate

Returns:

Docker image

Return type:

docker.models.images.Image

biosimulators_test_suite.exceptions module#

Common exceptions for test cases

Author:

Jonathan Karr <karr@mssm.edu>

Date:

2020-12-21

Copyright:

2020, Center for Reproducible Biomedical Modeling

License:

MIT

exception biosimulators_test_suite.exceptions.InvalidOutputsException[source]#

Bases: TestCaseException

Exception raised when outputs of execution of COMBINE/OMEX archive are not as expected

exception biosimulators_test_suite.exceptions.SkippedTestCaseException[source]#

Bases: TestCaseException

Exception raised that indicates that a test case should be skipped

exception biosimulators_test_suite.exceptions.TestCaseException[source]#

Bases: Exception

Exception raised when outputs of execution of COMBINE/OMEX archive are not as expected

exception biosimulators_test_suite.exceptions.TimeoutException[source]#

Bases: TestCaseException

Exception raised that indicates that a test case timed out

biosimulators_test_suite.exec_cli module#

Program for validating that simulation tools are consistent with the BioSimulators standards

Author:

Jonathan Karr <karr@mssm.edu>

Date:

2020-12-22

Copyright:

2020, BioSimulators Team

License:

MIT

class biosimulators_test_suite.exec_cli.App(label=None, **kw)[source]#

Bases: App

Command line application

class Meta[source]#

Bases: object

base_controller = 'base'[source]#
handlers = [<class 'biosimulators_test_suite.exec_cli.BaseController'>][source]#
label = 'biosimulators-test-suite'[source]#
class biosimulators_test_suite.exec_cli.BaseController(*args, **kw)[source]#

Bases: ArgparseController

Base controller for command line application

class Meta[source]#

Bases: object

arguments = [(['specifications'], {'type': <class 'str'>, 'help': 'Path or URL to the specifications of the simulator'}), (['-c', '--test-case'], {'type': <class 'str'>, 'nargs': '+', 'default': None, 'dest': 'case_ids', 'help': "Ids of test cases of evaluate (e.g., 'sedml.SimulatorSupportsModelAttributeChanges', 'published_project.SimulatorCanExecutePublishedProject:sbml-core/Caravagna-J-Theor-Biol-2010-tumor-suppressive-oscillations') substrings of ids of test cases to evaluate (e.g., 'sedml.', 'published_project.' to evaluate all SED-ML test cases or all test cases involving published projects). Default: evaluate all test cases"}), (['--cli'], {'default': None, 'help': 'Command-line interface to use to execute the tests involving the simulation of COMBINE/OMEX archives rather than a Docker image'}), (['--synthetic-archives-dir'], {'default': None, 'help': 'Directory to save the synthetic COMBINE/OMEX archives generated by the test cases'}), (['--report'], {'default': None, 'help': 'Path to save a report of the results in JSON format'}), (['--verbose'], {'action': 'store_true', 'help': 'If set, print the stdout and stderr of the execution of the tests in real time.'}), (['--work-dir'], {'default': None, 'help': 'Working directory for files for evaluating tests. This option enables intermediate files to be inspected rather than automatically cleaned up.'}), (['--do-not-validate-specs'], {'action': 'store_true', 'help': "If set, don't validate the specifications of the simulator."}), (['--do-not-log-std-out-err'], {'action': 'store_true', 'help': "If set, don't use capturer to collect stdout and stderr."}), (['--dry-run'], {'action': 'store_true', 'help': 'If set, create synthetic archives, but do not use the simulator to execute them.'}), (['-v', '--version'], {'action': 'version', 'version': '0.1.88'})][source]#
description = 'Validates that a simulation tool is consistent with the BioSimulators standards'[source]#
help = 'Validates that a simulation tool is consistent with the BioSimulators standards'[source]#
label = 'base'[source]#
biosimulators_test_suite.exec_cli.main()[source]#

biosimulators_test_suite.exec_core module#

Utilities for validate containerized simulators

Author:

Jonathan Karr <karr@mssm.edu>

Date:

2020-12-21

Copyright:

2020, Center for Reproducible Biomedical Modeling

License:

MIT

class biosimulators_test_suite.exec_core.SimulatorValidator(specifications, case_ids=None, verbose=False, synthetic_archives_dir=None, output_medium=OutputMedium.console, log_std_out_err=True, working_dirname=None, dry_run=False, cli=None, validate_specs=True)[source]#

Bases: object

Validate that a Docker image for a simulator implements the BioSimulations simulator interface by checking that the image produces the correct outputs for one of more test cases (e.g., COMBINE archive)

specifications[source]#

path or URL to the specifications of the simulator, or the specifications of the simulator

Type:

str or dict

cases[source]#

groups of test cases

Type:

collections.OrderedDict of types.ModuleType to TestCase

verbose[source]#

if True, display stdout/stderr from executing cases in real time

Type:

bool

synthetic_archives_dir[source]#

Directory to save the synthetic COMBINE/OMEX archives generated by the test cases

Type:

str

output_medium[source]#

environment where outputs will be sent

Type:

OutputMedium

log_std_out_err[source]#

whether to log the standard output and error generated by each test case

Type:

bool

working_dirname[source]#

directory for temporary files for evaluating test case

Type:

str

dry_run[source]#

if True, do not use the simulator to execute COMBINE/OMEX archives.

Type:

bool

cli[source]#

command-line interface to use to execute the tests involving the simulation of COMBINE/OMEX archives rather than a Docker image

Type:

str

eval_case(case, working_dirname)[source]#

Evaluate a test case for a simulator

Parameters:

  • case (TestCase) – test case

  • working_dirname (str) – directory for temporary files for evaluating test case

Returns:

test case result

Return type:

TestCaseResult

find_cases(ids=None)[source]#

Find test cases

Parameters:

ids (list of str, optional) – List of ids of test cases to verify. If ids is none, all test cases are verified.

Returns:

groups of test cases

Return type:

collections.OrderedDict of types.ModuleType to TestCase

find_cases_in_module(module, published_projects_test_cases, ids=None)[source]#

Discover test cases in a module

Parameters:

  • module (types.ModuleType) – module

  • ids (list of str, optional) – List of ids of test cases to verify. If ids is none, all test cases are verified.

  • published_projects_test_cases (list of published_project.SimulatorCanExecutePublishedProject) – test cases involving executing curated COMBINE/OMEX archives

Returns:

test cases

Return type:

list of TestCase

run()[source]#

Validate that a Docker image for a simulator implements the BioSimulations simulator interface by checking that the image produces the correct outputs for test cases (e.g., COMBINE archive)

Returns:

results of executing test cases

Return type:

list TestCaseResult

static summarize_results(results, debug=False, output_medium=OutputMedium.console)[source]#

Get a summary of the results of a set of test cases

Parameters:

  • results (list TestCaseResult) – results of executing test cases

  • debug (bool, optional) – whether to display traceback information about each error with additional information for debugging

  • output_medium (OutputMedium, optional) – environment where outputs will be sent

Returns:

tuple

  • str: summary of results of test cases

  • list of str: details of failures

  • list of str: details of warnings

  • list of str: details of skips

biosimulators_test_suite.exec_gh_action module#

Methods for processing submissions to the BioSimulators registry (CI workflows for reviewing and committing simulators to the registry)

Author:

Jonathan Karr <karr@mssm.edu>

Date:

2020-12-06

Copyright:

2020, Center for Reproducible Biomedical Modeling

License:

MIT

class biosimulators_test_suite.exec_gh_action.ValidateCommitSimulatorGitHubAction[source]#

Bases: GitHubAction

Action to validate a containerized simulator

issue_number[source]#

number of GitHub issue which triggered the action

Type:

str

classmethod add_comment_to_issue(issue_number, comment, alternative_comment=None, max_len=65536)[source]#

Post a comment to the GitHub issue

Parameters:

  • issue_number (str) – issue number

  • comment (str) – comment

  • alternative_comment (str, optional) – optional alternative comment to try posting to the GitHub issue

  • max_len (int, optional) – maximum comment length accepted by GitHub

commit_simulator(submission, specifications, existing_version_specifications, test_results)[source]#

Commit simulator to the BioSimulators registry

Parameters:

  • submission (SimulatorSubmission) – simulator submission

  • specifications (dict) – specifications of a simulation tool

  • existing_version_specifications (list of dict) – specifications of other versions of simulation tool

  • test_results (list of TestCaseResults) – results of test cases

does_submission_have_approved_label()[source]#

Determine whether an issue for submitting a simulator already has the approved label

Returns:

True, if the issue for the submission already has the approved label

Return type:

bool

exec_core(submission, submitter)[source]#

Validate simulator

  • Validate specifications

  • Validate image

Parameters:

  • submission (SimulatorSubmission) – simulator submission

  • submitter (str) – GitHub id of the submitter

Returns:

  • dict: specifications of a simulation tool

  • list of TestCaseResults: results of test cases

Return type:

tuple

get_auth_headers_for_biosimulations_api(endpoint, audience, client_id, client_secret)[source]#

Get authorization headers for using one of the BioSimulations REST APIs (BioSimulators, runBioSimulations, etc.).

Parameters:

  • endpoint (str) – URL for getting an authentication token

  • audience (str) – API audience

  • client_id (str) – id for this client

  • client_secret (str) – secret for this client

Returns:

authorization headers

Return type:

dict

get_initial_message(submission, submitter)[source]#

Peport message that validation is starting

Parameters:

  • submission (SimulatorSubmission) – simulator submission

  • submitter (str) – GitHub user name of person who executed the submission

is_other_version_of_simulator_validated(specifications, existing_version_specifications)[source]#

Determine whether another version of the simulation tool has already been approved

Parameters:

  • specifications (dict) – specifications of a simulation tool

  • existing_version_specifications (list of dict) – specifications of other versions of simulation tool

Returns:

True, if the simulation tool has already been approved

Return type:

bool

is_simulator_approved(specifications, existing_version_specifications)[source]#

Determine whether a simulation tool has already been approved

Parameters:

  • specifications (dict) – specifications of a simulation tool

  • existing_version_specifications (list of dict) – specifications of other versions of simulation tool

Returns:

True, if the simulation tool has already been approved

Return type:

bool

is_submission_latest_version_of_simulator(specifications, existing_version_specifications)[source]#

Determine whether the submitted version the latest version of the simulator (greatest tag)

Parameters:

  • specifications (dict) – specifications of a simulation tool

  • existing_version_specifications (list of dict) – specifications of other versions of simulation tool

Returns:

True if the submitted version if the latest version of the simulator

Return type:

bool

post_entry_to_biosimulators_api(specifications, existing_version_specifications, biosimulators_api_endpoint)[source]#

Post the simulation to the BioSimulators database

Parameters:

  • specifications (dict) – specifications of a simulation tool

  • existing_version_specifications (list of dict) – specifications of other versions of simulation tool

  • biosimulators_api_endpoint (str) – endpoint for the BioSimulators API (e.g., https://api.biosimulators.org)

push_image(specifications, existing_version_specifications)[source]#

Push the image for a simulation tool to the GitHub Container Registry

Parameters:

  • specifications (dict) – specifications of a simulation tool

  • existing_version_specifications (list of dict) – specifications of other versions of simulation tool

run()[source]#

Validate and commit a simulator.

trigger_conversion_of_docker_image_to_singularity(specifications)[source]#

Post the simulation to the BioSimulators database

Parameters:

specifications (dict) – specifications of a simulation tool

validate_image(specifications)[source]#

Validate a Docker image for simulation tool

Parameters:

specifications (dict) – specifications of a simulation tool

Returns:

results of test cases

Return type:

list of TestCaseResults

validate_permissions(simulator_id, simulator_name, submitter)[source]#

Validate that the submitter has permissions to submit or update the simulator

Parameters:

  • simulator_id (str) – simulator id

  • simulator_name (str) – simulator name

  • submitter (str) – GitHub id of the submitter

biosimulators_test_suite.utils module#

Utility methods

Author:

Jonathan Karr <karr@mssm.edu>

Date:

2021-06-05

Copyright:

2021, Center for Reproducible Biomedical Modeling

License:

MIT

biosimulators_test_suite.utils.get_singularity_image_filename(docker_image)[source]#

Get the location where a Singularity version of a Docker image should be saved

Parameters:

docker_image (str) – URL for the Docker version of the image

Returns:

path where a Singularity version of a Docker image should be saved

Return type:

str

biosimulators_test_suite.utils.simulation_results_isnan(value)[source]#

Determine whether a scalar or each element of an array is NaN.

Parameters:

value (int, float, or numpy.ndarray) – scalar or array

Returns:

whether the value or each element of the value is NaN

Return type:

bool or numpy.ndarray

biosimulators_test_suite.warnings module#

Common warnings for test cases

Author:

Jonathan Karr <karr@mssm.edu>

Date:

2020-12-21

Copyright:

2020, Center for Reproducible Biomedical Modeling

License:

MIT

exception biosimulators_test_suite.warnings.IgnoredTestCaseWarning[source]#

Bases: TestCaseWarning

Warning that a test case was ignored

exception biosimulators_test_suite.warnings.InvalidOutputsWarning[source]#

Bases: TestCaseWarning

Warning that the outputs of the execution of a COMBINE/OMEX archive were not as expected

exception biosimulators_test_suite.warnings.SimulatorRuntimeErrorWarning[source]#

Bases: TestCaseWarning, RuntimeWarning

Warning that the execution of a test case failed

exception biosimulators_test_suite.warnings.TestCaseWarning[source]#

Bases: UserWarning

Base class for warnings collected from test cases