From d36626caac8e1fed37f830c18e3203d487ceb5e8 Mon Sep 17 00:00:00 2001 From: Douglas Stebila Date: Sat, 16 Feb 2019 10:07:59 -0500 Subject: [PATCH] Move metadata check to Python testing framework --- .travis.yml | 4 +- Makefile | 4 -- test/check_metadata.py | 156 ----------------------------------------- test/test_metadata.py | 118 +++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 162 deletions(-) delete mode 100644 test/check_metadata.py create mode 100644 test/test_metadata.py diff --git a/.travis.yml b/.travis.yml index 763df772..2ee6b21f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,11 @@ dist: xenial matrix: include: - - name: "Linux + GCC + Linting + Metadata" + - name: "Linux + GCC + Linting" os: linux compiler: gcc env: - - MAKETARGET="test-all tidy-all check-format check-metadata" + - MAKETARGET="test-all tidy-all check-format" addons: apt: packages: diff --git a/Makefile b/Makefile index 80722184..0ca441fa 100644 --- a/Makefile +++ b/Makefile @@ -189,7 +189,3 @@ apply-tidy-all: @for scheme in $(ALL_SCHEMES); do \ $(MAKE) apply-tidy SCHEME=$$scheme; \ done - -.PHONY: check-metadata -check-metadata: - python3 test/check_metadata.py diff --git a/test/check_metadata.py b/test/check_metadata.py deleted file mode 100644 index b6b82042..00000000 --- a/test/check_metadata.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -Verify the metadata specified in the META.yml files. -""" - -import yaml -import glob -import sys -import itertools -import copy - -import os.path - -def eprint(*args, **kwargs): - """Write to stderr""" - global errors - - print(*args, file=sys.stderr, **kwargs) - errors += 1 - - -EXPECTED_FIELDS = { - 'name': {'type': str}, - 'type': {'type': str}, - 'claimed-nist-level': {'type': int, 'min': 1, 'max': 5}, - 'length-public-key': {'type': int, 'min': 1}, - 'testvectors-sha256': {'type': str, 'length': 64}, - 'principal-submitter': {'type': str}, - 'auxiliary-submitters': {'type': list, 'elements': {'type': str}}, - 'implementations': { - 'type': list, - 'elements': { - 'type': dict, - 'spec': { - 'name': {'type': str}, - 'version': {'type': str}, - }, - }, - }, -} - -KEM_FIELDS = { - 'length-ciphertext': {'type': int, 'min': 1}, -} - -SIGNATURE_FIELDS = { - 'length-signature': {'type': int, 'min': 1}, -} - -errors = 0 - - -def check_spec(metadata, spec, metafile): - for field, props in spec: - if field not in metadata: - eprint("Field '{}' not present in '{}'.".format( - field, metafile)) - continue - - # validate element - check_element(field, metadata[field], props, metafile) - - # delete it to detect extras - del metadata[field] - - # Done checking all specified fields, check if we have extras - for field, value in metadata.items(): - eprint("Unexpected item '{}' in '{}' with value '{}'" - .format(field, metafile, value)) - - -def check_element(field, element, props, metafile): - type_ = props['type'] - # Validate type of element - try: - type_(element) - - # Strs are valid lists otherwise - if type_ == list and type(element) != list: - raise ValueError("Not a list") - # lists are valid dicts otherwise - if type_ == dict and type(element) != dict: - raise ValueError("Not a dict") - except ValueError as e: - eprint("Field '{}' in '{}' seems to be of incorrect type. " - "Expected '{}'. Got error '{}'." - .format(field, metafile, type_.__name__, e)) - return - - if type_ == int: - element = int(element) - if 'min' in props: - if element < props['min']: - eprint("Value of field '{}' in '{}' is lower than minimum " - "value {}".format(field, metafile, props['min'])) - if 'max' in props: - if element > props['max']: - eprint("Value of field '{}' in '{}' is larger than maximum" - " value {}".format(field, metafile, props['max'])) - if type_ == str: - if 'length' in props: - actual_len = len(element) - if actual_len != props['length']: - eprint("Value of field '{}' in '{}' should be length {}" - " but was length {}" - .format(field, metafile, - props['length'], actual_len)) - - if type_ == list: # recursively check the elements - for el in element: - check_element('element of {}'.format(field), - el, props['elements'], metafile) - if type_ == dict: - check_spec(element, props['spec'].items(), metafile) - - -for directory in glob.iglob('crypto_*/*/'): - metafile = os.path.join(directory, 'META.yml') - try: - with open(metafile, encoding='utf-8') as f: - metadata = yaml.load(f.read()) - except Exception as e: - eprint("Can't open {}: {}".format(metafile, e)) - continue - - specification = EXPECTED_FIELDS.items() - if 'crypto_kem' in metafile: - specification = itertools.chain(specification, KEM_FIELDS.items()) - elif 'crypto_sign' in metafile: - specification = itertools.chain(specification, - SIGNATURE_FIELDS.items()) - - check_spec(copy.deepcopy(metadata), specification, metafile) - - unspecified_impls = glob.glob(directory + '*/') - if 'implementations' in metadata: - for implementation in metadata['implementations']: - if 'name' not in implementation: # problem is reported elsewhere - continue - implpath = os.path.join(directory, implementation['name']) - if not os.path.isdir(implpath): - eprint("There is no implementation at '{}' but '{}' was " - "specified in {}".format( - implpath, implementation['name'], metafile)) - implpath += '/' # adjust for trailing / in unspecified_impls - if implpath in unspecified_impls: - unspecified_impls.remove(implpath) - - for impl in unspecified_impls: - eprint("Implementation '{}' is not specified in '{}'." - .format(impl, metafile)) - - - - - -exit(errors) diff --git a/test/test_metadata.py b/test/test_metadata.py new file mode 100644 index 00000000..1faf8a73 --- /dev/null +++ b/test/test_metadata.py @@ -0,0 +1,118 @@ +""" +Verify the metadata specified in the META.yml files. +""" + +import copy +import itertools +import os +import pqclean +import yaml +import unittest + +def test_metadata(): + for scheme in pqclean.Scheme.all_schemes(): + yield check_metadata, scheme.name + +def check_metadata(scheme_name): + scheme = pqclean.Scheme.by_name(scheme_name) + metadata = scheme.metadata() + + specification = EXPECTED_FIELDS.items() + + if scheme.type == 'kem': + specification = itertools.chain(specification, KEM_FIELDS.items()) + elif scheme.type == 'sign': + specification = itertools.chain(specification, SIGNATURE_FIELDS.items()) + else: + assert(False) + + check_spec(copy.deepcopy(metadata), specification) + + implementation_names_in_yaml = set(i['name'] for i in metadata['implementations']) + implementations_on_disk = set(i.name for i in scheme.implementations) + if implementation_names_in_yaml != implementations_on_disk: + raise AssertionError("Implementations in YAML file {} and implementations on disk {} do not match" + .format(implementation_names_in_yaml, implementations_on_disk)) + +EXPECTED_FIELDS = { + 'name': {'type': str}, + 'type': {'type': str}, + 'claimed-nist-level': {'type': int, 'min': 1, 'max': 5}, + 'length-public-key': {'type': int, 'min': 1}, + 'testvectors-sha256': {'type': str, 'length': 64}, + 'principal-submitter': {'type': str}, + 'auxiliary-submitters': {'type': list, 'elements': {'type': str}}, + 'implementations': { + 'type': list, + 'elements': { + 'type': dict, + 'spec': { + 'name': {'type': str}, + 'version': {'type': str}, + }, + }, + }, +} + +KEM_FIELDS = { + 'length-ciphertext': {'type': int, 'min': 1}, +} + +SIGNATURE_FIELDS = { + 'length-signature': {'type': int, 'min': 1}, +} + +def check_spec(metadata, spec): + for field, props in spec: + if field not in metadata: + raise AssertionError("Field '{}' not present.".format(field)) + + # validate element + check_element(field, metadata[field], props) + + # delete it to detect extras + del metadata[field] + + # Done checking all specified fields, check if we have extras + for field, value in metadata.items(): + raise AssertionError("Unexpected item '{}' with value '{}'".format(field, value)) + + +def check_element(field, element, props): + type_ = props['type'] + # Validate type of element + type_(element) + + # Strs are valid lists otherwise + if type_ == list and type(element) != list: + raise ValueError("Field {} not a list".format(field)) + + # lists are valid dicts otherwise + if type_ == dict and type(element) != dict: + raise ValueError("Field {} not a dict".format(field)) + + if type_ == int: + element = int(element) + if 'min' in props: + if element < props['min']: + raise ValueError("Value of field '{}' is lower than minimum " + "value {}".format(field, props['min'])) + if 'max' in props: + if element > props['max']: + raise ValueError("Value of field '{}' is larger than maximum" + " value {}".format(field, metafile, props['max'])) + + if type_ == str: + if 'length' in props: + actual_len = len(element) + if actual_len != props['length']: + raise ValueError("Value of field '{}' should be length {}" + " but was length {}" + .format(field, props['length'], actual_len)) + + if type_ == list: # recursively check the elements + for el in element: + check_element('element of {}'.format(field), el, props['elements']) + + if type_ == dict: + check_spec(element, props['spec'].items())