verify command (#225)

* Add verify command

* Update help

* Broken tests

* Small startup performance improvments

* Dict for command names

* Fix bug and remove java from compilers

* Fix tests

* Add tests

* Fix tests

* Dont create ocen archive with no ocen tests, create dlazaw in attachments

* Add tests

* Remove .cache dir

* Add gitkeep in inwer package

* Validate tests in parallel

* Less colorful printing

* Refactor --cpus flag

* Refactor imports

* Fix test

* Remove cache on compilation flags or sanitizers change

* Add tests

* Fix again

* Fix tests

* Fix

* Development versions support (#230)

* Add dev versioning

* Update tests

* Bump version

* Add test

* Bump version

* Fixed finding newest development version

* Bump version

* Fix version tests

* Refactor

* Cache `get_task_id()`

* Allow caching with cwd

* Fix test

* Don't create ocen when dlazaw exists

* Add test

* Add flag for ignoring expected scores

* Add test

* Bump version

* Update README

* Add option to verify contest type

* Add tests

* Fix bugs

* Fix typo

* Refactor creating parsers

* Change version for release

Author: MasloMaslane

README.md

@@ -83,6 +83,9 @@ Run `sinol-make ingen --help` to see available flags.
 program to use, what tests to check and how many CPUs to use. Run `sinol-make inwer --help` to see available flags.
 - `sinol-make export` -- Creates archive ready to upload to sio2 or szkopul. Run `sinol-make export --help` to see all available flags.
 - `sinol-make doc` -- Compiles all LaTeX files in doc/ directory to PDF. Run `sinol-make doc --help` to see all available flags.
+- `sinol-make verify` -- Verifies the package. This command runs stress tests (if available), verifies the config,
+generates tests, generates problem statements, runs inwer and run all solutions. Ingen and inwer are compiled with
+address and UB sanitizers. Run `sinol-make verify --help` to see all available flags.
 - `sinol-make init [id]` -- Creates package from template [on github](https://github.com/sio2project/sinol-make/tree/main/example_package) and sets task id to provided `[id]`. Requires an internet connection to run.
 
 You can also run multiple commands at once, for example:

setup.cfg

@@ -29,6 +29,7 @@ install_requires =
     dictdiffer
     importlib-resources
     psutil
+    packaging
 
 [options.packages.find]
 where = src

src/sinol_make/__init__.py

@@ -6,7 +6,8 @@
 
 from sinol_make import util, oiejq
 
-__version__ = "1.5.30"
+
+__version__ = "1.6.0"
 
 
 def configure_parsers():
@@ -50,35 +51,40 @@ def main_exn():
     parser = configure_parsers()
     arguments = []
     curr_args = []
+    commands = util.get_commands()
+    commands_dict = {command.get_name(): command for command in commands}
     for arg in sys.argv[1:]:
-        if arg in util.get_command_names() and not (len(curr_args) > 0 and curr_args[0] == 'init'):
+        if arg in commands_dict.keys() and not (len(curr_args) > 0 and curr_args[0] == 'init'):
             if curr_args:
                 arguments.append(curr_args)
             curr_args = [arg]
         else:
             curr_args.append(arg)
     if curr_args:
         arguments.append(curr_args)
-    commands = util.get_commands()
+    if not arguments:
+        parser.print_help()
+        exit(1)
     check_oiejq()
 
     for curr_args in arguments:
         args = parser.parse_args(curr_args)
-        command_found = False
-        for command in commands:
-            if command.get_name() == args.command:
-                command_found = True
-                if len(arguments) > 1:
-                    print(f' {command.get_name()} command '.center(util.get_terminal_size()[1], '='))
-                command.run(args)
-        if not command_found:
+        command = commands_dict.get(args.command, None)
+        if command:
+            if len(arguments) > 1:
+                print(f' {command.get_name()} command '.center(util.get_terminal_size()[1], '='))
+            command.run(args)
+        else:
             parser.print_help()
             exit(1)
 
 
 def main():
     new_version = None
     try:
+        if util.is_dev(__version__):
+            print(util.warning('You are using a development version of sinol-make. '
+                               'It may be unstable and contain bugs.'))
         new_version = util.check_for_updates(__version__)
         main_exn()
     except argparse.ArgumentError as err:
@@ -92,6 +98,12 @@ def main():
                              'https://github.com/sio2project/sinol-make/#reporting-bugs-and-contributing-code')
     finally:
         if new_version is not None:
-            print(util.warning(
-                f'New version of sinol-make is available (your version: {__version__}, available version: '
-                f'{new_version}).\nYou can update it by running `pip3 install sinol-make --upgrade`.'))
+            if not util.is_dev(new_version):
+                print(util.warning(
+                    f'New version of sinol-make is available (your version: {__version__}, available version: '
+                    f'{new_version}).\nYou can update it by running `pip3 install sinol-make --upgrade`.'))
+            elif util.is_dev(new_version):
+                print(util.warning(
+                    f'New development version of sinol-make is available (your version: {__version__}, available '
+                    f'version: {new_version}).\nYou can update it by running `pip3 install sinol-make --pre --upgrade`.'
+                ))

src/sinol_make/commands/doc/__init__.py

@@ -18,7 +18,7 @@ def get_name(self):
         return "doc"
 
     def compile_file_latex_div(self, file_path):
-        print(util.info(f'Compiling {os.path.basename(file_path)} (latex to dvi)...'))
+        print(f'Compiling {os.path.basename(file_path)} (latex to dvi)...')
         os.chdir(os.path.dirname(file_path))
         subprocess.run(['latex', file_path])
         dvi_file = os.path.splitext(file_path)[0] + '.dvi'
@@ -35,7 +35,7 @@ def compile_file_latex_div(self, file_path):
         return True
 
     def compile_pdf_latex(self, file_path):
-        print(util.info(f'Compiling {os.path.basename(file_path)} (pdflatex)...'))
+        print(f'Compiling {os.path.basename(file_path)} (pdflatex)...')
         os.chdir(os.path.dirname(file_path))
         subprocess.run(['pdflatex', file_path])
         pdf_file = os.path.splitext(file_path)[0] + '.pdf'
@@ -62,7 +62,7 @@ def move_logs(self):
         for pattern in self.LOG_PATTERNS:
             for file in glob.glob(os.path.join(os.getcwd(), 'doc', pattern)):
                 os.rename(file, os.path.join(output_dir, os.path.basename(file)))
-        print(util.info(f'Compilation log files can be found in {os.path.relpath(output_dir, os.getcwd())}'))
+        print(f'Compilation log files can be found in {os.path.relpath(output_dir, os.getcwd())}')
 
     def configure_subparser(self, subparser: argparse.ArgumentParser):
         parser = subparser.add_parser(
@@ -76,6 +76,7 @@ def configure_subparser(self, subparser: argparse.ArgumentParser):
                                  ' pdflatex - uses pdflatex. Works with .png and .jpg images.\n'
                                  ' latex_dvi - uses latex and dvipdf. Works with .ps and .eps images.', default='auto')
         parser.add_argument('files', type=str, nargs='*', help='files to compile')
+        return parser
 
     def run(self, args: argparse.Namespace):
         args = util.init_package_command(args)

src/sinol_make/commands/export/__init__.py

@@ -28,15 +28,13 @@ def configure_subparser(self, subparser: argparse.ArgumentParser):
             self.get_name(),
             help='Create archive for oioioi upload',
             description='Creates archive in the current directory ready to upload to sio2 or szkopul.')
-        parser.add_argument('-c', '--cpus', type=int,
-                            help=f'number of cpus to use to generate output files '
-                                 f'(default: {util.default_cpu_count()})',
-                            default=util.default_cpu_count())
+        parsers.add_cpus_argument(parser, 'number of cpus to use to generate output files')
         parser.add_argument('--no-statement', dest='no_statement', action='store_true',
                             help='allow export without statement')
         parser.add_argument('--export-ocen', dest='export_ocen', action='store_true',
                             help='Create ocen archive')
         parsers.add_compilation_arguments(parser)
+        return parser
 
     def generate_input_tests(self):
         print('Generating tests...')
@@ -97,6 +95,7 @@ def create_ocen(self, target_dir: str):
         Creates ocen archive for sio2.
         :param target_dir: Path to exported package.
         """
+        print('Generating ocen archive...')
         attachments_dir = os.path.join(target_dir, 'attachments')
         if not os.path.exists(attachments_dir):
             os.makedirs(attachments_dir)
@@ -109,12 +108,27 @@ def create_ocen(self, target_dir: str):
             os.makedirs(in_dir)
             out_dir = os.path.join(ocen_dir, 'out')
             os.makedirs(out_dir)
+            num_tests = 0
             for ext in ['in', 'out']:
                 for test in glob.glob(os.path.join(tests_dir, ext, f'{self.task_id}0*.{ext}')) + \
                             glob.glob(os.path.join(tests_dir, ext, f'{self.task_id}*ocen.{ext}')):
                     shutil.copy(test, os.path.join(ocen_dir, ext, os.path.basename(test)))
-
-            shutil.make_archive(os.path.join(attachments_dir, f'{self.task_id}ocen'), 'zip', tmpdir)
+                    num_tests += 1
+
+            dlazaw_dir = os.path.join(os.getcwd(), 'dlazaw')
+            if num_tests == 0:
+                print(util.warning('No ocen tests found.'))
+            elif os.path.exists(dlazaw_dir):
+                print(util.warning('Skipping ocen archive creation because dlazaw directory exists.'))
+            else:
+                shutil.make_archive(os.path.join(attachments_dir, f'{self.task_id}ocen'), 'zip', tmpdir)
+
+            if os.path.exists(dlazaw_dir):
+                print('Archiving dlazaw directory and adding to attachments.')
+                os.makedirs(os.path.join(tmpdir, 'dlazaw'), exist_ok=True)
+                shutil.copytree(dlazaw_dir, os.path.join(tmpdir, 'dlazaw', 'dlazaw'))
+                shutil.make_archive(os.path.join(attachments_dir, 'dlazaw'), 'zip',
+                                    os.path.join(tmpdir, 'dlazaw'))
 
     def compile_statement(self):
         command = DocCommand()
@@ -168,7 +182,6 @@ def copy_package_required_files(self, target_dir: str):
 
         self.generate_output_files()
         if self.args.export_ocen:
-            print('Generating ocen archive...')
             self.create_ocen(target_dir)
 
     def clear_files(self, target_dir: str):

src/sinol_make/commands/gen/__init__.py

@@ -31,13 +31,12 @@ def configure_subparser(self, subparser):
                             help='path to ingen source file, for example prog/abcingen.cpp')
         parser.add_argument('-i', '--only-inputs', action='store_true', help='generate input files only')
         parser.add_argument('-o', '--only-outputs', action='store_true', help='generate output files only')
-        parser.add_argument('-c', '--cpus', type=int,
-                            help=f'number of cpus to use to generate output files '
-                                 f'(default: {util.default_cpu_count()})',
-                            default=util.default_cpu_count())
+        parsers.add_cpus_argument(parser, 'number of cpus to use to generate output files')
         parser.add_argument('-n', '--no-validate', default=False, action='store_true',
                             help='do not validate test contents')
+        parsers.add_fsanitize_argument(parser)
         parsers.add_compilation_arguments(parser)
+        return parser
 
     def run(self, args: argparse.Namespace):
         args = util.init_package_command(args)

src/sinol_make/commands/ingen/__init__.py

@@ -30,7 +30,10 @@ def configure_subparser(self, subparser):
                             help='path to ingen source file, for example prog/abcingen.cpp')
         parser.add_argument('-n', '--no-validate', default=False, action='store_true',
                             help='do not validate test contents')
+        parsers.add_cpus_argument(parser, 'number of cpus used for validating tests')
+        parsers.add_fsanitize_argument(parser)
         parsers.add_compilation_arguments(parser)
+        return parser
 
     def delete_dangling_files(self, dates):
         to_delete = set()
@@ -51,7 +54,7 @@ def delete_dangling_files(self, dates):
                 static_files = set([os.path.basename(test) for test in static_files])
                 to_delete = to_delete - static_files
                 if to_delete:
-                    print(util.info('Cleaning up old input files.'))
+                    print('Cleaning up old input files.')
                     for test in to_delete:
                         os.remove(os.path.join(os.getcwd(), "in", test))
 
@@ -61,11 +64,10 @@ def run(self, args: argparse.Namespace):
         self.args = args
 
         self.task_id = package_util.get_task_id()
-        package_util.validate_test_names(self.task_id)
         util.change_stack_size_to_unlimited()
         self.ingen = get_ingen(self.task_id, args.ingen_path)
-        print(util.info(f'Using ingen file {os.path.basename(self.ingen)}'))
-        self.ingen_exe = compile_ingen(self.ingen, self.args, self.args.compile_mode)
+        print(f'Using ingen file {os.path.basename(self.ingen)}')
+        self.ingen_exe = compile_ingen(self.ingen, self.args, self.args.compile_mode, self.args.fsanitize)
 
         previous_tests = []
         try:
@@ -89,8 +91,5 @@ def run(self, args: argparse.Namespace):
             f.write("\n".join(glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in"))))
 
         if not self.args.no_validate:
-            print(util.info('Validating input test contents.'))
             tests = sorted(glob.glob(os.path.join(os.getcwd(), "in", f"{self.task_id}*.in")))
-            for test in tests:
-                package_util.validate_test(test)
-            print(util.info('Input test contents are valid!'))
+            package_util.validate_tests(tests, self.args.cpus, 'input')

src/sinol_make/commands/ingen/ingen_util.py

@@ -47,7 +47,7 @@ def get_ingen(task_id, ingen_path=None):
     return correct_ingen
 
 
-def compile_ingen(ingen_path: str, args: argparse.Namespace, compilation_flags='default'):
+def compile_ingen(ingen_path: str, args: argparse.Namespace, compilation_flags='default', use_fsanitize=False):
     """
     Compiles ingen and returns path to compiled executable.
     If ingen_path is shell script, then it will be returned.
@@ -57,7 +57,7 @@ def compile_ingen(ingen_path: str, args: argparse.Namespace, compilation_flags='
 
     compilers = compiler.verify_compilers(args, [ingen_path])
     ingen_exe, compile_log_path = compile.compile_file(ingen_path, package_util.get_executable(ingen_path),
-                                                       compilers, compilation_flags, use_fsanitize=True,
+                                                       compilers, compilation_flags, use_fsanitize=use_fsanitize,
                                                        additional_flags='-D_INGEN')
 
     if ingen_exe is None:
@@ -86,11 +86,21 @@ def run_ingen(ingen_exe, working_dir=None):
     print(util.bold(' Ingen output '.center(util.get_terminal_size()[1], '=')))
     process = subprocess.Popen([ingen_exe], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                                cwd=working_dir, shell=is_shell)
+    whole_output = ''
     while process.poll() is None:
-        print(process.stdout.readline().decode('utf-8'), end='')
-
-    print(process.stdout.read().decode('utf-8'), end='')
+        out = process.stdout.readline().decode('utf-8')
+        if out != '':
+            print(out, end='')
+            whole_output += out
+    out = process.stdout.read().decode('utf-8')
+    whole_output += out
+    print(out, end='')
     exit_code = process.returncode
     print(util.bold(' End of ingen output '.center(util.get_terminal_size()[1], '=')))
 
+    if util.has_sanitizer_error(whole_output, exit_code):
+        print(util.warning('Warning: if ingen failed due to sanitizer errors, you can either run '
+                           '`sudo sysctl vm.mmap_rnd_bits = 28` to fix this or disable sanitizers with the '
+                           '--no-fsanitize flag.'))
+
     return exit_code == 0

src/sinol_make/commands/init/__init__.py

@@ -23,6 +23,7 @@ def configure_subparser(self, subparser: argparse.ArgumentParser):
             description='Create package from predefined template with given id.'
         )
         parser.add_argument('task_id', type=str, help='Id of the task to create')
+        return parser
 
     def download_template(self):
         repo = 'https://github.com/sio2project/sinol-make.git'
@@ -69,8 +70,7 @@ def run(self, args: argparse.Namespace):
 
         self.move_folder()
         self.update_config()
-        
+
         self.used_tmpdir.cleanup()
 
         print(util.info(f'Successfully created task "{self.task_id}"'))
-

src/sinol_make/commands/inwer/__init__.py

@@ -10,8 +10,7 @@
 
 from sinol_make import util
 from sinol_make.structs.inwer_structs import TestResult, InwerExecution, VerificationResult, TableData
-from sinol_make.helpers import package_util, compile, printer, paths
-from sinol_make.helpers.parsers import add_compilation_arguments
+from sinol_make.helpers import package_util, printer, paths, parsers
 from sinol_make.interfaces.BaseCommand import BaseCommand
 from sinol_make.commands.inwer import inwer_util
 
@@ -37,9 +36,10 @@ def configure_subparser(self, subparser: argparse.ArgumentParser):
                             help='path to inwer source file, for example prog/abcinwer.cpp')
         parser.add_argument('-t', '--tests', type=str, nargs='+',
                             help='test to verify, for example in/abc{0,1}*')
-        parser.add_argument('-c', '--cpus', type=int,
-                            help=f'number of cpus to use (default: {util.default_cpu_count()})')
-        add_compilation_arguments(parser)
+        parsers.add_cpus_argument(parser, 'number of cpus to use when verifying tests')
+        parsers.add_fsanitize_argument(parser)
+        parsers.add_compilation_arguments(parser)
+        return parser
 
     @staticmethod
     def verify_test(execution: InwerExecution) -> VerificationResult:
@@ -94,11 +94,14 @@ def verify_and_print_table(self) -> Dict[str, TestResult]:
             thr.start()
 
         keyboard_interrupt = False
+        sanitizer_error = False
         try:
             with mp.Pool(self.cpus) as pool:
                 for i, result in enumerate(pool.imap(self.verify_test, executions)):
                     table_data.results[result.test_path].set_results(result.valid, result.output)
                     table_data.i = i
+                    if util.has_sanitizer_error(result.output, 0 if result.valid else 1):
+                        sanitizer_error = True
         except KeyboardInterrupt:
             keyboard_interrupt = True
 
@@ -108,6 +111,10 @@ def verify_and_print_table(self) -> Dict[str, TestResult]:
 
         print("\n".join(inwer_util.print_view(terminal_width, terminal_height, table_data)[0]))
 
+        if sanitizer_error:
+            print(util.warning('Warning: if inwer failed due to sanitizer errors, you can either run '
+                               '`sudo sysctl vm.mmap_rnd_bits = 28` to fix this or disable sanitizers with the '
+                               '--no-fsanitize flag.'))
         if keyboard_interrupt:
             util.exit_with_error('Keyboard interrupt.')
 
@@ -203,7 +210,7 @@ def run(self, args: argparse.Namespace):
             print('Verifying tests: ' + util.bold(', '.join(self.tests)))
 
         util.change_stack_size_to_unlimited()
-        self.inwer_executable = inwer_util.compile_inwer(self.inwer, args, args.compile_mode)
+        self.inwer_executable = inwer_util.compile_inwer(self.inwer, args, args.compile_mode, args.fsanitize)
         results: Dict[str, TestResult] = self.verify_and_print_table()
         print('')
 
@@ -218,4 +225,3 @@ def run(self, args: argparse.Namespace):
             print("Verifying tests order...")
             self.verify_tests_order()
             print(util.info('Verification successful.'))
-            exit(0)