Improve query suite test based on feedback

Author: tamasvajk

java/ql/integration-tests/java/query-suite/test.py

@@ -1,23 +1,14 @@
 import runs_on
 import pytest
-import sys
-
-def get_test_module(semmle_code_dir):
-    import importlib.util
-    spec = importlib.util.spec_from_file_location('test-module', semmle_code_dir / 'ql' / 'misc' / 'pytest' / 'lib' / 'query-suite-test.py')
-    mod = importlib.util.module_from_spec(spec)
-    sys.modules["test-module"] = mod
-    spec.loader.exec_module(mod)
-    return mod
+from query_suites import *
 
 well_known_query_suites = ['java-code-quality.qls', 'java-security-and-quality.qls', 'java-security-extended.qls', 'java-code-scanning.qls']
 
 @runs_on.posix
 @pytest.mark.parametrize("query_suite", well_known_query_suites)
-def test(codeql, java, cwd, expected_files, semmle_code_dir, query_suite):
-    get_test_module(semmle_code_dir).test(codeql, cwd, expected_files, semmle_code_dir, query_suite)
-
+def test(codeql, java, check_query_suite, query_suite):
+    check_query_suite(query_suite)
 
 @runs_on.posix
-def test_not_included_queries(codeql, java, cwd, expected_files, semmle_code_dir):
-    get_test_module(semmle_code_dir).test_not_included_queries(codeql, 'java', cwd, expected_files, semmle_code_dir, well_known_query_suites)
+def test_not_included_queries(codeql, java, check_queries_not_included):
+    check_queries_not_included('java', well_known_query_suites)

misc/pytest/lib/query-suite-test.py

@@ -0,0 +1,30 @@
+
+import os
+import pytest
+
+@pytest.fixture
+def check_query_suite(codeql, cwd, expected_files, semmle_code_dir):
+    def ret(query_suite):
+        actual = codeql.resolve.queries(query_suite, _capture=True).strip()
+        actual = sorted(actual.splitlines())
+        actual = [os.path.relpath(q, semmle_code_dir) for q in actual]
+        actual_file_name = query_suite + '.actual'
+        expected_files.add(actual_file_name)
+        (cwd / actual_file_name).write_text('\n'.join(actual)+'\n')
+    return ret
+
+@pytest.fixture
+def check_queries_not_included(codeql, cwd, expected_files, semmle_code_dir):
+    def ret(lang_folder_name, query_suites):
+        all_queries = codeql.resolve.queries(semmle_code_dir / 'ql' / lang_folder_name / 'ql' / 'src', _capture=True).strip().splitlines()
+
+        included_in_qls = set()
+        for query_suite in query_suites:
+            included_in_qls |= set(codeql.resolve.queries(query_suite, _capture=True).strip().splitlines())
+
+        not_included = sorted(set(all_queries) - included_in_qls)
+        not_included = [os.path.relpath(q, semmle_code_dir) for q in not_included]
+        not_included_file_name = 'not_included_in_qls.actual'
+        expected_files.add(not_included_file_name)
+        (cwd / not_included_file_name).write_text('\n'.join(not_included)+'\n')
+    return ret

misc/pytest/lib/query_suites.py

@@ -1,23 +1,14 @@
 import runs_on
 import pytest
-import sys
-
-def get_test_module(semmle_code_dir):
-    import importlib.util
-    spec = importlib.util.spec_from_file_location('test-module', semmle_code_dir / 'ql' / 'misc' / 'pytest' / 'lib' / 'query-suite-test.py')
-    mod = importlib.util.module_from_spec(spec)
-    sys.modules["test-module"] = mod
-    spec.loader.exec_module(mod)
-    return mod
-
+from query_suites import *
 
 well_known_query_suites = ['swift-code-quality.qls', 'swift-security-and-quality.qls', 'swift-security-extended.qls', 'swift-code-scanning.qls']
 
 @runs_on.posix
 @pytest.mark.parametrize("query_suite", well_known_query_suites)
-def test(codeql, swift, cwd, expected_files, semmle_code_dir, query_suite):
-    get_test_module(semmle_code_dir).test(codeql, cwd, expected_files, semmle_code_dir, query_suite)
+def test(codeql, swift, check_query_suite, query_suite):
+    check_query_suite(query_suite)
 
 @runs_on.posix
-def test_not_included_queries(codeql, swift, cwd, expected_files, semmle_code_dir):
-    get_test_module(semmle_code_dir).test_not_included_queries(codeql, 'swift', cwd, expected_files, semmle_code_dir, well_known_query_suites)
+def test_not_included_queries(codeql, swift, check_queries_not_included):
+    check_queries_not_included('swift', well_known_query_suites)