Update for new resolver

Author: mxsasha

checks/http_client.py

@@ -10,8 +10,6 @@
 from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter
 
 from checks.resolver import dns_resolve_aaaa, dns_resolve_a
-from checks.tasks.tls_connection import DEFAULT_TIMEOUT
-from checks.tasks.tls_connection_exceptions import NoIpError
 from django.conf import settings
 from internetnl import log
 

checks/tasks/tls.py

@@ -656,7 +656,6 @@ def save_results(model, results, addr, domain, category):
                 model.cert_hostmatch_score = result.get("hostmatch_score")
                 model.cert_hostmatch_bad = result.get("hostmatch_bad")
                 model.caa_enabled = result.get("caa_result").caa_found
-                model.caa_records = result.get("caa_result").caa_records_str
                 model.caa_error = [ttti.to_dict() for ttti in result.get("caa_result").errors]
                 model.caa_recommendations = [ttti.to_dict() for ttti in result.get("caa_result").recommendations]
                 model.caa_score = result.get("caa_result").score
@@ -730,7 +729,6 @@ def save_results(model, results, addr, domain, category):
                     model.cert_hostmatch_score = result.get("hostmatch_score")
                     model.cert_hostmatch_bad = result.get("hostmatch_bad")
                     model.caa_enabled = result.get("caa_result").caa_found
-                    model.caa_records = result.get("caa_result").caa_records_str
                     model.caa_error = [ttti.to_dict() for ttti in result.get("caa_result").errors]
                     model.caa_recommendations = [ttti.to_dict() for ttti in result.get("caa_result").recommendations]
                     model.caa_score = result.get("caa_result").score
@@ -903,15 +901,10 @@ def annotate_and_combine_all(good_items, sufficient_items, bad_items, phaseout_i
                 caa_tech_table = caa_host_message + dttls.caa_errors + dttls.caa_recommendations
                 for record in dttls.caa_records:
                     caa_tech_table.append(
-                        TranslatableTechTableItem(msgid="caa-record", context={"record": record}).to_dict()
+                        TranslatableTechTableItem(msgid="caa_record", context={"record": record}).to_dict()
                     )
                 if not dttls.caa_enabled:
                     category.subtests["web_caa"].result_bad(caa_tech_table)
-                elif dttls.caa_errors:
-                    if all([ttti.msgid != CAA_MSGID_INSUFFICIENT_POLICY for ttti in dttls.caa_errors]):
-                        category.subtests["web_caa"].result_syntax_error(caa_tech_table)
-                    else:
-                        category.subtests["web_caa"].result_insufficient(caa_tech_table)
                 elif dttls.caa_recommendations:
                     category.subtests["web_caa"].result_recommendations(caa_tech_table)
                 else:
@@ -1082,15 +1075,10 @@ def annotate_and_combine_all(good_items, sufficient_items, bad_items, phaseout_i
             caa_tech_table = caa_host_message + dttls.caa_errors + dttls.caa_recommendations
             for record in dttls.caa_records:
                 caa_tech_table.append(
-                    TranslatableTechTableItem(msgid="caa-record", context={"record": record}).to_dict()
+                    TranslatableTechTableItem(msgid="caa_record", context={"record": record}).to_dict()
                 )
             if not dttls.caa_enabled:
                 category.subtests["mail_caa"].result_bad(caa_tech_table)
-            elif dttls.caa_errors:
-                if all([error["msgid"] != CAA_MSGID_INSUFFICIENT_POLICY for error in dttls.caa_errors]):
-                    category.subtests["mail_caa"].result_syntax_error(caa_tech_table)
-                else:
-                    category.subtests["mail_caa"].result_insufficient(caa_tech_table)
             elif dttls.caa_recommendations:
                 category.subtests["mail_caa"].result_recommendations(caa_tech_table)
             else:

checks/tasks/tls/http.py

@@ -12,16 +12,16 @@
 )
 
 
-def http_checks(af_ip_pair, url, task):
+def http_checks(af_ip_pair, url):
     """
     Perform the HTTP header and HTTPS redirection checks for this webserver.
     """
-    forced_https_score, forced_https = forced_http_check(af_ip_pair, url, task)
+    forced_https_score, forced_https = forced_http_check(af_ip_pair, url)
     header_checkers = [
         HeaderCheckerContentEncoding(),
         HeaderCheckerStrictTransportSecurity(),
     ]
-    header_results = http_headers_check(af_ip_pair, url, header_checkers, task)
+    header_results = http_headers_check(af_ip_pair, url, header_checkers)
     results = {
         "forced_https": forced_https,
         "forced_https_score": forced_https_score,
@@ -30,7 +30,7 @@ def http_checks(af_ip_pair, url, task):
     return results
 
 
-def forced_http_check(af_ip_pair, url, task):
+def forced_http_check(af_ip_pair, url):
     """
     Check if the webserver is properly configured with HTTPS redirection.
     """

checks/tasks/tls/scans.py

@@ -49,6 +49,7 @@
 from sslyze.server_connectivity import ServerConnectivityInfo
 
 from checks import scoring
+from checks.caa.retrieval import retrieve_parse_caa
 from checks.models import (
     DaneStatus,
     ZeroRttStatus,
@@ -119,8 +120,6 @@ def dane(
     url: str,
     port: int,
     chain: List[Certificate],
-    task,
-    dane_cb_data,
     score_none: scoring.Score,
     score_none_bogus: scoring.Score,
     score_failed: scoring.Score,
@@ -139,7 +138,7 @@ def dane(
 
     continue_testing = False
 
-    cb_data = dane_cb_data or resolve_dane(task, port, url)
+    cb_data = resolve_dane(port, url)
 
     # Check if there is a TLSA record, if TLSA records are bogus or NXDOMAIN is
     # returned for the TLSA domain (faulty signer).
@@ -149,7 +148,7 @@ def dane(
     elif cb_data.get("data") and cb_data.get("secure"):
         # If there is a secure TLSA record check for the existence of
         # possible bogus (unsigned) NXDOMAIN in A.
-        tmp_data = resolve_dane(task, port, url, check_nxdomain=True)
+        tmp_data = resolve_dane(port, url, check_nxdomain=True)
         if tmp_data.get("nxdomain") and tmp_data.get("bogus"):
             status = DaneStatus.none_bogus
             score = score_none_bogus
@@ -280,7 +279,7 @@ def get_common_name(cert: Certificate) -> str:
     return value
 
 
-def cert_checks(hostname: str, mode: ChecksMode, task, af_ip_pair=None, dane_cb_data=None, *args, **kwargs):
+def cert_checks(hostname: str, mode: ChecksMode, af_ip_pair=None, *args, **kwargs):
     """
     Perform certificate checks, such as trust, name match. Also scans the server.
     """
@@ -384,14 +383,14 @@ def cert_checks(hostname: str, mode: ChecksMode, task, af_ip_pair=None, dane_cb_
         hostname,
         port,
         cert_deployment.received_certificate_chain,
-        task,
-        dane_cb_data,
         scoring.WEB_TLS_DANE_NONE,
         scoring.WEB_TLS_DANE_NONE_BOGUS,
         scoring.WEB_TLS_DANE_FAILED,
         scoring.WEB_TLS_DANE_VALIDATED,
     )
 
+    caa_result = retrieve_parse_caa(hostname)
+
     results = dict(
         tls_cert=True,
         chain=chain_str,
@@ -407,6 +406,7 @@ def cert_checks(hostname: str, mode: ChecksMode, task, af_ip_pair=None, dane_cb_
         sigalg_score=sigalg_score,
         hostmatch_bad=hostmatch_bad,
         hostmatch_score=hostmatch_score,
+        caa_result=caa_result,
     )
     results.update(dane_results)
 
@@ -484,7 +484,7 @@ def check_pubkey(certificates: List[Certificate], mode: ChecksMode):
     return pubkey_score, bad_pubkey, phase_out_pubkey
 
 
-def check_mail_tls_multiple(server_tuples, task) -> Dict[str, Dict[str, Any]]:
+def check_mail_tls_multiple(server_tuples) -> Dict[str, Dict[str, Any]]:
     """
     Perform sslyze probing on all mail servers, in parallel.
     """
@@ -518,7 +518,7 @@ def check_mail_tls_multiple(server_tuples, task) -> Dict[str, Dict[str, Any]]:
             continue
         log.debug(f"sslyze mail scan complete for {result.server_location.hostname}, other scans may be pending")
         dane_cb_data = dane_cb_per_server[result.server_location.hostname]
-        results[result.server_location.hostname] = check_mail_tls(result, all_suites, dane_cb_data, task)
+        results[result.server_location.hostname] = check_mail_tls(result, all_suites, dane_cb_data)
         log.debug(f"check_mail_tls complete for {result.server_location.hostname}")
     return results
 
@@ -576,7 +576,7 @@ def _generate_mail_server_scan_request(mx_hostname: str) -> Optional[ServerScanR
     )
 
 
-def check_mail_tls(result: ServerScanResult, all_suites: List[CipherSuitesScanAttempt], dane_cb_data, task):
+def check_mail_tls(result: ServerScanResult, all_suites: List[CipherSuitesScanAttempt], dane_cb_data):
     """
     Perform evaluation and additional probes for a single mail server.
     This happens after sslyze has already been run on it.
@@ -597,7 +597,7 @@ def check_mail_tls(result: ServerScanResult, all_suites: List[CipherSuitesScanAt
         prots_accepted,
         cipher_evaluation,
     )
-    cert_results = cert_checks(result.server_location.hostname, ChecksMode.MAIL, task, dane_cb_data)
+    cert_results = cert_checks(result.server_location.hostname, ChecksMode.MAIL, dane_cb_data)
 
     # HACK for DANE-TA(2) and hostname mismatch!
     # Give a good hosmatch score if DANE-TA *is not* present.

checks/tasks/tls/tasks_reports.py

@@ -22,7 +22,6 @@
     ZeroRttStatus,
     WebTestTls,
 )
-from checks.tasks import SetupUnboundContext
 from checks.tasks.dispatcher import check_registry, post_callback_hook
 from checks.tasks.shared import (
     aggregate_subreports,
@@ -32,6 +31,7 @@
     mail_get_servers,
     resolve_a_aaaa,
     results_per_domain,
+    TranslatableTechTableItem,
 )
 from checks.tasks.tls.http import http_checks
 from interface import batch, batch_shared_task, redis_id
@@ -158,7 +158,6 @@ def callback_null_mx(results, domain, test_type):
     bind=True,
     soft_time_limit=settings.SHARED_TASK_SOFT_TIME_LIMIT_HIGH,
     time_limit=settings.SHARED_TASK_TIME_LIMIT_HIGH,
-    base=SetupUnboundContext,
 )
 def web_cert(self, af_ip_pairs, url, *args, **kwargs):
     return do_web_cert(af_ip_pairs, url, self, *args, **kwargs)
@@ -169,7 +168,6 @@ def web_cert(self, af_ip_pairs, url, *args, **kwargs):
     bind=True,
     soft_time_limit=settings.BATCH_SHARED_TASK_SOFT_TIME_LIMIT_HIGH,
     time_limit=settings.BATCH_SHARED_TASK_TIME_LIMIT_HIGH,
-    base=SetupUnboundContext,
 )
 def batch_web_cert(self, af_ip_pairs, url, *args, **kwargs):
     return do_web_cert(af_ip_pairs, url, self, *args, **kwargs)
@@ -180,7 +178,6 @@ def batch_web_cert(self, af_ip_pairs, url, *args, **kwargs):
     bind=True,
     soft_time_limit=settings.SHARED_TASK_SOFT_TIME_LIMIT_HIGH,
     time_limit=settings.SHARED_TASK_TIME_LIMIT_HIGH,
-    base=SetupUnboundContext,
 )
 def web_conn(self, af_ip_pairs, url, *args, **kwargs):
     return do_web_conn(af_ip_pairs, url, *args, **kwargs)
@@ -191,7 +188,6 @@ def web_conn(self, af_ip_pairs, url, *args, **kwargs):
     bind=True,
     soft_time_limit=settings.BATCH_SHARED_TASK_SOFT_TIME_LIMIT_HIGH,
     time_limit=settings.BATCH_SHARED_TASK_TIME_LIMIT_HIGH,
-    base=SetupUnboundContext,
 )
 def batch_web_conn(self, af_ip_pairs, url, *args, **kwargs):
     return do_web_conn(af_ip_pairs, url, *args, **kwargs)
@@ -202,7 +198,6 @@ def batch_web_conn(self, af_ip_pairs, url, *args, **kwargs):
     bind=True,
     soft_time_limit=settings.SHARED_TASK_SOFT_TIME_LIMIT_HIGH,
     time_limit=settings.SHARED_TASK_TIME_LIMIT_HIGH,
-    base=SetupUnboundContext,
 )
 def mail_smtp_starttls(self, mailservers, url, *args, **kwargs):
     return do_mail_smtp_starttls(mailservers, url, self, *args, **kwargs)
@@ -213,7 +208,6 @@ def mail_smtp_starttls(self, mailservers, url, *args, **kwargs):
     bind=True,
     soft_time_limit=settings.BATCH_SHARED_TASK_SOFT_TIME_LIMIT_HIGH,
     time_limit=settings.BATCH_SHARED_TASK_TIME_LIMIT_HIGH,
-    base=SetupUnboundContext,
 )
 def batch_mail_smtp_starttls(self, mailservers, url, *args, **kwargs):
     return do_mail_smtp_starttls(mailservers, url, self, *args, **kwargs)
@@ -224,7 +218,6 @@ def batch_mail_smtp_starttls(self, mailservers, url, *args, **kwargs):
     bind=True,
     soft_time_limit=settings.SHARED_TASK_SOFT_TIME_LIMIT_HIGH,
     time_limit=settings.SHARED_TASK_TIME_LIMIT_HIGH,
-    base=SetupUnboundContext,
 )
 def web_http(self, af_ip_pairs, url, *args, **kwargs):
     return do_web_http(af_ip_pairs, url, self, *args, **kwargs)
@@ -235,7 +228,6 @@ def web_http(self, af_ip_pairs, url, *args, **kwargs):
     bind=True,
     soft_time_limit=settings.BATCH_SHARED_TASK_SOFT_TIME_LIMIT_HIGH,
     time_limit=settings.BATCH_SHARED_TASK_TIME_LIMIT_HIGH,
-    base=SetupUnboundContext,
 )
 def batch_web_http(self, af_ip_pairs, url, *args, **kwargs):
     return do_web_http(af_ip_pairs, url, self, args, **kwargs)
@@ -293,6 +285,11 @@ def save_results(model, results, addr, domain, category):
                 model.cert_signature_score = result.get("sigalg_score")
                 model.cert_hostmatch_score = result.get("hostmatch_score")
                 model.cert_hostmatch_bad = result.get("hostmatch_bad")
+                model.caa_enabled = result.get("caa_result").caa_found
+                model.caa_error = [ttti.to_dict() for ttti in result.get("caa_result").errors]
+                model.caa_recommendations = [ttti.to_dict() for ttti in result.get("caa_result").recommendations]
+                model.caa_score = result.get("caa_result").score
+                model.caa_found_on_domain = result.get("caa_result").canonical_name
                 model.dane_log = result.get("dane_log")
                 model.dane_score = result.get("dane_score")
                 model.dane_status = result.get("dane_status")
@@ -361,6 +358,11 @@ def save_results(model, results, addr, domain, category):
                     model.cert_signature_score = result.get("sigalg_score")
                     model.cert_hostmatch_score = result.get("hostmatch_score")
                     model.cert_hostmatch_bad = result.get("hostmatch_bad")
+                    model.caa_enabled = result.get("caa_result").caa_found
+                    model.caa_error = [ttti.to_dict() for ttti in result.get("caa_result").errors]
+                    model.caa_recommendations = [ttti.to_dict() for ttti in result.get("caa_result").recommendations]
+                    model.caa_score = result.get("caa_result").score
+                    model.caa_found_on_domain = result.get("caa_result").canonical_name
                     model.dane_log = result.get("dane_log")
                     model.dane_score = result.get("dane_score")
                     model.dane_status = result.get("dane_status")
@@ -500,6 +502,22 @@ def annotate_and_combine_all(good_items, sufficient_items, bad_items, phaseout_i
                 else:
                     category.subtests["cert_hostmatch"].result_good()
 
+                if dttls.caa_enabled:
+                    caa_host_message = [
+                        TranslatableTechTableItem(
+                            msgid="found_host", context={"host": dttls.caa_found_on_domain}
+                        ).to_dict()
+                    ]
+                else:
+                    caa_host_message = [TranslatableTechTableItem(msgid="not_found").to_dict()]
+                caa_tech_table = caa_host_message + dttls.caa_errors + dttls.caa_recommendations
+                if not dttls.caa_enabled or dttls.caa_errors:
+                    category.subtests["web_caa"].result_bad(caa_tech_table)
+                elif dttls.caa_recommendations:
+                    category.subtests["web_caa"].result_recommendations(caa_tech_table)
+                else:
+                    category.subtests["web_caa"].result_good(caa_tech_table)
+
             if dttls.dane_status == DaneStatus.none:
                 category.subtests["dane_exists"].result_bad()
             elif dttls.dane_status == DaneStatus.none_bogus:
@@ -637,6 +655,20 @@ def annotate_and_combine_all(good_items, sufficient_items, bad_items, phaseout_i
                 else:
                     category.subtests["cert_hostmatch"].result_good()
 
+            if dttls.caa_enabled:
+                caa_host_message = [
+                    TranslatableTechTableItem(msgid="found_host", context={"host": dttls.caa_found_on_domain}).to_dict()
+                ]
+            else:
+                caa_host_message = [TranslatableTechTableItem(msgid="not_found").to_dict()]
+            caa_tech_table = caa_host_message + dttls.caa_errors + dttls.caa_recommendations
+            if not dttls.caa_enabled or dttls.caa_errors:
+                category.subtests["mail_caa"].result_bad(caa_tech_table)
+            elif dttls.caa_recommendations:
+                category.subtests["mail_caa"].result_recommendations(caa_tech_table)
+            else:
+                category.subtests["mail_caa"].result_good(caa_tech_table)
+
             if dttls.dane_status == DaneStatus.none:
                 category.subtests["dane_exists"].result_bad()
             elif dttls.dane_status == DaneStatus.none_bogus:
@@ -711,15 +743,15 @@ def build_summary_report(testtls, category):
     testtls.report = report
 
 
-def do_web_cert(af_ip_pairs, url, task, *args, **kwargs):
+def do_web_cert(af_ip_pairs, url, *args, **kwargs):
     """
     Check the web server's certificate.
 
     """
     try:
         results = {}
         for af_ip_pair in af_ip_pairs:
-            results[af_ip_pair[1]] = cert_checks(url, ChecksMode.WEB, task, af_ip_pair, *args, **kwargs)
+            results[af_ip_pair[1]] = cert_checks(url, ChecksMode.WEB, af_ip_pair, *args, **kwargs)
     except SoftTimeLimitExceeded:
         log.debug("Soft time limit exceeded. Url: %s", url)
         for af_ip_pair in af_ip_pairs:
@@ -747,7 +779,7 @@ def do_web_conn(af_ip_pairs, url, *args, **kwargs):
     return ("tls_conn", results)
 
 
-def do_mail_smtp_starttls(mailservers, url, task, *args, **kwargs):
+def do_mail_smtp_starttls(mailservers, url, *args, **kwargs):
     """
     Start all the TLS related checks for the mail test.
 
@@ -782,7 +814,7 @@ def do_mail_smtp_starttls(mailservers, url, task, *args, **kwargs):
                 (server, dane_cb_data) for server, dane_cb_data, _ in mailservers if not results[server]
             ]
             log.debug(f"=========== checking remaining {servers_to_check=}")
-            results.update(check_mail_tls_multiple(servers_to_check, task))
+            results.update(check_mail_tls_multiple(servers_to_check))
             time.sleep(1)
         for server, server_result in results.items():
             cache_id = redis_id.mail_starttls.id.format(server)
@@ -798,15 +830,15 @@ def do_mail_smtp_starttls(mailservers, url, task, *args, **kwargs):
     return "smtp_starttls", results
 
 
-def do_web_http(af_ip_pairs, url, task, *args, **kwargs):
+def do_web_http(af_ip_pairs, url, *args, **kwargs):
     """
     Start all the HTTP related checks for the web test.
 
     """
     try:
         results = {}
         for af_ip_pair in af_ip_pairs:
-            results[af_ip_pair[1]] = http_checks(af_ip_pair, url, task)
+            results[af_ip_pair[1]] = http_checks(af_ip_pair, url)
 
     except SoftTimeLimitExceeded:
         log.debug("Soft time limit exceeded.")