From a432e0bb60a5f13e3d0ee8ba0001a460f3306e3d Mon Sep 17 00:00:00 2001 From: Adam <1392689+coffeegist@users.noreply.github.com> Date: Thu, 7 Jan 2021 21:05:35 -0600 Subject: [PATCH 1/4] * Add route53 configuration to command center --- ghostwriter/commandcenter/admin.py | 2 ++ .../migrations/0006_route53configuration.py | 25 +++++++++++++++++++ ghostwriter/commandcenter/models.py | 16 ++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 ghostwriter/commandcenter/migrations/0006_route53configuration.py diff --git a/ghostwriter/commandcenter/admin.py b/ghostwriter/commandcenter/admin.py index 5cf2abebe..d33cb33c1 100644 --- a/ghostwriter/commandcenter/admin.py +++ b/ghostwriter/commandcenter/admin.py @@ -14,6 +14,7 @@ ReportConfiguration, SlackConfiguration, VirusTotalConfiguration, + Route53Configuration, ) admin.site.register(CloudServicesConfiguration, SingletonModelAdmin) @@ -21,6 +22,7 @@ admin.site.register(NamecheapConfiguration, SingletonModelAdmin) admin.site.register(SlackConfiguration, SingletonModelAdmin) admin.site.register(VirusTotalConfiguration, SingletonModelAdmin) +admin.site.register(Route53Configuration, SingletonModelAdmin) class ReportConfigurationAdmin(SingletonModelAdmin): diff --git a/ghostwriter/commandcenter/migrations/0006_route53configuration.py b/ghostwriter/commandcenter/migrations/0006_route53configuration.py new file mode 100644 index 000000000..771fbe2d0 --- /dev/null +++ b/ghostwriter/commandcenter/migrations/0006_route53configuration.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.10 on 2021-01-08 03:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('commandcenter', '0005_auto_20201102_2207'), + ] + + operations = [ + migrations.CreateModel( + name='Route53Configuration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enable', models.BooleanField(default=False)), + ('access_key', models.CharField(default='Route53 Access Key', max_length=255)), + ('secret_access_key', models.CharField(default='Route53 Secret Access Key', max_length=255)), + ], + options={ + 'verbose_name': 'Route53 Configuration', + }, + ), + ] diff --git a/ghostwriter/commandcenter/models.py b/ghostwriter/commandcenter/models.py index 1e4ddf871..521c9edbb 100644 --- a/ghostwriter/commandcenter/models.py +++ b/ghostwriter/commandcenter/models.py @@ -57,6 +57,22 @@ def sanitized_api_key(self): return sanitize(self.api_key) +class Route53Configuration(SingletonModel): + enable = models.BooleanField(default=False) + access_key = models.CharField(max_length=255, default="Route53 Access Key") + secret_access_key = models.CharField(max_length=255, default="Route53 Secret Access Key") + + def __str__(self): + return "Route53 Configuration" + + class Meta: + verbose_name = "Route53 Configuration" + + @property + def sanitized_secret_access_key(self): + return sanitize(self.secret_access_key) + + class ReportConfiguration(SingletonModel): enable_borders = models.BooleanField( default=False, help_text="Enable borders around images in Word documents" From 4be8d496c47eb41cc31d51e18958412102750b1c Mon Sep 17 00:00:00 2001 From: Adam <1392689+coffeegist@users.noreply.github.com> Date: Wed, 13 Jan 2021 17:11:44 -0600 Subject: [PATCH 2/4] * Add Route53 update task --- ghostwriter/shepherd/tasks.py | 331 ++++++++++++++++++ .../shepherd/templates/shepherd/update.html | 37 ++ ghostwriter/shepherd/urls.py | 5 + ghostwriter/shepherd/views.py | 68 ++++ 4 files changed, 441 insertions(+) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 1f47bb0cf..30374c7ef 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -1030,6 +1030,337 @@ def fetch_namecheap_domains(): return domain_changes +def fetch_route53_domains(): + """ + Fetch a list of registered domains for the configured AWS account. A valid set of AWS + API credentials must be used. Returns a dictionary containing errors and each domain + name paired with change status. + + Result statuses: created, updated, burned, updated & burned + + The returned JSON contains entries for domains like this: + + { + 'Domains': [ + { + 'DomainName': 'string', + 'AutoRenew': True|False, + 'TransferLock': True|False, + 'Expiry': datetime(2015, 1, 1), + } + ], + 'NextPageMarker': 'string', + 'ResponseMetadata': { + 'RequestId': 'string', + 'HTTPStatusCode': int, + 'HTTPHeaders': { + 'x-amzn-requestid': 'string', + 'content-type': 'string', + 'content-length': 'string', + 'date': 'string' + }, + 'RetryAttempts': int + } + } + + The returned JSON for domain details contains entries for domains like this: + + { + 'DomainName': 'string', + 'Nameservers': [ + { + 'Name': 'string', + 'GlueIps': [ + 'string', + ] + }, + ], + 'AutoRenew': True|False, + 'AdminContact': { + 'FirstName': 'string', + 'LastName': 'string', + 'ContactType': 'PERSON'|'COMPANY'|'ASSOCIATION'|'PUBLIC_BODY'|'RESELLER', + 'OrganizationName': 'string', + 'AddressLine1': 'string', + 'AddressLine2': 'string', + 'City': 'string', + 'State': 'string', + 'CountryCode': 'AD'|'AE'|'AF'|'AG'|'AI'|'AL'|'AM'|'AN'|'AO'|'AQ'|'AR'|'AS'|'AT'|'AU'|'AW'|'AZ'|'BA'|'BB'|'BD'|'BE'|'BF'|'BG'|'BH'|'BI'|'BJ'|'BL'|'BM'|'BN'|'BO'|'BR'|'BS'|'BT'|'BW'|'BY'|'BZ'|'CA'|'CC'|'CD'|'CF'|'CG'|'CH'|'CI'|'CK'|'CL'|'CM'|'CN'|'CO'|'CR'|'CU'|'CV'|'CX'|'CY'|'CZ'|'DE'|'DJ'|'DK'|'DM'|'DO'|'DZ'|'EC'|'EE'|'EG'|'ER'|'ES'|'ET'|'FI'|'FJ'|'FK'|'FM'|'FO'|'FR'|'GA'|'GB'|'GD'|'GE'|'GH'|'GI'|'GL'|'GM'|'GN'|'GQ'|'GR'|'GT'|'GU'|'GW'|'GY'|'HK'|'HN'|'HR'|'HT'|'HU'|'ID'|'IE'|'IL'|'IM'|'IN'|'IQ'|'IR'|'IS'|'IT'|'JM'|'JO'|'JP'|'KE'|'KG'|'KH'|'KI'|'KM'|'KN'|'KP'|'KR'|'KW'|'KY'|'KZ'|'LA'|'LB'|'LC'|'LI'|'LK'|'LR'|'LS'|'LT'|'LU'|'LV'|'LY'|'MA'|'MC'|'MD'|'ME'|'MF'|'MG'|'MH'|'MK'|'ML'|'MM'|'MN'|'MO'|'MP'|'MR'|'MS'|'MT'|'MU'|'MV'|'MW'|'MX'|'MY'|'MZ'|'NA'|'NC'|'NE'|'NG'|'NI'|'NL'|'NO'|'NP'|'NR'|'NU'|'NZ'|'OM'|'PA'|'PE'|'PF'|'PG'|'PH'|'PK'|'PL'|'PM'|'PN'|'PR'|'PT'|'PW'|'PY'|'QA'|'RO'|'RS'|'RU'|'RW'|'SA'|'SB'|'SC'|'SD'|'SE'|'SG'|'SH'|'SI'|'SK'|'SL'|'SM'|'SN'|'SO'|'SR'|'ST'|'SV'|'SY'|'SZ'|'TC'|'TD'|'TG'|'TH'|'TJ'|'TK'|'TL'|'TM'|'TN'|'TO'|'TR'|'TT'|'TV'|'TW'|'TZ'|'UA'|'UG'|'US'|'UY'|'UZ'|'VA'|'VC'|'VE'|'VG'|'VI'|'VN'|'VU'|'WF'|'WS'|'YE'|'YT'|'ZA'|'ZM'|'ZW', + 'ZipCode': 'string', + 'PhoneNumber': 'string', + 'Email': 'string', + 'Fax': 'string', + 'ExtraParams': [ + { + 'Name': 'DUNS_NUMBER'|'BRAND_NUMBER'|'BIRTH_DEPARTMENT'|'BIRTH_DATE_IN_YYYY_MM_DD'|'BIRTH_COUNTRY'|'BIRTH_CITY'|'DOCUMENT_NUMBER'|'AU_ID_NUMBER'|'AU_ID_TYPE'|'CA_LEGAL_TYPE'|'CA_BUSINESS_ENTITY_TYPE'|'CA_LEGAL_REPRESENTATIVE'|'CA_LEGAL_REPRESENTATIVE_CAPACITY'|'ES_IDENTIFICATION'|'ES_IDENTIFICATION_TYPE'|'ES_LEGAL_FORM'|'FI_BUSINESS_NUMBER'|'FI_ID_NUMBER'|'FI_NATIONALITY'|'FI_ORGANIZATION_TYPE'|'IT_NATIONALITY'|'IT_PIN'|'IT_REGISTRANT_ENTITY_TYPE'|'RU_PASSPORT_DATA'|'SE_ID_NUMBER'|'SG_ID_NUMBER'|'VAT_NUMBER'|'UK_CONTACT_TYPE'|'UK_COMPANY_NUMBER', + 'Value': 'string' + }, + ] + }, + 'RegistrantContact': { + 'FirstName': 'string', + 'LastName': 'string', + 'ContactType': 'PERSON'|'COMPANY'|'ASSOCIATION'|'PUBLIC_BODY'|'RESELLER', + 'OrganizationName': 'string', + 'AddressLine1': 'string', + 'AddressLine2': 'string', + 'City': 'string', + 'State': 'string', + 'CountryCode': 'AD'|'AE'|'AF'|'AG'|'AI'|'AL'|'AM'|'AN'|'AO'|'AQ'|'AR'|'AS'|'AT'|'AU'|'AW'|'AZ'|'BA'|'BB'|'BD'|'BE'|'BF'|'BG'|'BH'|'BI'|'BJ'|'BL'|'BM'|'BN'|'BO'|'BR'|'BS'|'BT'|'BW'|'BY'|'BZ'|'CA'|'CC'|'CD'|'CF'|'CG'|'CH'|'CI'|'CK'|'CL'|'CM'|'CN'|'CO'|'CR'|'CU'|'CV'|'CX'|'CY'|'CZ'|'DE'|'DJ'|'DK'|'DM'|'DO'|'DZ'|'EC'|'EE'|'EG'|'ER'|'ES'|'ET'|'FI'|'FJ'|'FK'|'FM'|'FO'|'FR'|'GA'|'GB'|'GD'|'GE'|'GH'|'GI'|'GL'|'GM'|'GN'|'GQ'|'GR'|'GT'|'GU'|'GW'|'GY'|'HK'|'HN'|'HR'|'HT'|'HU'|'ID'|'IE'|'IL'|'IM'|'IN'|'IQ'|'IR'|'IS'|'IT'|'JM'|'JO'|'JP'|'KE'|'KG'|'KH'|'KI'|'KM'|'KN'|'KP'|'KR'|'KW'|'KY'|'KZ'|'LA'|'LB'|'LC'|'LI'|'LK'|'LR'|'LS'|'LT'|'LU'|'LV'|'LY'|'MA'|'MC'|'MD'|'ME'|'MF'|'MG'|'MH'|'MK'|'ML'|'MM'|'MN'|'MO'|'MP'|'MR'|'MS'|'MT'|'MU'|'MV'|'MW'|'MX'|'MY'|'MZ'|'NA'|'NC'|'NE'|'NG'|'NI'|'NL'|'NO'|'NP'|'NR'|'NU'|'NZ'|'OM'|'PA'|'PE'|'PF'|'PG'|'PH'|'PK'|'PL'|'PM'|'PN'|'PR'|'PT'|'PW'|'PY'|'QA'|'RO'|'RS'|'RU'|'RW'|'SA'|'SB'|'SC'|'SD'|'SE'|'SG'|'SH'|'SI'|'SK'|'SL'|'SM'|'SN'|'SO'|'SR'|'ST'|'SV'|'SY'|'SZ'|'TC'|'TD'|'TG'|'TH'|'TJ'|'TK'|'TL'|'TM'|'TN'|'TO'|'TR'|'TT'|'TV'|'TW'|'TZ'|'UA'|'UG'|'US'|'UY'|'UZ'|'VA'|'VC'|'VE'|'VG'|'VI'|'VN'|'VU'|'WF'|'WS'|'YE'|'YT'|'ZA'|'ZM'|'ZW', + 'ZipCode': 'string', + 'PhoneNumber': 'string', + 'Email': 'string', + 'Fax': 'string', + 'ExtraParams': [ + { + 'Name': 'DUNS_NUMBER'|'BRAND_NUMBER'|'BIRTH_DEPARTMENT'|'BIRTH_DATE_IN_YYYY_MM_DD'|'BIRTH_COUNTRY'|'BIRTH_CITY'|'DOCUMENT_NUMBER'|'AU_ID_NUMBER'|'AU_ID_TYPE'|'CA_LEGAL_TYPE'|'CA_BUSINESS_ENTITY_TYPE'|'CA_LEGAL_REPRESENTATIVE'|'CA_LEGAL_REPRESENTATIVE_CAPACITY'|'ES_IDENTIFICATION'|'ES_IDENTIFICATION_TYPE'|'ES_LEGAL_FORM'|'FI_BUSINESS_NUMBER'|'FI_ID_NUMBER'|'FI_NATIONALITY'|'FI_ORGANIZATION_TYPE'|'IT_NATIONALITY'|'IT_PIN'|'IT_REGISTRANT_ENTITY_TYPE'|'RU_PASSPORT_DATA'|'SE_ID_NUMBER'|'SG_ID_NUMBER'|'VAT_NUMBER'|'UK_CONTACT_TYPE'|'UK_COMPANY_NUMBER', + 'Value': 'string' + }, + ] + }, + 'TechContact': { + 'FirstName': 'string', + 'LastName': 'string', + 'ContactType': 'PERSON'|'COMPANY'|'ASSOCIATION'|'PUBLIC_BODY'|'RESELLER', + 'OrganizationName': 'string', + 'AddressLine1': 'string', + 'AddressLine2': 'string', + 'City': 'string', + 'State': 'string', + 'CountryCode': 'AD'|'AE'|'AF'|'AG'|'AI'|'AL'|'AM'|'AN'|'AO'|'AQ'|'AR'|'AS'|'AT'|'AU'|'AW'|'AZ'|'BA'|'BB'|'BD'|'BE'|'BF'|'BG'|'BH'|'BI'|'BJ'|'BL'|'BM'|'BN'|'BO'|'BR'|'BS'|'BT'|'BW'|'BY'|'BZ'|'CA'|'CC'|'CD'|'CF'|'CG'|'CH'|'CI'|'CK'|'CL'|'CM'|'CN'|'CO'|'CR'|'CU'|'CV'|'CX'|'CY'|'CZ'|'DE'|'DJ'|'DK'|'DM'|'DO'|'DZ'|'EC'|'EE'|'EG'|'ER'|'ES'|'ET'|'FI'|'FJ'|'FK'|'FM'|'FO'|'FR'|'GA'|'GB'|'GD'|'GE'|'GH'|'GI'|'GL'|'GM'|'GN'|'GQ'|'GR'|'GT'|'GU'|'GW'|'GY'|'HK'|'HN'|'HR'|'HT'|'HU'|'ID'|'IE'|'IL'|'IM'|'IN'|'IQ'|'IR'|'IS'|'IT'|'JM'|'JO'|'JP'|'KE'|'KG'|'KH'|'KI'|'KM'|'KN'|'KP'|'KR'|'KW'|'KY'|'KZ'|'LA'|'LB'|'LC'|'LI'|'LK'|'LR'|'LS'|'LT'|'LU'|'LV'|'LY'|'MA'|'MC'|'MD'|'ME'|'MF'|'MG'|'MH'|'MK'|'ML'|'MM'|'MN'|'MO'|'MP'|'MR'|'MS'|'MT'|'MU'|'MV'|'MW'|'MX'|'MY'|'MZ'|'NA'|'NC'|'NE'|'NG'|'NI'|'NL'|'NO'|'NP'|'NR'|'NU'|'NZ'|'OM'|'PA'|'PE'|'PF'|'PG'|'PH'|'PK'|'PL'|'PM'|'PN'|'PR'|'PT'|'PW'|'PY'|'QA'|'RO'|'RS'|'RU'|'RW'|'SA'|'SB'|'SC'|'SD'|'SE'|'SG'|'SH'|'SI'|'SK'|'SL'|'SM'|'SN'|'SO'|'SR'|'ST'|'SV'|'SY'|'SZ'|'TC'|'TD'|'TG'|'TH'|'TJ'|'TK'|'TL'|'TM'|'TN'|'TO'|'TR'|'TT'|'TV'|'TW'|'TZ'|'UA'|'UG'|'US'|'UY'|'UZ'|'VA'|'VC'|'VE'|'VG'|'VI'|'VN'|'VU'|'WF'|'WS'|'YE'|'YT'|'ZA'|'ZM'|'ZW', + 'ZipCode': 'string', + 'PhoneNumber': 'string', + 'Email': 'string', + 'Fax': 'string', + 'ExtraParams': [ + { + 'Name': 'DUNS_NUMBER'|'BRAND_NUMBER'|'BIRTH_DEPARTMENT'|'BIRTH_DATE_IN_YYYY_MM_DD'|'BIRTH_COUNTRY'|'BIRTH_CITY'|'DOCUMENT_NUMBER'|'AU_ID_NUMBER'|'AU_ID_TYPE'|'CA_LEGAL_TYPE'|'CA_BUSINESS_ENTITY_TYPE'|'CA_LEGAL_REPRESENTATIVE'|'CA_LEGAL_REPRESENTATIVE_CAPACITY'|'ES_IDENTIFICATION'|'ES_IDENTIFICATION_TYPE'|'ES_LEGAL_FORM'|'FI_BUSINESS_NUMBER'|'FI_ID_NUMBER'|'FI_NATIONALITY'|'FI_ORGANIZATION_TYPE'|'IT_NATIONALITY'|'IT_PIN'|'IT_REGISTRANT_ENTITY_TYPE'|'RU_PASSPORT_DATA'|'SE_ID_NUMBER'|'SG_ID_NUMBER'|'VAT_NUMBER'|'UK_CONTACT_TYPE'|'UK_COMPANY_NUMBER', + 'Value': 'string' + }, + ] + }, + 'AdminPrivacy': True|False, + 'RegistrantPrivacy': True|False, + 'TechPrivacy': True|False, + 'RegistrarName': 'string', + 'WhoIsServer': 'string', + 'RegistrarUrl': 'string', + 'AbuseContactEmail': 'string', + 'AbuseContactPhone': 'string', + 'RegistryDomainId': 'string', + 'CreationDate': datetime(2015, 1, 1), + 'UpdatedDate': datetime(2015, 1, 1), + 'ExpirationDate': datetime(2015, 1, 1), + 'Reseller': 'string', + 'DnsSec': 'string', + 'StatusList': [ + 'string', + ] + } + """ + domains_list = [] + domain_changes = {} + domain_changes["errors"] = {} + domain_changes["updates"] = {} + + logger.info( + "Starting Route53 synchronization task at %s", datetime.datetime.now() + ) + + # Fetch cloud API keys and tokens + cloud_config = CloudServicesConfiguration.get_solo() + + # Set timezone for dates to UTC + utc = pytz.UTC + + try: + client = boto3.client( + "route53domains", + region_name="us-east-1", + aws_access_key_id=cloud_config.aws_key, + aws_secret_access_key=cloud_config.aws_secret, + ) + + domain_req = client.list_domains(MaxItems=100) + if domain_req["ResponseMetadata"]["HTTPStatusCode"] == 200: + while True: + for domain in domain_req["Domains"]: + domains_list.append(domain) + if "NextPageMarker" in domain_req: + domain_req = client.list_domains(MaxItems=100, Marker=domain_req["NextPageMarker"]) + else: + break + else: + logger.error( + "Route53 returned a %s response", domain_req["ResponseMetadata"]["HTTPStatusCode"] + ) + domain_changes["errors"][ + "route53" + ] = "Route53 returned a {status_code} response".format( + status_code=domain_req["ResponseMetadata"]["HTTPStatusCode"] + ) + return domain_changes + except ClientError: + logger.error("Route53 API request failed") + domain_changes["errors"][ + "route53" + ] = "The Route53 API request failed: {traceback}".format(traceback=traceback.format_exc()) + return domain_changes + + # There's a chance no domains are returned if the provided usernames don't have any domains + if domains_list: + # Get the current list of Route53 domains in the library + domain_queryset = Domain.objects.filter(registrar="Route53") + expired_status = DomainStatus.objects.get(domain_status="Expired") + for domain in domain_queryset: + # Check if a domain in the library is _not_ in the Route53 response + if not any(d["DomainName"] == domain.name for d in domains_list): + # Domains not found in Route53 have expired and fallen off the account + if not domain.expired: + logger.info( + "Domain %s is not in the Route53 data so it is now marked as expired", + domain.name, + ) + # Mark the domain as Expired + domain_changes["updates"][domain.id] = {} + domain_changes["updates"][domain.id]["domain"] = domain.name + domain_changes["updates"][domain.id]["change"] = "expired" + entry = {} + domain.expired = True + domain.auto_renew = False + domain.domain_status = expired_status + # If the domain expiration date is in the future, adjust it + if domain.expiration >= date.today(): + domain.expiration = domain.expiration - datetime.timedelta( + days=365 + ) + try: + for attr, value in entry.items(): + setattr(domain, attr, value) + domain.save() + except Exception: + trace = traceback.format_exc() + domain_changes["errors"][ + domain + ] = "Failed to update the entry for {domain}: {traceback}".format( + domain=domain, traceback=trace + ) + logger.exception( + "Failed to update the entry for %s", domain.name + ) + pass + instance = DomainNote.objects.create( + domain=domain, + note="Automatically set to Expired because the domain did not appear in Route53 during a sync.", + ) + + # perform domain updates + for domain in domains_list: + logger.info("Domain %s is now being processed", domain["DomainName"]) + + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/route53domains.html#Route53Domains.Client.get_domain_detail + domain_details = client.get_domain_detail(DomainName=domain["DomainName"]) + + # Prepare domain attributes for Domain model + entry = {} + entry["name"] = domain["DomainName"] + entry["registrar"] = "Route53" + + # Set the WHOIS status based on WhoisGuard + if domain["Expiry"].replace(tzinfo=utc) < datetime.datetime.now().replace(tzinfo=utc): + entry["expired"] = True + # Expired domains have WhoisGuard set to ``NOTPRESENT`` + entry["whois_status"] = WhoisStatus.objects.get(pk=2) + else: + try: + whois_status = "Disabled" + if domain_details["AdminPrivacy"] \ + and domain_details["RegistrantPrivacy"] \ + and domain_details["TechPrivacy"]: + whois_status = "Enabled" + + entry["whois_status"] = WhoisStatus.objects.get( + whois_status__iexact=whois_status + ) + # Anything not ``Enabled`` or ``Disabled``, set to ``Unknown`` + except Exception: + logger.exception( + "Route53 WHOIS status (%s) was not found in the database, so defaulted to `Unknown`", + domain["WhoisGuard"].capitalize(), + ) + entry["whois_status"] = WhoisStatus.objects.get(pk=3) + + # Check if the domain is locked - locked generally means it's burned + newly_burned = False + if domain["TransferLock"]: + logger.warning( + "Domain %s is marked as LOCKED by Route53", domain["DomainName"] + ) + newly_burned = True + entry["health_status"] = HealthStatus.objects.get( + health_status="Burned" + ) + entry["domain_status"] = DomainStatus.objects.get( + domain_status="Burned" + ) + entry[ + "burned_explanation" + ] = "

Route53 has locked the domain. This is usually the result of a legal complaint related to phishing/malicious activities.

" + + # Set AutoRenew status + if not domain["AutoRenew"]: + entry["auto_renew"] = False + + # Convert Route53 dates to Django + entry["creation"] = domain_details["CreationDate"].strftime("%Y-%m-%d") + entry["expiration"] = domain_details["ExpirationDate"].strftime("%Y-%m-%d") + + # Update or create the domain record with assigned attrs + try: + instance, created = Domain.objects.update_or_create( + name=domain.get("DomainName"), defaults=entry + ) + for attr, value in entry.items(): + setattr(instance, attr, value) + + logger.debug( + "Domain %s is being saved with this data: %s", domain["DomainName"], entry + ) + instance.save() + + # Add entry to domain change tracking dict + domain_changes["updates"][instance.id] = {} + domain_changes["updates"][instance.id]["domain"] = domain["DomainName"] + if created and domain["TransferLock"]: + domain_changes["updates"][instance.id][ + "change" + ] = "created & burned" + elif created: + domain_changes["updates"][instance.id]["change"] = "created" + else: + if newly_burned: + domain_changes["updates"][instance.id]["change"] = "burned" + else: + domain_changes["updates"][instance.id]["change"] = "updated" + except Exception: + trace = traceback.format_exc() + logger.exception( + "Encountered an exception while trying to create or update %s", + domain["DomainName"], + ) + domain_changes["errors"][domain["DomainName"]] = {} + domain_changes["errors"][domain["DomainName"]]["error"] = trace + logger.info( + "Route53 synchronization completed at %s with these changes:\n%s", + datetime.datetime.now(), + domain_changes, + ) + else: + logger.warning("No domains were returned for the provided Route53 account!") + + return domain_changes + + def months_between(date1, date2): """ Compare two dates and return the number of months beetween them. diff --git a/ghostwriter/shepherd/templates/shepherd/update.html b/ghostwriter/shepherd/templates/shepherd/update.html index 1546bd94c..1f6e47728 100644 --- a/ghostwriter/shepherd/templates/shepherd/update.html +++ b/ghostwriter/shepherd/templates/shepherd/update.html @@ -57,6 +57,43 @@
Pull Domains from Namecheap
+ + {% if enable_route53 %} +
Pull Domains from Route53
+
+ +

The domain library sync with Route53 was last requested on:

+

+ {{ route53_last_update_requested }} +

+ + {% if route53_last_update_completed %} + {% if route53_last_update_completed == 'Failed' %} +

Request Status: {{ route53_last_update_completed }}

+ {% if cat_last_result %} +
+ Error: + + {{ route53_last_result }} + +
+ {% endif %} + {% else %} + {% if route53_last_update_completed %} +

Request Status: Completed on {{ route53_last_update_completed }} in {{ route53_last_update_time }} minutes

+ {% endif %} + {% endif %} + {% endif %} + +
+ {% csrf_token %} + + +
+ {% endif %} + + +
Domain Categories
diff --git a/ghostwriter/shepherd/urls.py b/ghostwriter/shepherd/urls.py index ebe2a11f6..08bc03060 100644 --- a/ghostwriter/shepherd/urls.py +++ b/ghostwriter/shepherd/urls.py @@ -76,6 +76,11 @@ views.RegistrarSyncNamecheap.as_view(), name="ajax_update_namecheap", ), + path( + "ajax/update/route53", + views.RegistrarSyncRoute53.as_view(), + name="ajax_update_route53", + ), path( "ajax/update/cloud", views.MonitorCloudInfrastructure.as_view(), diff --git a/ghostwriter/shepherd/views.py b/ghostwriter/shepherd/views.py index 100df0c05..6c98b8764 100644 --- a/ghostwriter/shepherd/views.py +++ b/ghostwriter/shepherd/views.py @@ -27,6 +27,7 @@ from ghostwriter.commandcenter.models import ( CloudServicesConfiguration, NamecheapConfiguration, + Route53Configuration, VirusTotalConfiguration, ) from ghostwriter.rolodex.models import Project @@ -361,6 +362,37 @@ def post(self, request, *args, **kwargs): return JsonResponse(data) +class RegistrarSyncRoute53(LoginRequiredMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``Route53 Update`` with + :task:`shepherd.tasks.fetch_route53_domains` to create or update one or more + :model:`shepherd.Domain`. + """ + + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``Route53 Update`` + result = "success" + try: + task_id = async_task( + "ghostwriter.shepherd.tasks.fetch_route53_domains", + group="Route53 Update", + ) + message = ( + "Successfully queued Route53 update task (Task ID {task})".format( + task=task_id + ) + ) + except Exception: + result = "error" + message = "Route53 update task could not be queued" + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) + + class MonitorCloudInfrastructure(LoginRequiredMixin, View): """ Create an individual :model:`django_q.Task` under group ``Cloud Infrastructure Review`` @@ -800,6 +832,16 @@ def update(request): End time of latest :model:`django_q.Task` for group "Namecheap Update" ``namecheap_last_result`` Result of latest :model:`django_q.Task` for group "Namecheap Update" + ``enable_route53`` + The associated value from :model:`commandcenter.Route53Configuration` + ``route53_last_update_requested`` + Start time of latest :model:`django_q.Task` for group "Route53 Update" + ``route53_last_update_completed`` + End time of latest :model:`django_q.Task` for group "Route53 Update" + ``route53_last_update_time`` + End time of latest :model:`django_q.Task` for group "Route53 Update" + ``route53_last_result`` + Result of latest :model:`django_q.Task` for group "Route53 Update" ``enable_cloud_monitor`` The associated value from :model:`commandcenter.CloudServicesConfiguration` ``cloud_last_update_requested`` @@ -824,6 +866,8 @@ def update(request): enable_cloud_monitor = cloud_config.enable namecheap_config = NamecheapConfiguration.get_solo() enable_namecheap = namecheap_config.enable + route53_config = Route53Configuration.get_solo() + enable_route53 = route53_config.enable # Collect data for category updates cat_last_update_completed = "" @@ -892,6 +936,25 @@ def update(request): else: namecheap_last_update_requested = "Namecheap Syncing is Disabled" + # Collect data for Route53 updates + route53_last_update_completed = "" + route53_last_update_time = "" + route53_last_result = "" + if enable_route53: + try: + queryset = Task.objects.filter(group="Route53 Update")[0] + route53_last_update_requested = queryset.started + route53_last_result = queryset.result + if queryset.success: + route53_last_update_completed = queryset.stopped + route53_last_update_time = round(queryset.time_taken() / 60, 2) + else: + route53_last_update_completed = "Failed" + except Exception: + route53_last_update_requested = "Route53 Sync Has Not Been Run Yet" + else: + route53_last_update_requested = "Route53 Syncing is Disabled" + # Collect data for cloud monitoring cloud_last_update_completed = "" cloud_last_update_time = "" @@ -928,6 +991,11 @@ def update(request): "namecheap_last_update_completed": namecheap_last_update_completed, "namecheap_last_update_time": namecheap_last_update_time, "namecheap_last_result": namecheap_last_result, + "enable_route53": enable_route53, + "route53_last_update_requested": route53_last_update_requested, + "route53_last_update_completed": route53_last_update_completed, + "route53_last_update_time": route53_last_update_time, + "route53_last_result": route53_last_result, "enable_cloud_monitor": enable_cloud_monitor, "cloud_last_update_requested": cloud_last_update_requested, "cloud_last_update_completed": cloud_last_update_completed, From 2e7d9aa480948f3aea364f8a14767f3987e7b7c9 Mon Sep 17 00:00:00 2001 From: Adam <1392689+coffeegist@users.noreply.github.com> Date: Wed, 13 Jan 2021 18:49:02 -0600 Subject: [PATCH 3/4] * Use correct key configuration object * Fix inaccurate burn status --- ghostwriter/shepherd/tasks.py | 37 +++++++---------------------------- 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 30374c7ef..82e197ba1 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -24,6 +24,7 @@ from ghostwriter.commandcenter.models import ( CloudServicesConfiguration, NamecheapConfiguration, + Route53Configuration, SlackConfiguration, VirusTotalConfiguration, ) @@ -1167,8 +1168,8 @@ def fetch_route53_domains(): "Starting Route53 synchronization task at %s", datetime.datetime.now() ) - # Fetch cloud API keys and tokens - cloud_config = CloudServicesConfiguration.get_solo() + # Fetch route53 API keys and tokens + route53_config = Route53Configuration.get_solo() # Set timezone for dates to UTC utc = pytz.UTC @@ -1177,8 +1178,8 @@ def fetch_route53_domains(): client = boto3.client( "route53domains", region_name="us-east-1", - aws_access_key_id=cloud_config.aws_key, - aws_secret_access_key=cloud_config.aws_secret, + aws_access_key_id=route53_config.access_key, + aws_secret_access_key=route53_config.secret_access_key, ) domain_req = client.list_domains(MaxItems=100) @@ -1290,23 +1291,6 @@ def fetch_route53_domains(): ) entry["whois_status"] = WhoisStatus.objects.get(pk=3) - # Check if the domain is locked - locked generally means it's burned - newly_burned = False - if domain["TransferLock"]: - logger.warning( - "Domain %s is marked as LOCKED by Route53", domain["DomainName"] - ) - newly_burned = True - entry["health_status"] = HealthStatus.objects.get( - health_status="Burned" - ) - entry["domain_status"] = DomainStatus.objects.get( - domain_status="Burned" - ) - entry[ - "burned_explanation" - ] = "

Route53 has locked the domain. This is usually the result of a legal complaint related to phishing/malicious activities.

" - # Set AutoRenew status if not domain["AutoRenew"]: entry["auto_renew"] = False @@ -1331,17 +1315,10 @@ def fetch_route53_domains(): # Add entry to domain change tracking dict domain_changes["updates"][instance.id] = {} domain_changes["updates"][instance.id]["domain"] = domain["DomainName"] - if created and domain["TransferLock"]: - domain_changes["updates"][instance.id][ - "change" - ] = "created & burned" - elif created: + if created: domain_changes["updates"][instance.id]["change"] = "created" else: - if newly_burned: - domain_changes["updates"][instance.id]["change"] = "burned" - else: - domain_changes["updates"][instance.id]["change"] = "updated" + domain_changes["updates"][instance.id]["change"] = "updated" except Exception: trace = traceback.format_exc() logger.exception( From 921851b2eb72cce379c9d6debb9e688040e51c5c Mon Sep 17 00:00:00 2001 From: Adam <1392689+coffeegist@users.noreply.github.com> Date: Wed, 13 Jan 2021 19:09:40 -0600 Subject: [PATCH 4/4] * Add configuration review and tests for Route53 --- .../home/templates/home/management.html | 21 ++++++++ ghostwriter/home/urls.py | 5 ++ ghostwriter/home/views.py | 35 ++++++++++++ ghostwriter/shepherd/tasks.py | 53 +++++++++++++++++++ 4 files changed, 114 insertions(+) diff --git a/ghostwriter/home/templates/home/management.html b/ghostwriter/home/templates/home/management.html index c7a1498a4..f30248e43 100644 --- a/ghostwriter/home/templates/home/management.html +++ b/ghostwriter/home/templates/home/management.html @@ -18,6 +18,7 @@ {% get_solo "commandcenter.CompanyInformation" as company_config %} {% get_solo "commandcenter.CloudServicesConfiguration" as cloud_config %} {% get_solo "commandcenter.NamecheapConfiguration" as namecheap_config %} + {% get_solo "commandcenter.Route53Configuration" as route53_config %} {% get_solo "commandcenter.ReportConfiguration" as report_config %} {% get_solo "commandcenter.SlackConfiguration" as slack_config %} {% get_solo "commandcenter.VirusTotalConfiguration" as vt_config %} @@ -122,6 +123,25 @@

API & Notification Configurations

Disabled {% endif %} + {% if route53_config.enable %} + + Route53 API Enabled + {{ route53_config.enable }} + + + Route53 Access Key + {{ route53_config.access_key }} + + + Route53 Secret Access Key + {{ route53_config.sanitized_secret_access_key }} + + {% else %} + + Route53 API Enabled + Disabled + + {% endif %} @@ -212,6 +232,7 @@
Test Configurations
+
diff --git a/ghostwriter/home/urls.py b/ghostwriter/home/urls.py index 03a9918d1..d2995b618 100644 --- a/ghostwriter/home/urls.py +++ b/ghostwriter/home/urls.py @@ -33,6 +33,11 @@ views.TestNamecheapConnection.as_view(), name="ajax_test_namecheap", ), + path( + "ajax/management/test/route53", + views.TestRoute53Connection.as_view(), + name="ajax_test_route53", + ), path( "ajax/management/test/slack", views.TestSlackConnection.as_view(), diff --git a/ghostwriter/home/views.py b/ghostwriter/home/views.py index b9996ad21..89ef0875e 100644 --- a/ghostwriter/home/views.py +++ b/ghostwriter/home/views.py @@ -274,6 +274,41 @@ def post(self, request, *args, **kwargs): return JsonResponse(data) +class TestRoute53Connection(LoginRequiredMixin, UserPassesTestMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``Route53 Test`` with + :task:`shepherd.tasks.test_route53` to test the Route53 API configuration stored + in :model:`commandcenter.Route53Configuration`. + """ + + def test_func(self): + return self.request.user.is_staff + + def handle_no_permission(self): + messages.error(self.request, "You do not have permission to access that") + return redirect("home:dashboard") + + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``Route53 Test`` + result = "success" + try: + task_id = async_task( + "ghostwriter.shepherd.tasks.test_route53", + self.request.user, + group="Route53 Test", + ) + message = "Route53 API test has been successfully queued" + except Exception: + result = "error" + message = "Route53 API test could not be queued" + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) + + class TestSlackConnection(LoginRequiredMixin, UserPassesTestMixin, View): """ Create an individual :model:`django_q.Task` under group ``Slack Test`` with diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 82e197ba1..babdbe03e 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -1925,6 +1925,59 @@ def test_namecheap(user): return {"result": level, "message": message} +def test_route53(user): + """ + Test the Route53 API configuration stored in :model:`commandcenter.Route53Configuration`. + """ + route53_config = Route53Configuration.get_solo() + level = "error" + logger.info("Starting Route53 API test at %s", datetime.datetime.now()) + try: + client = boto3.client( + "route53domains", + region_name="us-east-1", + aws_access_key_id=route53_config.access_key, + aws_secret_access_key=route53_config.secret_access_key, + ) + + domain_req = client.list_domains(MaxItems=100) + + # Check if request returned a 200 OK + status_code = domain_req["ResponseMetadata"]["HTTPStatusCode"] + if status_code == 200: + level = "success" + message = "Successfully authenticated to Route53" + else: + logger.error( + "Route53 returned HTTP code %s in its response", + status_code + ) + message = f"Route53 returned HTTP code {status_code} in its response" + except Exception: + trace = traceback.format_exc() + logger.exception("Route53 API request failed") + message = f"The Route53 API request failed: {trace}" + + # Send a message to the requesting user + async_to_sync(channel_layer.group_send)( + "notify_{}".format(user), + { + "type": "message", + "message": { + "message": message, + "level": level, + "title": "Route53 Test Complete", + }, + }, + ) + + logger.info( + "Test of the Route53 API configuration completed at %s", + datetime.datetime.now(), + ) + return {"result": level, "message": message} + + def test_slack_webhook(user): """ Test the Slack Webhook configuration stored in :model:`commandcenter.SlackConfiguration`.