From 798f05a1179b44aed479dd38f5c9c8fb6dc9e9bb Mon Sep 17 00:00:00 2001 From: Daniel Pascual Date: Wed, 7 Aug 2024 11:16:57 +0200 Subject: [PATCH 1/4] Add more attributes to the GTI enrichment --- .../expansion/google_threat_intelligence.py | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/misp_modules/modules/expansion/google_threat_intelligence.py b/misp_modules/modules/expansion/google_threat_intelligence.py index 220b96f4..32458300 100644 --- a/misp_modules/modules/expansion/google_threat_intelligence.py +++ b/misp_modules/modules/expansion/google_threat_intelligence.py @@ -72,6 +72,17 @@ def __init__(self, client: vt.Client, limit: int) -> None: } self.proxies = None + @staticmethod + def get_total_analysis(analysis: dict, + known_distributors: dict = None) -> int: + """Get total """ + if not analysis: + return 0 + count = sum([analysis['undetected'], + analysis['suspicious'], + analysis['harmless']]) + return count if known_distributors else count + analysis['malicious'] + def query_api(self, attribute: dict) -> None: """Get data from the API and parse it.""" self.attribute.from_dict(**attribute) @@ -91,19 +102,19 @@ def create_gti_report_object(self, report): report = report.to_dict() permalink = ('https://www.virustotal.com/gui/' f"{report['type']}/{report['id']}") - report_object = pymisp.MISPObject('Google-Threat-Intel-report') + report_object = pymisp.MISPObject('google-threat-intelligence-report') report_object.add_attribute('permalink', type='link', value=permalink) report_object.add_attribute( - 'Threat Score', type='text', + 'threat-score', type='text', value=get_key( report, 'attributes.gti_assessment.threat_score.value')) report_object.add_attribute( - 'Verdict', type='text', + 'verdict', type='text', value=get_key( report, 'attributes.gti_assessment.verdict.value').replace( 'VERDICT_', '')) report_object.add_attribute( - 'Severity', type='text', + 'severity', type='text', value=get_key( report, 'attributes.gti_assessment.severity.value').replace( 'SEVERITY_', '')) @@ -112,6 +123,13 @@ def create_gti_report_object(self, report): value=get_key( report, ('attributes.popular_threat_classification' '.suggested_threat_label'))) + analysis = report.get('last_analysis_stats') + total = self.get_total_analysis(analysis, + report.get('known_distributors')) + detection_ratio = f"{analysis['malicious']}/{total}" if analysis else '-/-' + report_object.add_attribute( + 'detection-ratio', type='text', + value=detection_ratio, disable_correlation=True) self.misp_event.add_object(**report_object) return report_object.uuid @@ -162,7 +180,7 @@ def parse_url(self, url: str) -> str: url_report = self.client.get_object(f'/urls/{url_id}') url_object = pymisp.MISPObject('url') - url_object.add_attribute('url', type='url', value=url_report.url) + url_object.add_attribute('url', type='url', value=url_report.id) report_uuid = self.create_gti_report_object(url_report) url_object.add_reference(report_uuid, 'analyzed-with') From 3b69446185972516d3fd74d097afae6a3367a761 Mon Sep 17 00:00:00 2001 From: Daniel Pascual Date: Wed, 7 Aug 2024 17:10:28 +0200 Subject: [PATCH 2/4] WIP --- .../modules/expansion/google_threat_intelligence.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_modules/modules/expansion/google_threat_intelligence.py b/misp_modules/modules/expansion/google_threat_intelligence.py index 32458300..2b967f20 100644 --- a/misp_modules/modules/expansion/google_threat_intelligence.py +++ b/misp_modules/modules/expansion/google_threat_intelligence.py @@ -123,9 +123,9 @@ def create_gti_report_object(self, report): value=get_key( report, ('attributes.popular_threat_classification' '.suggested_threat_label'))) - analysis = report.get('last_analysis_stats') - total = self.get_total_analysis(analysis, - report.get('known_distributors')) + analysis = report.get_key('attributes.last_analysis_stats') + total = self.get_total_analysis( + analysis, report.get_key('attributes.known_distributors')) detection_ratio = f"{analysis['malicious']}/{total}" if analysis else '-/-' report_object.add_attribute( 'detection-ratio', type='text', From f6305f40411c829e718082a2e9d1733a1c5d2ba2 Mon Sep 17 00:00:00 2001 From: Daniel Pascual Date: Thu, 8 Aug 2024 14:13:07 +0200 Subject: [PATCH 3/4] WIP --- .../expansion/google_threat_intelligence.py | 217 +++++++++++++----- 1 file changed, 165 insertions(+), 52 deletions(-) diff --git a/misp_modules/modules/expansion/google_threat_intelligence.py b/misp_modules/modules/expansion/google_threat_intelligence.py index 2b967f20..cfad0502 100644 --- a/misp_modules/modules/expansion/google_threat_intelligence.py +++ b/misp_modules/modules/expansion/google_threat_intelligence.py @@ -34,7 +34,7 @@ } MODULE_INFO = { - 'version': '1', + 'version': '2', 'author': 'Google Threat Intelligence team', 'description': ('An expansion module to have the observable\'s threat' ' score assessed by Google Threat Intelligence.'), @@ -68,7 +68,9 @@ def __init__(self, client: vt.Client, limit: int) -> None: 'md5': self.parse_hash, 'sha1': self.parse_hash, 'sha256': self.parse_hash, - 'url': self.parse_url + 'url': self.parse_url, + 'ip-src|port': self.parse_ip_port, + 'ip-dst|port': self.parse_ip_port, } self.proxies = None @@ -97,93 +99,204 @@ def get_results(self) -> dict: } return {'results': results} - def create_gti_report_object(self, report): - """Create GTI report object.""" - report = report.to_dict() - permalink = ('https://www.virustotal.com/gui/' - f"{report['type']}/{report['id']}") - report_object = pymisp.MISPObject('google-threat-intelligence-report') - report_object.add_attribute('permalink', type='link', value=permalink) - report_object.add_attribute( + + def add_gti_report(self, report: vt.Object) -> str: + analysis = report.get('last_analysis_stats') + total = self.get_total_analysis(analysis, + report.get('known_distributors')) + if report.type == 'ip_address': + rtype = 'ip-address' + else: + rtype = report.type + permalink = f'https://www.virustotal.com/gui/{rtype}/{report.id}' + + gti_object = pymisp.MISPObject('google-threat-intelligence-report') + gti_object.add_attribute('permalink', type='link', value=permalink) + ratio = f"{analysis['malicious']}/{total}" if analysis else '-/-' + gti_object.add_attribute('detection-ratio', + type='text', + value=ratio, + disable_correlation=True) + gti_object.add_attribute( 'threat-score', type='text', value=get_key( report, 'attributes.gti_assessment.threat_score.value')) - report_object.add_attribute( + gti_object.add_attribute( 'verdict', type='text', value=get_key( report, 'attributes.gti_assessment.verdict.value').replace( 'VERDICT_', '')) - report_object.add_attribute( + gti_object.add_attribute( 'severity', type='text', value=get_key( report, 'attributes.gti_assessment.severity.value').replace( 'SEVERITY_', '')) - report_object.add_attribute( - 'Threat Label', type='text', - value=get_key( - report, ('attributes.popular_threat_classification' - '.suggested_threat_label'))) - analysis = report.get_key('attributes.last_analysis_stats') - total = self.get_total_analysis( - analysis, report.get_key('attributes.known_distributors')) - detection_ratio = f"{analysis['malicious']}/{total}" if analysis else '-/-' - report_object.add_attribute( - 'detection-ratio', type='text', - value=detection_ratio, disable_correlation=True) - self.misp_event.add_object(**report_object) - return report_object.uuid + self.misp_event.add_object(**gti_object) + return gti_object.uuid + + def create_misp_object(self, report: vt.Object) -> pymisp.MISPObject: + misp_object = None + gti_uuid = self.add_gti_report(report) + + if report.type == 'file': + misp_object = pymisp.MISPObject('file') + for hash_type in ('md5', 'sha1', 'sha256', 'tlsh', + 'vhash', 'ssdeep', 'imphash'): + misp_object.add_attribute(hash_type, + **{'type': hash_type, + 'value': report.get(hash_type)}) + elif report.type == 'domain': + misp_object = pymisp.MISPObject('domain-ip') + misp_object.add_attribute('domain', type='domain', value=report.id) + elif report.type == 'ip_address': + misp_object = pymisp.MISPObject('domain-ip') + misp_object.add_attribute('ip', type='ip-dst', value=report.id) + elif report.type == 'url': + misp_object = pymisp.MISPObject('url') + misp_object.add_attribute('url', type='url', value=report.id) + misp_object.add_reference(gti_uuid, 'analyzed-with') + return misp_object def parse_domain(self, domain: str) -> str: - """Create domain MISP object.""" domain_report = self.client.get_object(f'/domains/{domain}') # DOMAIN - domain_object = pymisp.MISPObject('domain-ip') - domain_object.add_attribute( - 'domain', type='domain', value=domain_report.id) + domain_object = self.create_misp_object(domain_report) + + # WHOIS + if domain_report.whois: + whois_object = pymisp.MISPObject('whois') + whois_object.add_attribute('text', type='text', + value=domain_report.whois) + self.misp_event.add_object(**whois_object) + + # SIBLINGS AND SUBDOMAINS + for relationship_name, misp_name in [ + ('siblings', 'sibling-of'), ('subdomains', 'subdomain')]: + rel_iterator = self.client.iterator( + f'/domains/{domain_report.id}/{relationship_name}', + limit=self.limit) + for item in rel_iterator: + attr = pymisp.MISPAttribute() + attr.from_dict(**dict(type='domain', value=item.id)) + self.misp_event.add_attribute(**attr) + domain_object.add_reference(attr.uuid, misp_name) + + # RESOLUTIONS + resolutions_iterator = self.client.iterator( + f'/domains/{domain_report.id}/resolutions', limit=self.limit) + for resolution in resolutions_iterator: + domain_object.add_attribute('ip', type='ip-dst', + value=resolution.ip_address) + + # COMMUNICATING, DOWNLOADED AND REFERRER FILES + for relationship_name, misp_name in [ + ('communicating_files', 'communicates-with'), + ('downloaded_files', 'downloaded-from'), + ('referrer_files', 'referring') + ]: + files_iterator = self.client.iterator( + f'/domains/{domain_report.id}/{relationship_name}', + limit=self.limit) + for file in files_iterator: + file_object = self.create_misp_object(file) + file_object.add_reference(domain_object.uuid, misp_name) + self.misp_event.add_object(**file_object) + + # URLS + urls_iterator = self.client.iterator( + f'/domains/{domain_report.id}/urls', limit=self.limit) + for url in urls_iterator: + url_object = self.create_misp_object(url) + url_object.add_reference(domain_object.uuid, 'hosted-in') + self.misp_event.add_object(**url_object) - report_uuid = self.create_gti_report_object(domain_report) - domain_object.add_reference(report_uuid, 'analyzed-with') self.misp_event.add_object(**domain_object) return domain_object.uuid def parse_hash(self, file_hash: str) -> str: - """Create hash MISP object.""" file_report = self.client.get_object(f'/files/{file_hash}') - file_object = pymisp.MISPObject('file') - for hash_type in ('md5', 'sha1', 'sha256'): - file_object.add_attribute( - hash_type, - **{'type': hash_type, 'value': file_report.get(hash_type)}) - - report_uuid = self.create_gti_report_object(file_report) - file_object.add_reference(report_uuid, 'analyzed-with') + file_object = self.create_misp_object(file_report) + + # ITW URLS + urls_iterator = self.client.iterator( + f'/files/{file_report.id}/itw_urls', limit=self.limit) + for url in urls_iterator: + url_object = self.create_misp_object(url) + url_object.add_reference(file_object.uuid, 'downloaded') + self.misp_event.add_object(**url_object) + + # COMMUNICATING, DOWNLOADED AND REFERRER FILES + for relationship_name, misp_name in [ + ('contacted_urls', 'communicates-with'), + ('contacted_domains', 'communicates-with'), + ('contacted_ips', 'communicates-with') + ]: + related_files_iterator = self.client.iterator( + f'/files/{file_report.id}/{relationship_name}', limit=self.limit) + for related_file in related_files_iterator: + related_file_object = self.create_misp_object(related_file) + related_file_object.add_reference(file_object.uuid, misp_name) + self.misp_event.add_object(**related_file_object) + self.misp_event.add_object(**file_object) return file_object.uuid + def parse_ip_port(self, ipport: str) -> str: + ip = ipport.split('|')[0] + self.parse_ip(ip) + def parse_ip(self, ip: str) -> str: - """Create ip MISP object.""" ip_report = self.client.get_object(f'/ip_addresses/{ip}') # IP - ip_object = pymisp.MISPObject('domain-ip') - ip_object.add_attribute('ip', type='ip-dst', value=ip_report.id) + ip_object = self.create_misp_object(ip_report) + + # ASN + asn_object = pymisp.MISPObject('asn') + asn_object.add_attribute('asn', type='AS', value=ip_report.asn) + asn_object.add_attribute('subnet-announced', type='ip-src', + value=ip_report.network) + asn_object.add_attribute('country', type='text', + value=ip_report.country) + self.misp_event.add_object(**asn_object) + + # RESOLUTIONS + resolutions_iterator = self.client.iterator( + f'/ip_addresses/{ip_report.id}/resolutions', limit=self.limit) + for resolution in resolutions_iterator: + ip_object.add_attribute('domain', type='domain', + value=resolution.host_name) + + # URLS + urls_iterator = self.client.iterator( + f'/ip_addresses/{ip_report.id}/urls', limit=self.limit) + for url in urls_iterator: + url_object = self.create_misp_object(url) + url_object.add_reference(ip_object.uuid, 'hosted-in') + self.misp_event.add_object(**url_object) - report_uuid = self.create_gti_report_object(ip_report) - ip_object.add_reference(report_uuid, 'analyzed-with') self.misp_event.add_object(**ip_object) return ip_object.uuid def parse_url(self, url: str) -> str: - """Create URL MISP object.""" url_id = vt.url_id(url) url_report = self.client.get_object(f'/urls/{url_id}') + url_object = self.create_misp_object(url_report) + + # COMMUNICATING, DOWNLOADED AND REFERRER FILES + for relationship_name, misp_name in [ + ('communicating_files', 'communicates-with'), + ('downloaded_files', 'downloaded-from'), + ('referrer_files', 'referring') + ]: + files_iterator = self.client.iterator( + f'/urls/{url_report.id}/{relationship_name}', limit=self.limit) + for file in files_iterator: + file_object = self.create_misp_object(file) + file_object.add_reference(url_object.uuid, misp_name) + self.misp_event.add_object(**file_object) - url_object = pymisp.MISPObject('url') - url_object.add_attribute('url', type='url', value=url_report.id) - - report_uuid = self.create_gti_report_object(url_report) - url_object.add_reference(report_uuid, 'analyzed-with') self.misp_event.add_object(**url_object) return url_object.uuid From d720c38b0f08886e9a291f5ff411b1361a0dbc09 Mon Sep 17 00:00:00 2001 From: Daniel Pascual Date: Thu, 8 Aug 2024 16:58:17 +0200 Subject: [PATCH 4/4] WIP --- .../modules/expansion/google_threat_intelligence.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/expansion/google_threat_intelligence.py b/misp_modules/modules/expansion/google_threat_intelligence.py index cfad0502..cde28050 100644 --- a/misp_modules/modules/expansion/google_threat_intelligence.py +++ b/misp_modules/modules/expansion/google_threat_intelligence.py @@ -117,19 +117,20 @@ def add_gti_report(self, report: vt.Object) -> str: type='text', value=ratio, disable_correlation=True) + report_dict = report.to_dict() gti_object.add_attribute( 'threat-score', type='text', - value=get_key( - report, 'attributes.gti_assessment.threat_score.value')) + value=get_key(report_dict, + 'attributes.gti_assessment.threat_score.value')) gti_object.add_attribute( 'verdict', type='text', - value=get_key( - report, 'attributes.gti_assessment.verdict.value').replace( + value=get_key(report_dict, + 'attributes.gti_assessment.verdict.value').replace( 'VERDICT_', '')) gti_object.add_attribute( 'severity', type='text', - value=get_key( - report, 'attributes.gti_assessment.severity.value').replace( + value=get_key(report_dict, + 'attributes.gti_assessment.severity.value').replace( 'SEVERITY_', '')) self.misp_event.add_object(**gti_object) return gti_object.uuid