Skip to content

Find URL-decoded hosts for every URL in IoCExtract, not only the first #957

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Dennys Kaluzny Pereira (protonnegativo) wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Find URL-decoded hosts for every URL in IoCExtract, not only the first #957

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions msticpy/transform/iocextract.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -675,8 +675,7 @@ def _scan_for_iocs(
if ioc_types and ioc_type not in ioc_types:
continue

match_pos = 0
for rgx_match in rgx_def.comp_regex.finditer(src, match_pos):
for rgx_match in rgx_def.comp_regex.finditer(src):
if rgx_match is None:
break
# If the rgx_def names a group to match on, use that
Expand All @@ -691,8 +690,7 @@ def _scan_for_iocs(

self._add_highest_pri_match(iocs_found, match_str, rgx_def)
if ioc_type == "url":
self._check_decode_url(match_str, rgx_def, match_pos, iocs_found)
match_pos = rgx_match.end()
self._check_decode_url(match_str, rgx_def, iocs_found)

for ioc, ioc_result in iocs_found.items():
if not defanged and ioc_result[0] in [
Expand All @@ -707,10 +705,10 @@ def _scan_for_iocs(

return ioc_results

def _check_decode_url(self, match_str, rgx_def, match_pos, iocs_found):
def _check_decode_url(self, match_str, rgx_def, iocs_found):
"""Get any other IoCs from decoded URL."""
decoded_url = unquote(match_str)
for url_match in rgx_def.comp_regex.finditer(decoded_url, match_pos):
for url_match in rgx_def.comp_regex.finditer(decoded_url):
if url_match is not None:
self._add_highest_pri_match(iocs_found, url_match.group(), rgx_def)
self._add_highest_pri_match(
Expand Down
14 changes: 14 additions & 0 deletions tests/transform/test_ioc_extractor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,20 @@ def test_url(ioc_extract):
_run_extract(ioc_extract, "url2", {"url": 1, "dns": 1, "ipv4": 1})


@pytest.mark.parametrize(
"test_input",
[
"http://%65vil.com/x http://good.org/y",
"http://good.org/y http://%65vil.com/x",
],
)
def test_url_encoded_host(test_input, ioc_extract):
"""Test that URL-decoded hosts are found when the URL is not the first one."""
results = ioc_extract.extract(test_input)
check.is_in("http://evil.com/x", results["url"])
check.is_in("evil.com", results["dns"])


def test_windows_path(ioc_extract):
_run_extract(ioc_extract, "windows_path", {"windows_path": 3})

Expand Down