diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d9370ea9..d6a3ceb4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,11 @@ ### Features * selective_extractor: add field_template as `event_template` to configuration +* decoder: add `windows_event_log` source format to parse Windows Event Log XML into flat fields; all extracted fields can be configured with the `fields` rule option ### Improvements +* decoder: reject DOCTYPE declarations, entity expansion and excessively nested XML in the `windows_event_log` source format +* decoder: split the decoders module into one module per source format * util: add field mapping template utility * cache: hold a payload in addition to timestamps * domain_resolver: adapt `domain_resolver` to changes of `cache` diff --git a/logprep/ng/processor/decoder/processor.py b/logprep/ng/processor/decoder/processor.py index 9137e1ccb..ceda0d91c 100644 --- a/logprep/ng/processor/decoder/processor.py +++ b/logprep/ng/processor/decoder/processor.py @@ -1,3 +1,4 @@ +# pylint: disable=duplicate-code # legacy and ng processors intentionally mirror each other """ |PROCESSOR_NAME| ================ @@ -16,6 +17,7 @@ * cri * docker * decolorize (removing color codes in logs) +* windows_event_log (Windows Event Log XML, parsed into flat fields) Processor Configuration @@ -43,7 +45,7 @@ from typing_extensions import override from logprep.ng.processor.field_manager.processor import FieldManager -from logprep.processor.decoder.decoders import DECODERS, DecoderError +from logprep.processor.decoder.decoders import DecoderError, get_decoder from logprep.processor.decoder.rule import DecoderRule from logprep.processor.field_manager.rule import FieldManagerRule from logprep.util.helper import FieldValue @@ -63,7 +65,7 @@ def transform_values( rule: FieldManagerRule, ) -> list[FieldValue]: decoder_rule = typing.cast(DecoderRule, rule) - decoder = DECODERS[decoder_rule.source_format] + decoder = get_decoder(decoder_rule.source_format, decoder_rule.fields) return self._decode(event, decoder_rule, decoder, source_field_values) @override diff --git a/logprep/processor/decoder/decoders.py b/logprep/processor/decoder/decoders.py deleted file mode 100644 index 91a35a27f..000000000 --- a/logprep/processor/decoder/decoders.py +++ /dev/null @@ -1,248 +0,0 @@ -"""Utility decoders for common log formats used by Logprep. - -This module provides small, focused decoding helpers that convert raw -log lines into Python structures (typically dicts) for downstream -processors. Each decoder wraps parsing logic and maps decoding errors to -``DecoderError`` so callers can handle failures uniformly. - -Notes: - -- The internal ``_parse`` helper performs regex matching and supports - a single ``re.Pattern`` or an iterable of patterns. Avoid lookahead - and lookbehind in supplied regexes to prevent pathological cases. -- ``parse_json`` uses ``msgspec`` for fast, strict JSON decoding and - converts decode errors to ``DecoderError``. -- ``decolorize`` strips ANSI escape sequences from log lines. -""" - -import base64 -import binascii -import re -from functools import partial -from typing import Dict, Generic, Iterable, Protocol, TypeVar, runtime_checkable - -import msgspec - -from logprep.util.helper import FieldValue - -JSON_DECODER = msgspec.json.Decoder() - - -class DecoderError(Exception): - """Raised if decoding fails""" - - -DecoderFuncResult_co = TypeVar("DecoderFuncResult_co", bound=FieldValue, covariant=True) - - -@runtime_checkable -class DecoderFunc(Generic[DecoderFuncResult_co], Protocol): - """ - Covariant function type for decoding string values - into concrete variants of `FieldValue` - """ - - def __call__(self, log_line: str) -> DecoderFuncResult_co: ... - - -def _parse(log_line: str, regexes: Iterable[re.Pattern]) -> dict[str, str]: - """This parses a given log_line to a dict via provided regex - it is really important that you don't use lookahead or lookbehinds - in the provided regex, because they could break the application by - recursive expressions in strings - """ - - result = None - for regex in regexes: - result = re.match(regex, log_line) - if result is not None: - return result.groupdict() - raise DecoderError("no regex matches") - - -REGEX_CLF = ( - re.compile( - r"^" - r"^(?P[^\s]+)\s+" # hostname or ip - r"(?P[^\s]+)\s+" # identity RFC 1413 - r"(?P[^\s]+)\s+" # userid requesting the document - r"\[(?P[^\s]+\s+[^\s]+)\]\s+" # timestamp strftime format %d/%b/%Y:%H:%M:%S %z - r'"(?P.*)"\s+' # the requested document - r"(?P\d{3})\s+" # the http status returned to the client - r"(?P\d+)\s*" # content-length of the document transferred - r"$" - ), -) - - -REGEX_NGINX = ( - re.compile( - r"^" - r"(?P[^ ]*) - " - r"(?P[^ ]*) " - r"\[(?P