Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,11 @@

### Features
* selective_extractor: add field_template as `event_template` to configuration
* decoder: add `windows_event_log` source format to parse Windows Event Log XML into flat fields; all extracted fields can be configured with the `fields` rule option

### Improvements
* decoder: reject DOCTYPE declarations, entity expansion and excessively nested XML in the `windows_event_log` source format
* decoder: split the decoders module into one module per source format
* util: add field mapping template utility
* cache: hold a payload in addition to timestamps
* domain_resolver: adapt `domain_resolver` to changes of `cache`
Expand Down
6 changes: 4 additions & 2 deletions logprep/ng/processor/decoder/processor.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# pylint: disable=duplicate-code # legacy and ng processors intentionally mirror each other
"""
|PROCESSOR_NAME|
================
Expand All @@ -16,6 +17,7 @@
* cri
* docker
* decolorize (removing color codes in logs)
* windows_event_log (Windows Event Log XML, parsed into flat fields)


Processor Configuration
Expand Down Expand Up @@ -43,7 +45,7 @@
from typing_extensions import override

from logprep.ng.processor.field_manager.processor import FieldManager
from logprep.processor.decoder.decoders import DECODERS, DecoderError
from logprep.processor.decoder.decoders import DecoderError, get_decoder
from logprep.processor.decoder.rule import DecoderRule
from logprep.processor.field_manager.rule import FieldManagerRule
from logprep.util.helper import FieldValue
Expand All @@ -63,7 +65,7 @@ def transform_values(
rule: FieldManagerRule,
) -> list[FieldValue]:
decoder_rule = typing.cast(DecoderRule, rule)
decoder = DECODERS[decoder_rule.source_format]
decoder = get_decoder(decoder_rule.source_format, decoder_rule.fields)
return self._decode(event, decoder_rule, decoder, source_field_values)

@override
Expand Down
248 changes: 0 additions & 248 deletions logprep/processor/decoder/decoders.py

This file was deleted.

83 changes: 83 additions & 0 deletions logprep/processor/decoder/decoders/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
"""Decoders for common log formats used by Logprep.

Every decoder lives in its own module and converts a raw log line into a
Python structure (typically a dict). Decoding errors are mapped to
``DecoderError`` so the decoder processor can handle failures uniformly.
The ``DECODERS`` registry maps the configured ``source_format`` to the
corresponding decoder function.
"""

from logprep.processor.decoder.decoders.base64 import parse_base64
from logprep.processor.decoder.decoders.clf import REGEX_CLF, parse_clf
from logprep.processor.decoder.decoders.common import (
DecoderError,
DecoderFunc,
DecoderFuncResult_co,
parse_with_regexes,
)
from logprep.processor.decoder.decoders.cri import parse_cri
from logprep.processor.decoder.decoders.decolorize import ANSI_ESCAPE, decolorize
from logprep.processor.decoder.decoders.docker import DockerLog, parse_docker
from logprep.processor.decoder.decoders.json import JSON_DECODER, parse_json
from logprep.processor.decoder.decoders.logfmt import REGEX_LOGFMT_TOKEN, parse_logfmt
from logprep.processor.decoder.decoders.nginx import REGEX_NGINX, parse_nginx
from logprep.processor.decoder.decoders.registry import DECODERS, get_decoder
from logprep.processor.decoder.decoders.syslog_rfc3164 import (
REGEX_SYSLOG_RFC3164,
parse_syslog_rfc3164,
)
from logprep.processor.decoder.decoders.syslog_rfc3164_local import (
REGEX_SYSLOG_RFC3164_LOCAL,
parse_syslog_rfc3164_local,
)
from logprep.processor.decoder.decoders.syslog_rfc5424 import (
REGEX_ISO8601,
REGEX_SYSLOG_RFC5424,
parse_syslog_rfc5424,
)
from logprep.processor.decoder.decoders.windows_event_log import (
DATA_SECTIONS,
MAX_XML_DEPTH,
SECTIONS,
XML_FIELDS,
XmlFieldDefinition,
parse_windows_event_log,
validate_fields,
)

__all__ = [
"ANSI_ESCAPE",
"DATA_SECTIONS",
"DECODERS",
"DecoderError",
"DecoderFunc",
"DecoderFuncResult_co",
"DockerLog",
"JSON_DECODER",
"MAX_XML_DEPTH",
"REGEX_CLF",
"REGEX_ISO8601",
"REGEX_LOGFMT_TOKEN",
"REGEX_NGINX",
"REGEX_SYSLOG_RFC3164",
"REGEX_SYSLOG_RFC3164_LOCAL",
"REGEX_SYSLOG_RFC5424",
"SECTIONS",
"XML_FIELDS",
"XmlFieldDefinition",
"decolorize",
"get_decoder",
"parse_base64",
"parse_clf",
"parse_cri",
"parse_docker",
"parse_json",
"parse_logfmt",
"parse_nginx",
"parse_syslog_rfc3164",
"parse_syslog_rfc3164_local",
"parse_syslog_rfc5424",
"parse_with_regexes",
"parse_windows_event_log",
"validate_fields",
]
14 changes: 14 additions & 0 deletions logprep/processor/decoder/decoders/base64.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
"""Decoder for base64 encoded log lines."""

import base64
import binascii

from logprep.processor.decoder.decoders.common import DecoderError


def parse_base64(log_line: str) -> str:
"""Parses base64 and handles decode errors"""
try:
return base64.b64decode(log_line).decode("utf-8")
except binascii.Error as error:
raise DecoderError("can't decode base64") from error
Loading
Loading