We're just getting started so the docs and process could use some ironing out.
This is a start but quality of life improvements for writing and testing rules are welcome.
Target hyperscan compatible regex. Based on a review of different scanners, hyperscan seems to be the lowest common denominator in most cases.
If a specific target doesn't support something something that hyperscan supports, we can work through whether or not its better to handle that during translation or add further restrictions to the regex we can use in the patterns here.
A rules file is a YAML file containing rules. Rules should
be grouped by platform (e.g. AWS, GCP, GitHub, GitLab, etc). The file name
should be lowercased and use underscores if spaces are needed. The file
extension should be .yaml.
It should be a dict with a rules key that's a list of rules. See the next
section for the rules format.
Here is the "YAML-schema" (i.e. JSON schema dumped as YAML) for the rules. The source for this can also be found in schema.py.
# Generated by ./hack/translate/show-schema
$defs:
Analyzer:
properties:
meta:
$ref: '#/$defs/AnalyzerMeta'
action:
$ref: '#/$defs/AnalyzerHttpAction'
condition:
items:
$ref: '#/$defs/HttpMatcher'
title: Condition
type: array
required:
- meta
- action
- condition
title: Analyzer
type: object
AnalyzerHttpAction:
properties:
url:
format: uri
maxLength: 2083
minLength: 1
title: Url
type: string
method:
anyOf:
- type: string
- type: 'null'
default: null
title: Method
headers:
anyOf:
- additionalProperties:
type: string
type: object
- type: 'null'
default: null
title: Headers
body:
anyOf:
- type: string
- type: 'null'
default: null
title: Body
timeout:
anyOf:
- minimum: 0
type: number
- type: 'null'
default: null
title: Timeout
required:
- url
title: AnalyzerHttpAction
type: object
AnalyzerKind:
enum:
- http
title: AnalyzerKind
type: string
AnalyzerMeta:
properties:
confidence:
anyOf:
- $ref: '#/$defs/Confidence'
- type: 'null'
default: null
examples:
anyOf:
- $ref: '#/$defs/Examples'
- type: 'null'
default: null
references:
anyOf:
- items:
format: uri
maxLength: 2083
minLength: 1
type: string
type: array
- type: 'null'
default: null
title: References
report:
default: false
title: Report
type: boolean
tags:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Tags
kind:
$ref: '#/$defs/AnalyzerKind'
required:
- kind
title: AnalyzerMeta
type: object
Confidence:
enum:
- low
- medium
- high
title: Confidence
type: string
Dependancy:
properties:
rule_id:
pattern: ^S3IG[A-Z2-7]{16}$
title: Rule Id
type: string
varname:
pattern: ^[a-z](?:[a-z0-9_]*[a-z0-9])?$
title: Varname
type: string
within_lines:
anyOf:
- minimum: 0
type: integer
- type: 'null'
default: null
title: Within Lines
within_columns:
anyOf:
- minimum: 0
type: integer
- type: 'null'
default: null
title: Within Columns
required:
- rule_id
- varname
title: Dependancy
type: object
Examples:
properties:
positive:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Positive
negative:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Negative
title: Examples
type: object
ExcludeFilter:
description: Filters options supported only exclude
properties:
target_strings:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Target Strings
path_patterns:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Path Patterns
path_strings:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Path Strings
context_strings:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Context Strings
kind:
const: exclude
title: Kind
type: string
target_patterns:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Target Patterns
match_patterns:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Match Patterns
match_strings:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Match Strings
context_patterns:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Context Patterns
required:
- kind
title: ExcludeFilter
type: object
HttpMatcher:
properties:
statuses:
anyOf:
- items:
items:
type: integer
type: array
type: array
- type: 'null'
default: null
title: Statuses
headers:
anyOf:
- additionalProperties:
items:
type: string
type: array
type: object
- type: 'null'
default: null
title: Headers
body_strings:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Body Strings
body_patterns:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Body Patterns
body_syntax:
anyOf:
- $ref: '#/$defs/Syntax'
- type: 'null'
default: null
title: HttpMatcher
type: object
RequireFilter:
description: Filters options supported only require
properties:
target_strings:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Target Strings
path_patterns:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Path Patterns
path_strings:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Path Strings
context_strings:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Context Strings
kind:
const: require
title: Kind
type: string
target_min_entropy:
anyOf:
- minimum: 0
type: number
- type: 'null'
default: null
title: Target Min Entropy
required:
- kind
title: RequireFilter
type: object
Rule:
properties:
id:
pattern: ^S3IG[A-Z2-7]{16}$
title: Id
type: string
meta:
$ref: '#/$defs/RuleMeta'
dependencies:
anyOf:
- items:
$ref: '#/$defs/Dependancy'
type: array
- type: 'null'
default: null
title: Dependencies
target:
$ref: '#/$defs/Target'
filters:
anyOf:
- items:
discriminator:
mapping:
exclude: '#/$defs/ExcludeFilter'
require: '#/$defs/RequireFilter'
propertyName: kind
oneOf:
- $ref: '#/$defs/ExcludeFilter'
- $ref: '#/$defs/RequireFilter'
type: array
- type: 'null'
default: null
title: Filters
analyzers:
anyOf:
- items:
$ref: '#/$defs/Analyzer'
type: array
- type: 'null'
default: null
title: Analyzers
required:
- id
- meta
- target
title: Rule
type: object
RuleMeta:
properties:
confidence:
anyOf:
- $ref: '#/$defs/Confidence'
- type: 'null'
default: null
examples:
anyOf:
- $ref: '#/$defs/Examples'
- type: 'null'
default: null
references:
anyOf:
- items:
format: uri
maxLength: 2083
minLength: 1
type: string
type: array
- type: 'null'
default: null
title: References
report:
default: true
title: Report
type: boolean
tags:
anyOf:
- items:
type: string
type: array
- type: 'null'
default: null
title: Tags
kind:
$ref: '#/$defs/TargetKind'
default: unknown
name:
title: Name
type: string
description:
anyOf:
- type: string
- type: 'null'
default: null
title: Description
required:
- name
title: RuleMeta
type: object
Syntax:
enum:
- html
- json
- xml
title: Syntax
type: string
Target:
properties:
prefix_pattern:
anyOf:
- type: string
- type: 'null'
default: null
title: Prefix Pattern
pattern:
title: Pattern
type: string
suffix_pattern:
anyOf:
- type: string
- type: 'null'
default: null
title: Suffix Pattern
required:
- pattern
title: Target
type: object
TargetKind:
enum:
- unknown
- aws_access_key_id
- aws_secret_access_key
- github_fine_grained_personal_access_token
- github_personal_access_token
- hostname
- password
- username
title: TargetKind
type: string
properties:
rules:
items:
$ref: '#/$defs/Rule'
title: Rules
type: array
required:
- rules
title: Schema
type: objectUse ./hack/genid to generate new rule IDs.
There is a enum in schema.py that defines the types. As we add new targets that will need to be updated. Please try to match the official terms used by the platform.