committing changes in /etc made by "-bash"

Package changes:

crowdsec/hub/parsers/s00-raw/crowdsecurity/syslog-logs.yaml

@@ -0,0 +1,48 @@
+#If it's syslog, we are going to extract progname from it
+filter: "evt.Line.Labels.type == 'syslog'"
+onsuccess: next_stage
+pattern_syntax:
+  RAW_SYSLOG_PREFIX: '^<%{NUMBER:stuff1}>%{NUMBER:stuff2} %{SYSLOGBASE2} %{DATA:program} %{NUMBER:pid}'
+  RAW_SYSLOG_META: '\[meta sequenceId="%{NOTDQUOTE:seq_id}"\]'
+name: crowdsecurity/syslog-logs
+nodes:
+  - grok:
+      #this is a named regular expression. grok patterns can be kept into separate files for readability
+      pattern: "^%{SYSLOGLINE}" 
+      #This is the field of the `Event` to which the regexp should be applied
+      apply_on: Line.Raw
+  - grok:
+      #a second pattern for unparsed syslog lines, as saw in opnsense
+      pattern: '%{RAW_SYSLOG_PREFIX} - %{RAW_SYSLOG_META} %{GREEDYDATA:message}'
+      apply_on: Line.Raw
+#if the node was successfull, statics will be applied.
+statics:
+  - meta: machine
+    expression: evt.Parsed.logsource
+  - parsed: "logsource"
+    value: "syslog"
+# syslog date can be in two different fields (one of hte assignment will fail)
+  - target: evt.StrTime
+    expression: evt.Parsed.timestamp
+  - target: evt.StrTime
+    expression: evt.Parsed.timestamp8601
+  - meta: datasource_path
+    expression: evt.Line.Src
+  - meta: datasource_type
+    expression: evt.Line.Module
+---
+#if it's not syslog, the type is the progname
+filter: "evt.Line.Labels.type != 'syslog'"
+onsuccess: next_stage
+name: crowdsecurity/non-syslog
+#debug: true
+statics:
+  - parsed: message
+    expression: evt.Line.Raw
+  - parsed: program
+    expression: evt.Line.Labels.type
+  - meta: datasource_path
+    expression: evt.Line.Src
+  - meta: datasource_type
+    expression: evt.Line.Module
+

crowdsec/hub/parsers/s01-parse/crowdsecurity/apache2-logs.yaml

@@ -0,0 +1,93 @@
+#Apache access/errors logs
+#debug: true
+filter: "evt.Parsed.program startsWith 'apache2'"
+onsuccess: next_stage
+name: crowdsecurity/apache2-logs
+description: "Parse Apache2 access and error logs"
+#log line can be prefixed by a target_fqdn
+nodes:
+  - grok:
+      pattern: '(%{IPORHOST:target_fqdn}(:%{INT:port})? )?%{COMMONAPACHELOG}( "%{NOTDQUOTE:referrer}" "%{NOTDQUOTE:http_user_agent}")?'
+      apply_on: message
+      # these ones apply for both grok patterns
+      statics:
+        - meta: log_type
+          value: http_access-log
+        - target: evt.StrTime
+          expression: evt.Parsed.timestamp
+        - meta: service
+          value: http
+        - meta: source_ip
+          expression: evt.Parsed.clientip
+        - meta: http_status
+          expression: evt.Parsed.response
+        - meta: http_path
+          expression: evt.Parsed.request
+        - meta: http_verb
+          expression: "evt.Parsed.verb"
+        - meta: http_user_agent
+          expression: "evt.Parsed.http_user_agent"
+        - meta: target_fqdn
+          expression: "evt.Parsed.target_fqdn"
+    onsuccess: next_stage
+  - grok:
+      pattern: '%{HTTPD_ERRORLOG}'
+      apply_on: message
+    onsuccess: next_stage
+    pattern_syntax:
+      NOT_DOUBLE_POINT: '[^:]+'
+      NOT_DOUBLE_QUOTE: '[^"]+'    
+    nodes:
+      - filter: "evt.Parsed.module == 'auth_basic'"
+        onsuccess: next_stage
+        pattern_syntax:
+          EXTRACT_USER_AND_PATH: 'user %{NOT_DOUBLE_POINT:username}: authentication failure for "%{NOT_DOUBLE_QUOTE:target_uri}": Password Mismatch'
+          EXTRACT_USER_AND_PATH2: 'user %{NOT_DOUBLE_POINT:username} not found: "?%{NOT_DOUBLE_QUOTE:target_uri}"?'
+        grok:
+          pattern: '%{EXTRACT_USER_AND_PATH}|%{EXTRACT_USER_AND_PATH2}'
+          apply_on: message
+          # these ones apply for both grok patterns
+        statics:
+          - meta: username
+            expression: evt.Parsed.username
+          - meta: http_path
+            expression: evt.Parsed.target_uri
+          - meta: sub_type
+            value: "auth_fail"
+      - filter: "evt.Parsed.module == 'core' && evt.Parsed.message contains 'Invalid URI'"
+        onsuccess: next_stage
+        pattern_syntax:
+          EXTRACT_URIVERB: 'Invalid URI in request %{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})'
+        grok:
+          pattern: '%{EXTRACT_URIVERB}'
+          apply_on: message
+        statics:
+          - meta: http_path
+            expression: evt.Parsed.request
+          - meta: sub_type
+            value: "invalid_uri"
+      - filter: "evt.Parsed.module == 'authz_core' && evt.Parsed.message contains 'client denied'"
+        onsuccess: next_stage
+        pattern_syntax:
+          EXTRACT_PATH: 'client denied by server configuration: %{GREEDYDATA:target_uri}'
+        grok:
+          pattern: '%{EXTRACT_PATH}'
+          apply_on: message
+        statics:
+          - meta: http_path
+            expression: evt.Parsed.target_uri
+          - meta: sub_type
+            value: "permission_denied"
+    statics:
+      - meta: log_type
+        value: http_error-log
+      - target: evt.StrTime
+        expression: evt.Parsed.timestamp
+      - meta: service
+        value: http
+      - meta: source_ip
+        expression: evt.Parsed.client
+      - meta: http_status
+        expression: evt.Parsed.response
+
+      

crowdsec/hub/parsers/s01-parse/crowdsecurity/mysql-logs.yaml

@@ -0,0 +1,14 @@
+onsuccess: next_stage
+name: crowdsecurity/mysql-logs
+description: "Parse MySQL logs"
+filter: "evt.Parsed.program == 'mysql'"
+grok:
+  pattern: "%{TIMESTAMP_ISO8601:time} %{NUMBER} \\[Note\\]( \\[%{DATA:err_code}\\] \\[%{DATA:subsystem}\\])? Access denied for user '%{DATA:user}'@'%{IP:source_ip}' \\(using password: %{WORD:using_password}\\)"
+  apply_on: message
+statics:
+  - meta: log_type
+    value: mysql_failed_auth
+  - meta: source_ip
+    expression: "evt.Parsed.source_ip"
+  - meta: user
+    expression: "evt.Parsed.user"

crowdsec/hub/parsers/s01-parse/crowdsecurity/nginx-logs.yaml

@@ -0,0 +1,68 @@
+filter: "evt.Parsed.program startsWith 'nginx'"
+onsuccess: next_stage
+name: crowdsecurity/nginx-logs
+description: "Parse nginx access and error logs"
+nodes:
+  - grok:
+      pattern: '(%{IPORHOST:target_fqdn} )?%{IPORHOST:remote_addr} - (%{NGUSER:remote_user})? \[%{HTTPDATE:time_local}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{NOTDQUOTE:http_referer}" "%{NOTDQUOTE:http_user_agent}"( %{NUMBER:request_length} %{NUMBER:request_time} \[%{DATA:proxy_upstream_name}\] \[%{DATA:proxy_alternative_upstream_name}\])?'
+      apply_on: message
+      statics:
+        - meta: log_type
+          value: http_access-log
+        - target: evt.StrTime
+          expression: evt.Parsed.time_local
+  - grok:
+      # and this one the error log
+      pattern: '(%{IPORHOST:target_fqdn} )?%{NGINXERRTIME:time} \[%{LOGLEVEL:loglevel}\] %{NONNEGINT:pid}#%{NONNEGINT:tid}: (\*%{NONNEGINT:cid} )?%{GREEDYDATA:message}, client: %{IPORHOST:remote_addr}, server: %{DATA:target_fqdn}, request: "%{WORD:verb} ([^/]+)?%{URIPATHPARAM:request}( HTTP/%{NUMBER:http_version})?", host: "%{IPORHOST}(:%{NONNEGINT})?"'
+      apply_on: message
+      statics:
+        - meta: log_type
+          value: http_error-log
+        - target: evt.StrTime
+          expression: evt.Parsed.time
+    pattern_syntax:
+      NO_DOUBLE_QUOTE: '[^"]+'
+    onsuccess: next_stage
+    nodes:
+      - filter: "evt.Parsed.message contains 'was not found in'"
+        pattern_syntax:
+          USER_NOT_FOUND: 'user "%{NO_DOUBLE_QUOTE:username}" was not found in "%{NO_DOUBLE_QUOTE}"'
+        grok:
+          pattern: '%{USER_NOT_FOUND}'
+          apply_on: message
+        statics:
+          - meta: sub_type
+            value: "auth_fail"
+          - meta: username
+            expression: evt.Parsed.username
+      - filter: "evt.Parsed.message contains 'password mismatch'"
+        pattern_syntax:
+          PASSWORD_MISMATCH: 'user "%{NO_DOUBLE_QUOTE:username}": password mismatch'
+        grok:
+          pattern: '%{PASSWORD_MISMATCH}'
+          apply_on: message
+        statics:
+          - meta: sub_type
+            value: "auth_fail"
+          - meta: username
+            expression: evt.Parsed.username
+      - filter: "evt.Parsed.message contains 'limiting requests, excess'"
+        statics:
+          - meta: sub_type
+            value: "req_limit_exceeded"
+    # these ones apply for both grok patterns
+statics:
+  - meta: service
+    value: http
+  - meta: source_ip
+    expression: "evt.Parsed.remote_addr"
+  - meta: http_status
+    expression: "evt.Parsed.status"
+  - meta: http_path
+    expression: "evt.Parsed.request"
+  - meta: http_verb
+    expression: "evt.Parsed.verb"
+  - meta: http_user_agent
+    expression: "evt.Parsed.http_user_agent"
+  - meta: target_fqdn
+    expression: "evt.Parsed.target_fqdn"

crowdsec/hub/parsers/s01-parse/crowdsecurity/sshd-logs.yaml

@@ -0,0 +1,93 @@
+onsuccess: next_stage
+#debug: true
+filter: "evt.Parsed.program == 'sshd'"
+name: crowdsecurity/sshd-logs
+description: "Parse openSSH logs"
+pattern_syntax:
+# The IP grok pattern that ships with crowdsec is buggy and does not capture the last digit of an IP if it is the last thing it matches, and the last octet starts with a 2
+# https://github.com/crowdsecurity/crowdsec/issues/938
+  IPv4_WORKAROUND: (?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)
+  IP_WORKAROUND: (?:%{IPV6}|%{IPv4_WORKAROUND})
+  SSHD_AUTH_FAIL: 'pam_%{DATA:pam_type}\(sshd:auth\): authentication failure; logname= uid=%{NUMBER:uid}? euid=%{NUMBER:euid}? tty=ssh ruser= rhost=%{IP_WORKAROUND:sshd_client_ip}( %{SPACE}user=%{USERNAME:sshd_invalid_user})?'
+  SSHD_MAGIC_VALUE_FAILED: 'Magic value check failed \(\d+\) on obfuscated handshake from %{IP_WORKAROUND:sshd_client_ip} port \d+'
+  SSHD_INVALID_USER: 'Invalid user\s*%{USERNAME:sshd_invalid_user}? from %{IP_WORKAROUND:sshd_client_ip}( port \d+)?'
+  SSHD_INVALID_BANNER: 'banner exchange: Connection from %{IP_WORKAROUND:sshd_client_ip} port \d+: invalid format'
+  SSHD_PREAUTH_AUTHENTICATING_USER: 'Connection closed by (authenticating|invalid) user %{USERNAME:sshd_invalid_user} %{IP_WORKAROUND:sshd_client_ip} port \d+ \[preauth\]'
+  #following: https://github.com/crowdsecurity/crowdsec/issues/1201 - some scanners behave differently and trigger this one
+  SSHD_PREAUTH_AUTHENTICATING_USER_ALT: 'Disconnected from (authenticating|invalid) user %{USERNAME:sshd_invalid_user} %{IP_WORKAROUND:sshd_client_ip} port \d+ \[preauth\]'
+nodes:
+  - grok:
+      name: "SSHD_FAIL"
+      apply_on: message
+      statics:
+        - meta: log_type
+          value: ssh_failed-auth
+        - meta: target_user
+          expression: "evt.Parsed.sshd_invalid_user"
+  - grok:
+      name: "SSHD_PREAUTH_AUTHENTICATING_USER_ALT"
+      apply_on: message
+      statics:
+        - meta: log_type
+          value: ssh_failed-auth
+        - meta: target_user
+          expression: "evt.Parsed.sshd_invalid_user"
+  - grok:
+      name: "SSHD_PREAUTH_AUTHENTICATING_USER"
+      apply_on: message
+      statics:
+        - meta: log_type
+          value: ssh_failed-auth
+        - meta: target_user
+          expression: "evt.Parsed.sshd_invalid_user"
+  - grok:
+      name: "SSHD_DISC_PREAUTH"
+      apply_on: message
+  - grok:
+      name: "SSHD_BAD_VERSION"
+      apply_on: message
+  - grok:
+      name: "SSHD_INVALID_USER"
+      apply_on: message
+      statics:
+        - meta: log_type
+          value: ssh_failed-auth
+        - meta: target_user
+          expression: "evt.Parsed.sshd_invalid_user"
+  - grok:
+      name: "SSHD_INVALID_BANNER"
+      apply_on: message
+      statics:
+        - meta: log_type
+          value: ssh_failed-auth
+        - meta: extra_log_type
+          value: ssh_bad_banner
+  - grok:
+      name: "SSHD_USER_FAIL"
+      apply_on: message
+      statics:
+        - meta: log_type
+          value: ssh_failed-auth
+        - meta: target_user
+          expression: "evt.Parsed.sshd_invalid_user"
+  - grok: 
+      name: "SSHD_AUTH_FAIL"
+      apply_on: message
+      statics:
+        - meta: log_type
+          value: ssh_failed-auth
+        - meta: target_user
+          expression: "evt.Parsed.sshd_invalid_user"
+  - grok: 
+      name: "SSHD_MAGIC_VALUE_FAILED"
+      apply_on: message
+      statics:
+        - meta: log_type
+          value: ssh_failed-auth
+        - meta: target_user
+          expression: "evt.Parsed.sshd_invalid_user"
+statics:
+    - meta: service
+      value: ssh
+    - meta: source_ip
+      expression: "evt.Parsed.sshd_client_ip"

crowdsec/hub/parsers/s02-enrich/crowdsecurity/dateparse-enrich.yaml

@@ -0,0 +1,11 @@
+filter: "evt.StrTime != ''"
+name: crowdsecurity/dateparse-enrich
+#debug: true
+#it's a hack lol
+statics:
+  - method: ParseDate
+    expression: evt.StrTime
+  - target: MarshaledTime
+    expression: evt.Enriched.MarshaledTime
+  - meta: timestamp
+    expression: evt.Enriched.MarshaledTime

crowdsec/hub/parsers/s02-enrich/crowdsecurity/geoip-enrich.yaml

@@ -0,0 +1,27 @@
+filter: "'source_ip' in evt.Meta"
+name: crowdsecurity/geoip-enrich
+description: "Populate event with geoloc info : as, country, coords, source range."
+data:
+  - source_url: https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/GeoLite2-City.mmdb
+    dest_file: GeoLite2-City.mmdb
+  - source_url: https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/GeoLite2-ASN.mmdb
+    dest_file: GeoLite2-ASN.mmdb
+statics:
+  - method: GeoIpCity
+    expression: evt.Meta.source_ip
+  - meta: IsoCode
+    expression: evt.Enriched.IsoCode
+  - meta: IsInEU
+    expression: evt.Enriched.IsInEU
+  - meta: GeoCoords
+    expression: evt.Enriched.GeoCoords
+  - method: GeoIpASN
+    expression: evt.Meta.source_ip
+  - meta: ASNNumber
+    expression: evt.Enriched.ASNNumber
+  - meta: ASNOrg
+    expression: evt.Enriched.ASNOrg
+  - method: IpToRange
+    expression: evt.Meta.source_ip
+  - meta: SourceRange
+    expression: evt.Enriched.SourceRange

crowdsec/hub/parsers/s02-enrich/crowdsecurity/http-logs.yaml

@@ -0,0 +1,33 @@
+filter: "evt.Meta.service == 'http' && evt.Meta.log_type in ['http_access-log', 'http_error-log']"
+description: "Parse more Specifically HTTP logs, such as HTTP Code, HTTP path, HTTP args and if its a static ressource"
+name: crowdsecurity/http-logs
+pattern_syntax:
+  DIR: "^.*/"
+  FILE: "[^/].*?"
+  EXT: "\\.[^.]*$|$"
+nodes:
+  - statics:
+     - parsed: "impact_completion"
+       # the value of a field can as well be determined as the result of an expression
+       expression: "evt.Meta.http_status in ['404', '403', '502'] ? 'false' : 'true'"
+     - target: evt.Parsed.static_ressource
+       value: 'false'
+  # let's split the path?query if possible
+  - grok:
+      pattern: "^%{GREEDYDATA:request}\\?%{GREEDYDATA:http_args}$"
+      apply_on: request
+  # this is another node, with its own pattern_syntax
+  - #debug: true
+    grok:
+      pattern: "%{DIR:file_dir}(%{FILE:file_frag}%{EXT:file_ext})?"
+      apply_on: request
+      statics:
+        - meta: http_path
+          expression: "evt.Parsed.http_path"
+          # meta af
+        - meta: http_args_len
+          expression: "len(evt.Parsed.http_args)"
+        - parsed: file_name
+          expression: evt.Parsed.file_frag + evt.Parsed.file_ext
+        - parsed: static_ressource
+          expression: "Upper(evt.Parsed.file_ext) in ['.JPG', '.CSS', '.JS', '.JPEG', '.PNG', '.SVG', '.MAP', '.ICO', '.OTF', '.GIF', '.MP3', '.MP4', '.WOFF', '.WOFF2', '.TTF', '.OTF', '.EOT', '.WEBP', '.WAV', '.GZ', '.BROTLI', '.BVR', '.TS', '.BMP'] ? 'true' : 'false'"

crowdsec/hub/parsers/s02-enrich/crowdsecurity/whitelists.yaml

@@ -0,0 +1,14 @@
+name: crowdsecurity/whitelists
+description: "Whitelist events from private ipv4 addresses"
+whitelist:
+  reason: "private ipv4/ipv6 ip/ranges"
+  ip: 
+    - "127.0.0.1"
+    - "::1"
+  cidr:
+    - "192.168.0.0/16"
+    - "10.0.0.0/8"
+    - "172.16.0.0/12"
+  # expression:
+  #   - "'foo.com' in evt.Meta.source_ip.reverse" 
+