description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent
  plugins.
processors:
- set:
    field: event.ingested
    value: '{{_ingest.timestamp}}'
- rename:
    field: message
    target_field: event.original
- grok:
    field: event.original
    patterns:
    - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address})
      - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}"
      %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long}
      "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})"
    pattern_definitions:
      NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?
      NGINX_NOTSEPARATOR: "[^\t ,:]+"
      NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))*
    ignore_missing: true
- grok:
    field: nginx.access.info
    patterns:
    - '%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}'
    - ""
    ignore_missing: true
- uri_parts:
    field: _tmp.url_orig
    ignore_failure: true
- set:
    field: url.domain
    value: "{{destination.domain}}"
    if: ctx.url?.domain == null && ctx.destination?.domain != null
- remove:
    field:
      - nginx.access.info
      - _tmp.url_orig
    ignore_missing: true
- split:
    field: nginx.access.remote_ip_list
    separator: '"?,?\s+'
    ignore_missing: true
- split:
    field: nginx.access.origin
    separator: '"?,?\s+'
    ignore_missing: true
- set:
    field: source.address
    if: ctx.source?.address == null
    value: ""
- script:
    if: ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.access.remote_ip_list.length > 0
    lang: painless
    source: >-
      boolean isPrivate(def dot, def ip) {
        try {
          StringTokenizer tok = new StringTokenizer(ip, dot);
          int firstByte = Integer.parseInt(tok.nextToken());
          int secondByte = Integer.parseInt(tok.nextToken());
          if (firstByte == 10) {
            return true;
          }
          if (firstByte == 192 && secondByte == 168) {
            return true;
          }
          if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) {
            return true;
          }
          if (firstByte == 127) {
            return true;
          }
          return false;
        }
        catch (Exception e) {
          return false;
        }
      }
      try {
        ctx.source.address = null;
        if (ctx.nginx.access.remote_ip_list == null) {
          return;
        }
        def found = false;
        for (def item : ctx.nginx.access.remote_ip_list) {
          if (!isPrivate(params.dot, item)) {
            ctx.source.address = item;
            found = true;
            break;
          }
        }
        if (!found) {
          ctx.source.address = ctx.nginx.access.remote_ip_list[0];
        }
      }
      catch (Exception e) {
        ctx.source.address = null;
      }
    params:
      dot: .
- remove:
    field: source.address
    if: ctx.source.address == null
- grok:
    field: source.address
    patterns:
    - ^%{IP:source.ip}$
    ignore_failure: true
- set:
    copy_from: '@timestamp'
    field: event.created
- date:
    field: nginx.access.time
    target_field: '@timestamp'
    formats:
    - dd/MMM/yyyy:H:m:s Z
    on_failure:
    - append:
        field: error.message
        value: '{{ _ingest.on_failure_message }}'
- remove:
    field: nginx.access.time
- user_agent:
    field: user_agent.original
    ignore_missing: true
- geoip:
    field: source.ip
    target_field: source.geo
    ignore_missing: true
- geoip:
    database_file: GeoLite2-ASN.mmdb
    field: source.ip
    target_field: source.as
    properties:
    - asn
    - organization_name
    ignore_missing: true
- rename:
    field: source.as.asn
    target_field: source.as.number
    ignore_missing: true
- rename:
    field: source.as.organization_name
    target_field: source.as.organization.name
    ignore_missing: true
- set:
    field: event.kind
    value: event
- append:
    field: event.category
    value: web
- append:
    field: event.type
    value: access
- set:
    field: event.outcome
    value: success
    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
- set:
    field: event.outcome
    value: failure
    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"
- append:
    field: related.ip
    value: "{{source.ip}}"
    if: "ctx?.source?.ip != null"
- append:
    field: related.ip
    value: "{{destination.ip}}"
    if: "ctx?.destination?.ip != null"
- append:
    field: related.user
    value: "{{user.name}}"
    if: "ctx?.user?.name != null"
- script:
    lang: painless
    description: This script processor iterates over the whole document to remove fields with null values.
    source: |
      void handleMap(Map map) {
        for (def x : map.values()) {
          if (x instanceof Map) {
              handleMap(x);
          } else if (x instanceof List) {
              handleList(x);
          }
        }
        map.values().removeIf(v -> v == null);
      }
      void handleList(List list) {
        for (def x : list) {
            if (x instanceof Map) {
                handleMap(x);
            } else if (x instanceof List) {
                handleList(x);
            }
        }
      }
      handleMap(ctx);
on_failure:
- set:
    field: error.message
    value: '{{ _ingest.on_failure_message }}'
