Fortinet black logo

New Features

Enhance the DLP backend and configurations

Copy Link
Copy Doc ID 77966226-6996-11ec-bdf2-fa163e15d75b:871992
Download PDF

Enhance the DLP backend and configurations

The DLP backend has been enhanced to use Hyperscan to perform a one-parse algorithm for scanning multiple patterns. This allows DLP to scale up without any performance downgrade.

DLP configurations have been improved and changed in the following ways:

  • Separate DLP settings into data type, dictionary, sensor, and profile configurations.
  • Add DLP data type that includes five pre-defined data types to match for keyword, regex, hex, credit card, and social security number (SSN). Custom data types can be added.
    config dlp data-type
        edit "keyword"
            set pattern "built-in"
        next
        edit "regex"
            set pattern "built-in"
        next
        edit "hex"
            set pattern "built-in"
        next
        edit "credit-card"
            set pattern "\\b([2-6]{1}\\d{3})[- ]?(\\d{4})[- ]?(\\d{2})[- ]?(\\d{2})[- ]?(\\d{2,4})\\b"
            set verify "built-in"
            set look-back 20
            set transform "\\b\\1[- ]?\\2[- ]?\\3[- ]?\\4[- ]?\\5\\b"
        next
        edit "ssn-us"
            set pattern "\\b(\\d{3})-(\\d{2})-(\\d{4})\\b"
            set verify "(?<!-)\\b(?!666|000|9\\d{2})\\d{3}-(?!00)\\d{2}-(?!0{4})\\d{4}\\b(?!-)"
            set look-back 12
            set transform "\\b\\1-\\2-\\3\\b"
        next
    end
  • Add DLP dictionary (config dlp dictionary), which is a collection of data type entries.
    config dlp dictionary
        edit <name>
            config entries
                edit 1
                    set type {credit-card | hex | keyword | regex | ssn-us}
                    set pattern <string>
                    set repeat {enable | disable}
                    set status {enable | disable}
                next
            end
        next
    end
    
  • Add new DLP sensor (config dlp sensor), which defines which dictionary to check. It counts the number of dictionary matches to trigger the sensor.
    config dlp sensor
        edit <name>
            set match-type {match-all | match-any | match-eval}
            set eval <string>
            config entries
                edit <id>
                    set dictionary <dlp_dictionary>
                    set count <integer>
                    set status {enable | disable}
                next
            end
        next
    end
    
  • Rename config dlp sensor to config dlp profile. DLP profiles allow filtering by size and file type.
    config dlp profile
        edit <name>
            set feature-set {flow | proxy}
            config rule
                edit <id>
                    set proto <protocol> <protocol> ...
                    set sensor <dlp_sensor>
                    set action {allow | log-only | block | quarantine-ip}
                next
            end
        next
    end
  • Allow DLP profiles to be applied in firewall policies.
To add a custom DLP data type:
config dlp data-type
    edit <name>
        set pattern <string>
        set verify <string>
        set transform <string>
    next
end

pattern <string>

Enter a regular expression pattern string without a look around.

verify <string>

Enter a regular expression pattern string used to verify the data type.

transform <string>

Enter the template to transform user input to a pattern using the capture group from pattern.

Example 1

This configuration will block HTTPS upload traffic that includes credit card or social security number (SSN) information. The pre-defined data types for credit-card and ssn-us are used in the dictionary.

To block HTTPS upload traffic that includes credit card or SSN information:
  1. Configure the DLP dictionary:
    config dlp dictionary
        edit "dic-case1-cc-ssn"
            config entries
                edit 1
                    set type "credit-card"
                next
                edit 2
                    set type "ssn-us"
                next
            end
        next
    end
  2. Configure the DLP sensor:
    config dlp sensor
        edit "sensor-case1-cc-ssn"
            config entries
                edit 1
                    set dictionary "dic-case1-cc-ssn"
                next
            end
        next
    end
  3. Configure the DLP profile:
    config dlp profile
        edit "profile-case1-cc-ssn"
            config rule
                edit 1
                    set proto http-post
                    set sensor "sensor-case1-cc-ssn"
                    set action block
                next
            end
        next
    end
  4. Add the DLP profile to a firewall policy:
    config firewall policy
        edit 1
            set srcintf "port2"
            set dstintf "port1"
            set action accept
            set srcaddr "all"
            set dstaddr "all"
            set srcaddr6 "all"
            set dstaddr6 "all"
            set schedule "always"
            set service "ALL"
            set utm-status enable
            set inspection-mode proxy
            set ssl-ssh-profile "custom-deep-inspection"
            set dlp-profile "profile-case1-cc-ssn"
            set logtraffic all
            set nat enable
        next
    end

When a credit card or SSN is included in HTTP POST traffic, a replacement message appears because it is blocked. A DLP log is generated.

Sample log
5: date=2022-02-15 time=09:49:04 eventtime=1644947344512841971 tz="-0800" logid="0954024576" type="utm" subtype="dlp" eventtype="dlp" level="warning" vd="root" filteridx=1 dlpextra="sensor-case1-cc-ssn " filtertype="rule" filtercat="file" severity="medium" policyid=1 poluuid="905fb604-7ed4-51ec-0853-79e498591bf8" policytype="policy" sessionid=9290 epoch=64494265 eventid=0 srcip=10.1.100.106 srcport=64006 srccountry="Reserved" srcintf="port2" srcintfrole="undefined" srcuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" dstip=35.209.241.59 dstport=443 dstcountry="United States" dstintf="port1" dstintfrole="undefined" dstuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" proto=6 service="HTTPS" filetype="unknown" direction="outgoing" action="block" hostname="dlptest.com" url="https://dlptest.com/https-post/" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KH" filename="item_meta[6]" filesize=19 profile="profile-case1-cc-ssn"

Example 2

This configuration will log FTP upload traffic with the following patterns:

  • keyword = demo
  • regex = demo(regex){1,5}
  • hex = e6b58be8af95

The dictionary entries have repeat match enabled. The DLP sensor is set so this is repeated five times.

To log FTP upload traffic that has specific keyword, regex, and hex patterns repeated for five times:
  1. Configure the DLP dictionary:
    config dlp dictionary
        edit "dic-case2-keyword-regex-hex"
            config entries
                edit 1
                    set type "keyword"
                    set pattern "demo"
                    set repeat enable
                next
                edit 2
                    set type "regex"
                    set pattern "demo(regex){1,5}"
                    set repeat enable
                next
                edit 3
                    set type "hex"
                    set pattern "e6b58be8af95"
                    set repeat enable
                next
            end
        next
    end
  2. Configure the DLP sensor:
    config dlp sensor
        edit "sensor-case2-keyword-regex-hex"
            config entries
                edit 1
                    set dictionary "dic-case2-keyword-regex-hex"
                    set count 5
                next
            end
        next
    end
  3. Configure the DLP profile:
    config dlp profile
        edit "profile-case2-keyword-regex-hex"
            config rule
                edit 1
                    set proto ftp
                    set sensor "sensor-case2-keyword-regex-hex"
                    set action log-only
                next
            end
        next
    end
  4. Add the DLP profile to a firewall policy:
    config firewall policy
        edit 1
            set srcintf "port2"
            set dstintf "port1"
            set action accept
            set srcaddr "all"
            set dstaddr "all"
            set srcaddr6 "all"
            set dstaddr6 "all"
            set schedule "always"
            set service "ALL"
            set utm-status enable
            set inspection-mode proxy
            set ssl-ssh-profile "custom-deep-inspection"
            set dlp-profile "profile-case2-keyword-regex-hex"
            set logtraffic all
            set nat enable
        next
    end
  5. Upload a Word document that contains "demo, demo, demo, demoregexregex," using FTP.

A DLP log is generated after the FTP traffic passes.

Sample log
3: date=2022-02-15 time=10:42:34 eventtime=1644950554735620032 tz="-0800" logid="0954024577" type="utm" subtype="dlp" eventtype="dlp" level="notice" vd="root" filteridx=1 dlpextra="sensor-case2-keyword-regex-hex " filtertype="rule" filtercat="file" severity="medium" policyid=1 poluuid="905fb604-7ed4-51ec-0853-79e498591bf8" policytype="policy" sessionid=10551 epoch=64494633 eventid=0 srcip=10.1.100.106 srcport=55647 srccountry="Reserved" srcintf="port2" srcintfrole="undefined" srcuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" dstip=35.163.228.146 dstport=1048 dstcountry="United States" dstintf="port1" dstintfrole="undefined" dstuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" proto=6 service="FTP" filetype="msofficex" direction="outgoing" action="log-only" filename="dlp-test.docx" filesize=11627 profile="profile-case2-keyword-regex-hex" infectedfilename="word/document.xml" infectedfilesize=2448 infectedfiletype="html" infectedfilelevel=1

Example 3

This configuration will block HTTPS downloads of EXE files and log HTTPS downloads of files larger than 500 KB.

To block HTTPS download of EXE files and log downloads larger than 500 KB:
  1. Configure the DLP file pattern:
    config dlp filepattern
        edit 3
            set name "case3-exe"
            config entries
                edit "exe"
                    set filter-type type
                    set file-type exe
                next
            end
        next
    end
  2. Configure the DLP profile:
    config dlp profile
        edit "profile-case3-type-size"
            config rule
                edit 1
                    set proto http-get
                    set filter-by none
                    set file-type 3
                    set action block
                next
                edit 2
                    set proto http-get
                    set filter-by none
                    set file-size 500
                    set action log-only
                next
            end
        next
    end
  3. Add the DLP profile to a firewall policy:
    config firewall policy
        edit 1
            set srcintf "port2"
            set dstintf "port1"
            set action accept
            set srcaddr "all"
            set dstaddr "all"
            set srcaddr6 "all"
            set dstaddr6 "all"
            set schedule "always"
            set service "ALL"
            set utm-status enable
            set inspection-mode proxy
            set ssl-ssh-profile "custom-deep-inspection"
            set dlp-profile "profile-case3-type-size"
            set logtraffic all
            set nat enable
        next
    end
  4. Download an EXE file using HTTPS. The download is blocked, a replacement message appears, and a DLP log is generated.
Sample log
1: date=2022-02-15 time=11:54:29 eventtime=1644954869682887856 tz="-0800" logid="0954024577" type="utm" subtype="dlp" eventtype="dlp" level="notice" vd="root" filteridx=2 dlpextra="500 kB" filtertype="none" filtercat="file" severity="medium" policyid=1 poluuid="905fb604-7ed4-51ec-0853-79e498591bf8" policytype="policy" sessionid=12082 epoch=901683674 eventid=0 srcip=10.1.100.18 srcport=59520 srccountry="Reserved" srcintf="port2" srcintfrole="undefined" srcuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" dstip=51.81.186.201 dstport=443 dstcountry="United States" dstintf="port1" dstintfrole="undefined" dstuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" proto=6 service="HTTPS" direction="incoming" action="log-only" hostname="2.na.dl.wireshark.org" url="https://2.na.dl.wireshark.org/win64/Wireshark-win64-3.6.2.exe" agent="curl/7.61.1" filename="Wireshark-win64-3.6.2.exe" filesize=10502090 profile="profile-case3-type-size"

Enhance the DLP backend and configurations

The DLP backend has been enhanced to use Hyperscan to perform a one-parse algorithm for scanning multiple patterns. This allows DLP to scale up without any performance downgrade.

DLP configurations have been improved and changed in the following ways:

  • Separate DLP settings into data type, dictionary, sensor, and profile configurations.
  • Add DLP data type that includes five pre-defined data types to match for keyword, regex, hex, credit card, and social security number (SSN). Custom data types can be added.
    config dlp data-type
        edit "keyword"
            set pattern "built-in"
        next
        edit "regex"
            set pattern "built-in"
        next
        edit "hex"
            set pattern "built-in"
        next
        edit "credit-card"
            set pattern "\\b([2-6]{1}\\d{3})[- ]?(\\d{4})[- ]?(\\d{2})[- ]?(\\d{2})[- ]?(\\d{2,4})\\b"
            set verify "built-in"
            set look-back 20
            set transform "\\b\\1[- ]?\\2[- ]?\\3[- ]?\\4[- ]?\\5\\b"
        next
        edit "ssn-us"
            set pattern "\\b(\\d{3})-(\\d{2})-(\\d{4})\\b"
            set verify "(?<!-)\\b(?!666|000|9\\d{2})\\d{3}-(?!00)\\d{2}-(?!0{4})\\d{4}\\b(?!-)"
            set look-back 12
            set transform "\\b\\1-\\2-\\3\\b"
        next
    end
  • Add DLP dictionary (config dlp dictionary), which is a collection of data type entries.
    config dlp dictionary
        edit <name>
            config entries
                edit 1
                    set type {credit-card | hex | keyword | regex | ssn-us}
                    set pattern <string>
                    set repeat {enable | disable}
                    set status {enable | disable}
                next
            end
        next
    end
    
  • Add new DLP sensor (config dlp sensor), which defines which dictionary to check. It counts the number of dictionary matches to trigger the sensor.
    config dlp sensor
        edit <name>
            set match-type {match-all | match-any | match-eval}
            set eval <string>
            config entries
                edit <id>
                    set dictionary <dlp_dictionary>
                    set count <integer>
                    set status {enable | disable}
                next
            end
        next
    end
    
  • Rename config dlp sensor to config dlp profile. DLP profiles allow filtering by size and file type.
    config dlp profile
        edit <name>
            set feature-set {flow | proxy}
            config rule
                edit <id>
                    set proto <protocol> <protocol> ...
                    set sensor <dlp_sensor>
                    set action {allow | log-only | block | quarantine-ip}
                next
            end
        next
    end
  • Allow DLP profiles to be applied in firewall policies.
To add a custom DLP data type:
config dlp data-type
    edit <name>
        set pattern <string>
        set verify <string>
        set transform <string>
    next
end

pattern <string>

Enter a regular expression pattern string without a look around.

verify <string>

Enter a regular expression pattern string used to verify the data type.

transform <string>

Enter the template to transform user input to a pattern using the capture group from pattern.

Example 1

This configuration will block HTTPS upload traffic that includes credit card or social security number (SSN) information. The pre-defined data types for credit-card and ssn-us are used in the dictionary.

To block HTTPS upload traffic that includes credit card or SSN information:
  1. Configure the DLP dictionary:
    config dlp dictionary
        edit "dic-case1-cc-ssn"
            config entries
                edit 1
                    set type "credit-card"
                next
                edit 2
                    set type "ssn-us"
                next
            end
        next
    end
  2. Configure the DLP sensor:
    config dlp sensor
        edit "sensor-case1-cc-ssn"
            config entries
                edit 1
                    set dictionary "dic-case1-cc-ssn"
                next
            end
        next
    end
  3. Configure the DLP profile:
    config dlp profile
        edit "profile-case1-cc-ssn"
            config rule
                edit 1
                    set proto http-post
                    set sensor "sensor-case1-cc-ssn"
                    set action block
                next
            end
        next
    end
  4. Add the DLP profile to a firewall policy:
    config firewall policy
        edit 1
            set srcintf "port2"
            set dstintf "port1"
            set action accept
            set srcaddr "all"
            set dstaddr "all"
            set srcaddr6 "all"
            set dstaddr6 "all"
            set schedule "always"
            set service "ALL"
            set utm-status enable
            set inspection-mode proxy
            set ssl-ssh-profile "custom-deep-inspection"
            set dlp-profile "profile-case1-cc-ssn"
            set logtraffic all
            set nat enable
        next
    end

When a credit card or SSN is included in HTTP POST traffic, a replacement message appears because it is blocked. A DLP log is generated.

Sample log
5: date=2022-02-15 time=09:49:04 eventtime=1644947344512841971 tz="-0800" logid="0954024576" type="utm" subtype="dlp" eventtype="dlp" level="warning" vd="root" filteridx=1 dlpextra="sensor-case1-cc-ssn " filtertype="rule" filtercat="file" severity="medium" policyid=1 poluuid="905fb604-7ed4-51ec-0853-79e498591bf8" policytype="policy" sessionid=9290 epoch=64494265 eventid=0 srcip=10.1.100.106 srcport=64006 srccountry="Reserved" srcintf="port2" srcintfrole="undefined" srcuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" dstip=35.209.241.59 dstport=443 dstcountry="United States" dstintf="port1" dstintfrole="undefined" dstuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" proto=6 service="HTTPS" filetype="unknown" direction="outgoing" action="block" hostname="dlptest.com" url="https://dlptest.com/https-post/" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KH" filename="item_meta[6]" filesize=19 profile="profile-case1-cc-ssn"

Example 2

This configuration will log FTP upload traffic with the following patterns:

  • keyword = demo
  • regex = demo(regex){1,5}
  • hex = e6b58be8af95

The dictionary entries have repeat match enabled. The DLP sensor is set so this is repeated five times.

To log FTP upload traffic that has specific keyword, regex, and hex patterns repeated for five times:
  1. Configure the DLP dictionary:
    config dlp dictionary
        edit "dic-case2-keyword-regex-hex"
            config entries
                edit 1
                    set type "keyword"
                    set pattern "demo"
                    set repeat enable
                next
                edit 2
                    set type "regex"
                    set pattern "demo(regex){1,5}"
                    set repeat enable
                next
                edit 3
                    set type "hex"
                    set pattern "e6b58be8af95"
                    set repeat enable
                next
            end
        next
    end
  2. Configure the DLP sensor:
    config dlp sensor
        edit "sensor-case2-keyword-regex-hex"
            config entries
                edit 1
                    set dictionary "dic-case2-keyword-regex-hex"
                    set count 5
                next
            end
        next
    end
  3. Configure the DLP profile:
    config dlp profile
        edit "profile-case2-keyword-regex-hex"
            config rule
                edit 1
                    set proto ftp
                    set sensor "sensor-case2-keyword-regex-hex"
                    set action log-only
                next
            end
        next
    end
  4. Add the DLP profile to a firewall policy:
    config firewall policy
        edit 1
            set srcintf "port2"
            set dstintf "port1"
            set action accept
            set srcaddr "all"
            set dstaddr "all"
            set srcaddr6 "all"
            set dstaddr6 "all"
            set schedule "always"
            set service "ALL"
            set utm-status enable
            set inspection-mode proxy
            set ssl-ssh-profile "custom-deep-inspection"
            set dlp-profile "profile-case2-keyword-regex-hex"
            set logtraffic all
            set nat enable
        next
    end
  5. Upload a Word document that contains "demo, demo, demo, demoregexregex," using FTP.

A DLP log is generated after the FTP traffic passes.

Sample log
3: date=2022-02-15 time=10:42:34 eventtime=1644950554735620032 tz="-0800" logid="0954024577" type="utm" subtype="dlp" eventtype="dlp" level="notice" vd="root" filteridx=1 dlpextra="sensor-case2-keyword-regex-hex " filtertype="rule" filtercat="file" severity="medium" policyid=1 poluuid="905fb604-7ed4-51ec-0853-79e498591bf8" policytype="policy" sessionid=10551 epoch=64494633 eventid=0 srcip=10.1.100.106 srcport=55647 srccountry="Reserved" srcintf="port2" srcintfrole="undefined" srcuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" dstip=35.163.228.146 dstport=1048 dstcountry="United States" dstintf="port1" dstintfrole="undefined" dstuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" proto=6 service="FTP" filetype="msofficex" direction="outgoing" action="log-only" filename="dlp-test.docx" filesize=11627 profile="profile-case2-keyword-regex-hex" infectedfilename="word/document.xml" infectedfilesize=2448 infectedfiletype="html" infectedfilelevel=1

Example 3

This configuration will block HTTPS downloads of EXE files and log HTTPS downloads of files larger than 500 KB.

To block HTTPS download of EXE files and log downloads larger than 500 KB:
  1. Configure the DLP file pattern:
    config dlp filepattern
        edit 3
            set name "case3-exe"
            config entries
                edit "exe"
                    set filter-type type
                    set file-type exe
                next
            end
        next
    end
  2. Configure the DLP profile:
    config dlp profile
        edit "profile-case3-type-size"
            config rule
                edit 1
                    set proto http-get
                    set filter-by none
                    set file-type 3
                    set action block
                next
                edit 2
                    set proto http-get
                    set filter-by none
                    set file-size 500
                    set action log-only
                next
            end
        next
    end
  3. Add the DLP profile to a firewall policy:
    config firewall policy
        edit 1
            set srcintf "port2"
            set dstintf "port1"
            set action accept
            set srcaddr "all"
            set dstaddr "all"
            set srcaddr6 "all"
            set dstaddr6 "all"
            set schedule "always"
            set service "ALL"
            set utm-status enable
            set inspection-mode proxy
            set ssl-ssh-profile "custom-deep-inspection"
            set dlp-profile "profile-case3-type-size"
            set logtraffic all
            set nat enable
        next
    end
  4. Download an EXE file using HTTPS. The download is blocked, a replacement message appears, and a DLP log is generated.
Sample log
1: date=2022-02-15 time=11:54:29 eventtime=1644954869682887856 tz="-0800" logid="0954024577" type="utm" subtype="dlp" eventtype="dlp" level="notice" vd="root" filteridx=2 dlpextra="500 kB" filtertype="none" filtercat="file" severity="medium" policyid=1 poluuid="905fb604-7ed4-51ec-0853-79e498591bf8" policytype="policy" sessionid=12082 epoch=901683674 eventid=0 srcip=10.1.100.18 srcport=59520 srccountry="Reserved" srcintf="port2" srcintfrole="undefined" srcuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" dstip=51.81.186.201 dstport=443 dstcountry="United States" dstintf="port1" dstintfrole="undefined" dstuuid="358d0f56-7ed4-51ec-50f7-a5e4525a641d" proto=6 service="HTTPS" direction="incoming" action="log-only" hostname="2.na.dl.wireshark.org" url="https://2.na.dl.wireshark.org/win64/Wireshark-win64-3.6.2.exe" agent="curl/7.61.1" filename="Wireshark-win64-3.6.2.exe" filesize=10502090 profile="profile-case3-type-size"