From a4497f331d0e440d4f2f948e0752131492ece2a4 Mon Sep 17 00:00:00 2001 From: Alan Yan Date: Fri, 16 Oct 2020 20:34:37 -0400 Subject: [PATCH 1/5] bump to using condition sets --- .nightfalldlp/config.json | 11 ++++++----- orb.yml | 2 +- src/jobs/scan.yml | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.nightfalldlp/config.json b/.nightfalldlp/config.json index fbd6e74..26fb391 100644 --- a/.nightfalldlp/config.json +++ b/.nightfalldlp/config.json @@ -1,9 +1,10 @@ { - "detectors": [ - "API_KEY", - "CRYPTOGRAPHIC_KEY" - ], + "maxNumberConcurrentRoutines": 5, "tokenExclusionList": [ - "NF-gGpblN9cXW2ENcDBapUNaw3bPZMgcABs" + "4916-6734-7572-5015", + "4242-4242-4242-4242", + "301-123-4567", + "1-240-925-5721", + "xG0Ct4Wsu3OTcJnE1dFLAQfRgL6b8tIv" ] } diff --git a/orb.yml b/orb.yml index 2a4784a..22347f5 100644 --- a/orb.yml +++ b/orb.yml @@ -9,7 +9,7 @@ jobs: description: | Scan Pull Requests and Commits for sensitive findings. You must set the NIGHTFALL_API_KEY as a CircleCI project environment variable to use this orb. View this orb's source and README for usage instructions. docker: - - image: nightfallai/nightfall_code_scanner:v0.0.7 + - image: nightfallai/nightfall_code_scanner:v1.0.0 parameters: base_branch: default: "" diff --git a/src/jobs/scan.yml b/src/jobs/scan.yml index 68be78c..94bcdfb 100644 --- a/src/jobs/scan.yml +++ b/src/jobs/scan.yml @@ -1,7 +1,7 @@ description: | Scan Pull Requests and Commits for sensitive findings. You must set the NIGHTFALL_API_KEY as a CircleCI project environment variable to use this orb. View this orb's source and README for usage instructions. docker: - - image: 'nightfallai/nightfall_code_scanner:v0.0.7' + - image: 'nightfallai/nightfall_code_scanner:v1.0.0' parameters: event_before: default: "" From 955e4b378839944a509ec7c1f1e2e412aa4265db Mon Sep 17 00:00:00 2001 From: Alan Yan Date: Fri, 16 Oct 2020 20:38:33 -0400 Subject: [PATCH 2/5] add inline condition set --- .nightfalldlp/config.json | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.nightfalldlp/config.json b/.nightfalldlp/config.json index 26fb391..5ee8f92 100644 --- a/.nightfalldlp/config.json +++ b/.nightfalldlp/config.json @@ -1,4 +1,24 @@ { + "conditions": [ + { + "detector": { + "detectorType": "NIGHTFALL_DETECTOR", + "nightfallDetector": "API_KEY", + "displayName": "api key" + }, + "minNumFindings": 1, + "minConfidence": "LIKELY" + }, + { + "detector": { + "detectorType": "NIGHTFALL_DETECTOR", + "nightfallDetector": "CRYPTOGRAPHIC_KEY", + "displayName": "crypto key" + }, + "minNumFindings": 1, + "minConfidence": "LIKELY" + } + ], "maxNumberConcurrentRoutines": 5, "tokenExclusionList": [ "4916-6734-7572-5015", From 489e4c09be224d6cb5576a4140ebde36d5b9b3a5 Mon Sep 17 00:00:00 2001 From: Alan Yan Date: Tue, 20 Oct 2020 17:01:00 -0400 Subject: [PATCH 3/5] update README --- README.md | 213 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 181 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 23b692b..e7b282b 100755 --- a/README.md +++ b/README.md @@ -35,30 +35,9 @@ The Nightfall DLP Orb is powered by the Nightfall DLP API. Learn more and reques **2. Set up config file to specify detectors.** -- Place a `.nightfalldlp/` directory within the root of your target repository, and inside it a `config.json` file in which you can configure your detectors (see `Detectors` section below for more information on Detectors) +- Place a `.nightfalldlp/` directory within the root of your target repository, and inside it a `config.json` file in which you can configure your condition set (see the `Conditions` section below for more information) - See `Additional Configuration` section for more advanced configuration options -- Below is a sample `.nightfalldlp/config.json` file: -```json -{ - "detectors": [ - "CREDIT_CARD_NUMBER", - "PHONE_NUMBER", - "API_KEY", - "CRYPTOGRAPHIC_KEY", - "RANDOMLY_GENERATED_TOKEN", - "US_SOCIAL_SECURITY_NUMBER", - "AMERICAN_BANKERS_CUSIP_ID", - "US_BANK_ROUTING_MICR", - "ICD9_CODE", - "ICD10_CODE", - "US_DRIVERS_LICENSE_NUMBER", - "US_PASSPORT", - "EMAIL_ADDRESS", - "IP_ADDRESS" - ] -} -``` **3. Set up a few environment variables.** These variables should be made available to the Nightfall DLP orb by adding them to `Environment Variables` under `Project Settings` on the CircleCI console. Instructions [here](https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-project): @@ -80,6 +59,7 @@ These variables should be made available to the Nightfall DLP orb by adding them **4. Allow Uncertified Orbs** To use our orb, make sure to allow uncertified orbs in Organization Settings. Instructions are [here](https://circleci.com/docs/2.0/orbs-faq/#using-uncertified-orbs). At this time, CircleCI still requires allowing un-certified orbs to use orbs developed by CircleCI Partners. + ## Supported GitHub Events The Nightfall DLP CircleCI Orb can run in a Circle Workflow triggered by the following events: @@ -88,21 +68,190 @@ The Nightfall DLP CircleCI Orb can run in a Circle Workflow triggered by the fol The Nightfall DLP CircleCI Orb does not currently support forked repositories due to potential permissioning issues that may occur. -## Detectors -Each detector represents a type of information you want to search for in your code scans. A few examples of detectors Nightfall supports includes: +## Nightfalldlp Config File + +The .nightfalldlp/config.json file is used as a centralized config file to control what conditions/detectors to scan with and what content you want to scan for in your pull requests. It includes the following adjustable fields to fit your needs. + +### ConditionSetUUID + +A condition set uuid is a unique identifier of a condition set, which can be created via [app.nightfall.ai](app.nightfall.ai). +Once defined, you can simply input the uuid in the your config file, e.g. + +```json +{ "conditionSetUUID": "A0BA0D76-78B4-4E10-B653-32996060316B" } +``` + +Note: by default, if both conditionSetUUID and conditions are specified, only the conditionSetUUID will be used. + +### Conditions + +Conditions are a list of conditions specified inline. Each condition contains a nested detector object as well as two additional parameters: minNumFindings and minConfidence. -- `API_KEY`: A freeform string used for user verification to access online program functions. -- `CREDIT_CARD_NUMBER`: A 12 to 19 digit number used for payments and other monetary transactions. -- `CRYPTOGRAPHIC_KEY`: A string of characters used by an encryption algorithm to generate seemingly random tokens. -- `RANDOMLY_GENERATED_TOKEN`: A pseudo-random string generated by an encryption algorithm. This detector is more general than the API_KEY detector. -- `US_SOCIAL_SECURITY_NUMBER`: A 9 digit numeric string often used as a unique identification number for United States citizens and residents. -- Many more... +```json +{ + "conditions": [ + { + "minNumFindings": 1, + "minConfidence": "POSSIBLE", + "detector": {} + } + ] +} +``` -**Find a full list of supported detectors in the Nightfall API Documentation, after creating your account (per the instructions above).** +minNumFindings specifies the minimal number of findings required to return for one request, e.g. if you set minNumFindings to be 2, and only 1 finding within the request payload related to that detector, that finding then will be filtered out from the response. + +minConfidence specifies the minimal threshold to trigger a potential finding to be captured. We have five levels of confidence from least sensitive to most sensitive: + +- VERY_LIKELY +- LIKELY +- POSSIBLE +- UNLIKELY +- VERY_UNLIKELY + +### Detector + +A detector is either a prebuilt Nightfall detector or custom regex or wordlist detector that you can create. This is specified by the detectorType field. + +- nightfall prebuilt detector + + ```json + { + "detector": { + "detectorType": "NIGHTFALL_DETECTOR", + "nightfallDetector": "API_KEY", + "displayName": "apiKeyDetector" + } + } + ``` + + Within detector struct + + - First specify detectorType as NIGHTFALL_DETECTOR + - Pick the nightfall detector you want from the list + - API_KEY + - CRYPTOGRAPHIC_KEY + - RANDOMLY_GENERATED_TOKEN + - CREDIT_CARD_NUMBER + - US_SOCIAL_SECURITY_NUMBER + - AMERICAN_BANKERS_CUSIP_ID + - US_BANK_ROUTING_MICR + - ICD9_CODE + - ICD10_CODE + - US_DRIVERS_LICENSE_NUMBER + - US_PASSPORT + - PHONE_NUMBER + - IP_ADDRESS + - EMAIL_ADDRESS + - Put a display name for your detector, as this will be attached on your findings + +- customized regex + + We also support customized regex as a detector, which are defined as followed: + + ```json + { + "detector": { + "detectorType": "REGEX", + "regex": { + "pattern": "coupon:\\d{4,}", + "isCaseSensitive": false + }, + "displayName": "simpleRegexCouponDetector" + } + } + ``` + +- word list + + Word list detectors look for words you specify in its list. Example below: + + ```json + { + "detector": { + "detectorType": "WORD_LIST", + "wordList": { + "values": ["key", "credential"], + "isCaseSensitive": false + }, + "displayName": "simpleWordListKeyDetector" + } + } + ``` + +- [Extra Parameters Within Detector] + + Aside from specifying which detector to use for a condition, you can also specify some additional rules to attach. They are contextRules and exclusionRules. + + - contextRules + A context rule evaluates the surrounding context(pre/post chars) of a finding and adjusts the finding's confidence if the input context rule pattern exists. + + Example: + + ```json + { + "detector": { + // ...... other detector fields + "contextRules": [ + { + "regex": { + "pattern": "my cc", + "isCaseSensitive": true + }, + "proximity": { + "windowBefore": 30, + "windowAfter": 30 + }, + "confidenceAdjustment": { + "fixedConfidence": "VERY_LIKELY" + } + } + ] + } + } + ``` + + - regex field specifies a regex to trigger + - proximity is defined as the number pre|post chars surrounding the finding to conduct the search + - confidenceAdjustment is the confidence level to adjust the finding to upon existence of the input context + + As an example, say we have the following line of text in a file `my cc number: 4242-4242-4242-4242`, and `4242-4242-4242-4242` is detected as a credit card number with confidence of POSSIBLE. If we had the context rule above, the confidence level of this finding will be bumped up to `VERY_LIKELY` because the characters preceding the finding, `my cc`, match the regex. + + - exclusionRules + Exclusion rules on individual conditions are used to mute findings related to that condition's detector. + + Example: + + ```json + { + "detector": { + // ...... other detector fields + "exclusionRules": [ + { + "matchType": "PARTIAL", + "exclusionType": "REGEX", + // specify one of these values based on the type specified above + "regex": { + "pattern": "4242-4242-\\d{4}-\\d{4}", + "isCaseSensitive": true + }, + "wordList": { + "values": ["4242"], + "isCaseSensitive": false + } + } + ] + } + } + ``` + + - exclusionType specifies either a REGEX or WORD_LIST + - regex field specifies a regex to trigger, if you choose to use REGEX type + - matchType could be either PARTIAL or FULL, To be a full match, the entire finding must match the regex pattern or word exactly, whereas findings containing more than just the regex pattern or word are considered partial matches. Example: suppose we have a finding of "4242-4242" with exclusion regex of 4242. If you use PARTIAL, this finding will be deactivated, while FULL not, since the regex only matches partial of the finding ## Additional Configuration -You can add additional fields to your config to ignore tokens and files from being flagged, as well as specify which files to exclusively scan. +You can add additional fields to your Nightfall config to ignore tokens and files from being flagged, as well as specify which files to exclusively scan. ### Token Exclusion From 54b9ab12419ef765fa77d83257b97b14727a2619 Mon Sep 17 00:00:00 2001 From: Alan Yan Date: Tue, 20 Oct 2020 17:17:04 -0400 Subject: [PATCH 4/5] make additional readme updates --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e7b282b..a847834 100755 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ The orb runs when configured as a job in your CircleCI config: version: 2.1 orbs: - nightfall_code_scanner: nightfall/nightfall_code_scanner@v1.0.0 + nightfall_code_scanner: nightfall/nightfall_code_scanner@v2.0.0 workflows: test_scanner: @@ -33,7 +33,7 @@ workflows: The Nightfall DLP Orb is powered by the Nightfall DLP API. Learn more and request access to a free API key **[here](https://nightfall.ai/api/)**. Alternatively, you can email **[sales@nightfall.ai](mailto:sales@nightfall.ai)** to provision a free account. -**2. Set up config file to specify detectors.** +**2. Set up config file to specify condition set.** - Place a `.nightfalldlp/` directory within the root of your target repository, and inside it a `config.json` file in which you can configure your condition set (see the `Conditions` section below for more information) - See `Additional Configuration` section for more advanced configuration options @@ -75,7 +75,7 @@ The .nightfalldlp/config.json file is used as a centralized config file to contr ### ConditionSetUUID A condition set uuid is a unique identifier of a condition set, which can be created via [app.nightfall.ai](app.nightfall.ai). -Once defined, you can simply input the uuid in the your config file, e.g. +Once defined, you can simply input the uuid in your config file, e.g. ```json { "conditionSetUUID": "A0BA0D76-78B4-4E10-B653-32996060316B" } From a92a3d5951279bdfb5eb81069df729777eee7368 Mon Sep 17 00:00:00 2001 From: Alan Yan Date: Tue, 20 Oct 2020 17:23:05 -0400 Subject: [PATCH 5/5] add line about default condition set --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a847834..1e196cd 100755 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ The Nightfall DLP Orb is powered by the Nightfall DLP API. Learn more and reques - Place a `.nightfalldlp/` directory within the root of your target repository, and inside it a `config.json` file in which you can configure your condition set (see the `Conditions` section below for more information) - See `Additional Configuration` section for more advanced configuration options +- In the absence of a Nightfall config file, the scan will be performed with a default condition set containing the `API_KEY` and `CRYPTOGRAPHIC_KEY` detectors will be used. **3. Set up a few environment variables.** These variables should be made available to the Nightfall DLP orb by adding them to `Environment Variables` under `Project Settings` on the CircleCI console. Instructions [here](https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-project):