From a82a362a594f72bed24b60d6f7bc21e61d2727aa Mon Sep 17 00:00:00 2001 From: Tim Rogers Date: Tue, 19 Dec 2023 16:50:30 +0000 Subject: [PATCH 1/2] Document installing the GitHub Enterprise Importer CLIs as binaries, as an alternative to GitHub CLI extensions (#47360) Co-authored-by: Matt Pollard Co-authored-by: isaacmbrown --- ...m-github-enterprise-server-to-github-enterprise-cloud.md | 4 ++++ ...epositories-from-githubcom-to-github-enterprise-cloud.md | 6 ++++++ ...sitories-from-azure-devops-to-github-enterprise-cloud.md | 4 ++++ ...ries-from-bitbucket-server-to-github-enterprise-cloud.md | 4 ++++ .../ado2gh-binary-generate-script.md | 1 + data/reusables/enterprise-migration-tool/ado2gh-binary.md | 1 + .../bbs2gh-binary-generate-script.md | 1 + data/reusables/enterprise-migration-tool/bbs2gh-binary.md | 1 + .../enterprise-migration-tool/gei-binary-generate-script.md | 1 + data/reusables/enterprise-migration-tool/gei-binary.md | 1 + data/variables/product.yml | 3 +++ 11 files changed, 27 insertions(+) create mode 100644 data/reusables/enterprise-migration-tool/ado2gh-binary-generate-script.md create mode 100644 data/reusables/enterprise-migration-tool/ado2gh-binary.md create mode 100644 data/reusables/enterprise-migration-tool/bbs2gh-binary-generate-script.md create mode 100644 data/reusables/enterprise-migration-tool/bbs2gh-binary.md create mode 100644 data/reusables/enterprise-migration-tool/gei-binary-generate-script.md create mode 100644 data/reusables/enterprise-migration-tool/gei-binary.md diff --git a/content/migrations/using-github-enterprise-importer/migrating-between-github-products/migrating-repositories-from-github-enterprise-server-to-github-enterprise-cloud.md b/content/migrations/using-github-enterprise-importer/migrating-between-github-products/migrating-repositories-from-github-enterprise-server-to-github-enterprise-cloud.md index 2f6e8b5b44e7..d001f70f9805 100644 --- a/content/migrations/using-github-enterprise-importer/migrating-between-github-products/migrating-repositories-from-github-enterprise-server-to-github-enterprise-cloud.md +++ b/content/migrations/using-github-enterprise-importer/migrating-between-github-products/migrating-repositories-from-github-enterprise-server-to-github-enterprise-cloud.md @@ -308,6 +308,8 @@ mutation startRepositoryMigration ( {% data reusables.enterprise-migration-tool.install-gei-extension-intro %} +{% data reusables.enterprise-migration-tool.gei-binary %} + {% data reusables.enterprise-migration-tool.install-github-cli %} {% data reusables.enterprise-migration-tool.install-gei-extension %} @@ -422,6 +424,8 @@ gh gei generate-script --github-source-org SOURCE \ {% data reusables.enterprise-migration-tool.skip-releases %} +{% data reusables.enterprise-migration-tool.gei-binary-generate-script %} + ## Step 6: Migrate repositories {% data reusables.enterprise-migration-tool.migrate-repos-gei %} diff --git a/content/migrations/using-github-enterprise-importer/migrating-between-github-products/migrating-repositories-from-githubcom-to-github-enterprise-cloud.md b/content/migrations/using-github-enterprise-importer/migrating-between-github-products/migrating-repositories-from-githubcom-to-github-enterprise-cloud.md index d4fda4e1a3cc..c1a87794719e 100644 --- a/content/migrations/using-github-enterprise-importer/migrating-between-github-products/migrating-repositories-from-githubcom-to-github-enterprise-cloud.md +++ b/content/migrations/using-github-enterprise-importer/migrating-between-github-products/migrating-repositories-from-githubcom-to-github-enterprise-cloud.md @@ -146,6 +146,8 @@ mutation startRepositoryMigration ( {% data reusables.enterprise-migration-tool.install-gei-extension-intro %} +{% data reusables.enterprise-migration-tool.gei-binary %} + {% data reusables.enterprise-migration-tool.install-github-cli %} {% data reusables.enterprise-migration-tool.install-gei-extension %} @@ -180,12 +182,16 @@ gh gei generate-script --github-source-org SOURCE --github-target-org DESTINATIO {% data reusables.enterprise-migration-tool.generate-script-table %} +{% data reusables.enterprise-migration-tool.gei-binary-generate-script %} + ### Reviewing the migration script {% data reusables.enterprise-migration-tool.review-migration-script %} {% data reusables.enterprise-migration-tool.skip-releases %} +{% data reusables.enterprise-migration-tool.gei-binary-generate-script %} + ## Step 5: Migrate repositories {% data reusables.enterprise-migration-tool.migrate-repos-gei %} diff --git a/content/migrations/using-github-enterprise-importer/migrating-from-azure-devops-to-github-enterprise-cloud/migrating-repositories-from-azure-devops-to-github-enterprise-cloud.md b/content/migrations/using-github-enterprise-importer/migrating-from-azure-devops-to-github-enterprise-cloud/migrating-repositories-from-azure-devops-to-github-enterprise-cloud.md index 04110deb59f0..cbaa76ce5686 100644 --- a/content/migrations/using-github-enterprise-importer/migrating-from-azure-devops-to-github-enterprise-cloud/migrating-repositories-from-azure-devops-to-github-enterprise-cloud.md +++ b/content/migrations/using-github-enterprise-importer/migrating-from-azure-devops-to-github-enterprise-cloud/migrating-repositories-from-azure-devops-to-github-enterprise-cloud.md @@ -146,6 +146,8 @@ mutation startRepositoryMigration ( If this is your first migration, you'll need to install the {% data variables.product.prodname_ado2gh_cli %}. For more information about {% data variables.product.prodname_cli %}, see "[AUTOTITLE](/github-cli/github-cli/about-github-cli)." +{% data reusables.enterprise-migration-tool.ado2gh-binary %} + {% data reusables.enterprise-migration-tool.install-github-cli %} 1. Install the {% data variables.product.prodname_ado2gh_cli_short %}. @@ -211,6 +213,8 @@ To add additional functionality to the script, such as rewiring pipelines, creat {% data reusables.enterprise-migration-tool.review-migration-script %} +{% data reusables.enterprise-migration-tool.ado2gh-binary-generate-script %} + ## Step 5: Migrate repositories You can migrate multiple repositories with a migration script or a single repository with the `gh ado2gh migrate-repo` command. diff --git a/content/migrations/using-github-enterprise-importer/migrating-from-bitbucket-server-to-github-enterprise-cloud/migrating-repositories-from-bitbucket-server-to-github-enterprise-cloud.md b/content/migrations/using-github-enterprise-importer/migrating-from-bitbucket-server-to-github-enterprise-cloud/migrating-repositories-from-bitbucket-server-to-github-enterprise-cloud.md index f06be3171010..af0fe649aa16 100644 --- a/content/migrations/using-github-enterprise-importer/migrating-from-bitbucket-server-to-github-enterprise-cloud/migrating-repositories-from-bitbucket-server-to-github-enterprise-cloud.md +++ b/content/migrations/using-github-enterprise-importer/migrating-from-bitbucket-server-to-github-enterprise-cloud/migrating-repositories-from-bitbucket-server-to-github-enterprise-cloud.md @@ -28,6 +28,8 @@ At this time, migrating from Bitbucket Server with the {% data variables.product If this is your first migration, you'll need to install the {% data variables.product.prodname_bbs2gh_cli %}. For more information about {% data variables.product.prodname_cli %}, see "[AUTOTITLE](/github-cli/github-cli/about-github-cli)." +{% data reusables.enterprise-migration-tool.bbs2gh-binary %} + {% data reusables.enterprise-migration-tool.install-github-cli %} 1. Install the {% data variables.product.prodname_bbs2gh_cli_short %}. @@ -251,6 +253,8 @@ After you generate the script, review the file and, optionally, edit the script. - If there are any repositories you don't want to migrate, delete or comment out the corresponding lines. - By default, repository names in {% data variables.product.prodname_dotcom %} will follow a `projectKey-repositoryName` convention. For example, a Bitbucket Server repository named `airports` that is part of the `open-source` project, which has the key `OS`, would be called `OS-airports` in {% data variables.product.prodname_dotcom %}. If you want any repositories to have a different name on {% data variables.product.prodname_dotcom %}, update the value for the corresponding `--github-repo` flag. +{% data reusables.enterprise-migration-tool.bbs2gh-binary-generate-script %} + ### Running your migration script To migrate your repositories, run the generated script. diff --git a/data/reusables/enterprise-migration-tool/ado2gh-binary-generate-script.md b/data/reusables/enterprise-migration-tool/ado2gh-binary-generate-script.md new file mode 100644 index 000000000000..7bddcab02c35 --- /dev/null +++ b/data/reusables/enterprise-migration-tool/ado2gh-binary-generate-script.md @@ -0,0 +1 @@ +If you downloaded {% data variables.product.prodname_ado2gh %} as a standalone binary rather than as an extension for the {% data variables.product.prodname_cli %}, you will need to update your generated script to run the binary instead of `gh ado2gh`. diff --git a/data/reusables/enterprise-migration-tool/ado2gh-binary.md b/data/reusables/enterprise-migration-tool/ado2gh-binary.md new file mode 100644 index 000000000000..4354907cc8eb --- /dev/null +++ b/data/reusables/enterprise-migration-tool/ado2gh-binary.md @@ -0,0 +1 @@ +Alternatively, you can download a standalone binary from the [releases page](https://github.com/github/gh-ado2gh/releases) for the `github/gh-ado2gh` repository. You can run this binary directly, without the `gh` prefix. diff --git a/data/reusables/enterprise-migration-tool/bbs2gh-binary-generate-script.md b/data/reusables/enterprise-migration-tool/bbs2gh-binary-generate-script.md new file mode 100644 index 000000000000..23b16eccc12f --- /dev/null +++ b/data/reusables/enterprise-migration-tool/bbs2gh-binary-generate-script.md @@ -0,0 +1 @@ +If you downloaded {% data variables.product.prodname_bbs2gh %} as a standalone binary rather than as an extension for the {% data variables.product.prodname_cli %}, you will need to update your generated script to run the binary instead of `gh bbs2gh`. diff --git a/data/reusables/enterprise-migration-tool/bbs2gh-binary.md b/data/reusables/enterprise-migration-tool/bbs2gh-binary.md new file mode 100644 index 000000000000..9f721a3c20ab --- /dev/null +++ b/data/reusables/enterprise-migration-tool/bbs2gh-binary.md @@ -0,0 +1 @@ +Alternatively, you can download a standalone binary from the [releases page](https://github.com/github/gh-bbs2gh/releases) for the `github/gh-bbs2gh` repository. You can run this binary directly, without the `gh` prefix. diff --git a/data/reusables/enterprise-migration-tool/gei-binary-generate-script.md b/data/reusables/enterprise-migration-tool/gei-binary-generate-script.md new file mode 100644 index 000000000000..872010f0554d --- /dev/null +++ b/data/reusables/enterprise-migration-tool/gei-binary-generate-script.md @@ -0,0 +1 @@ +If you downloaded {% data variables.product.prodname_gei %} as a standalone binary rather than as an extension for the {% data variables.product.prodname_cli %}, you will need to update your generated script to run the binary instead of `gh gei`. diff --git a/data/reusables/enterprise-migration-tool/gei-binary.md b/data/reusables/enterprise-migration-tool/gei-binary.md new file mode 100644 index 000000000000..12a3a5fbc928 --- /dev/null +++ b/data/reusables/enterprise-migration-tool/gei-binary.md @@ -0,0 +1 @@ +Alternatively, you can download a standalone binary from the [releases page](https://github.com/github/gh-gei/releases) for the `github/gh-gei` repository. You can run the binary directly, without the `gh` prefix. diff --git a/data/variables/product.yml b/data/variables/product.yml index fc894cdacb58..72a28295ae0c 100644 --- a/data/variables/product.yml +++ b/data/variables/product.yml @@ -37,10 +37,13 @@ prodname_github_connect: 'GitHub Connect' # GitHub Enterprise Importer prodname_importer_proper_name: 'GitHub Enterprise Importer' prodname_importer_secondary_name: Importer +prodname_gei: GEI prodname_gei_cli_short: GEI extension prodname_gei_cli: GEI extension of the GitHub CLI +prodname_ado2gh: ADO2GH prodname_ado2gh_cli: ADO2GH extension of the GitHub CLI prodname_ado2gh_cli_short: ADO2GH extension +prodname_bbs2gh: BBS2GH prodname_bbs2gh_cli: BBS2GH extension of the GitHub CLI prodname_bbs2gh_cli_short: BBS2GH extension From e0e1b483734846a3709acab0b4e74a13d073a5b4 Mon Sep 17 00:00:00 2001 From: Ben Ahmady <32935794+subatoi@users.noreply.github.com> Date: Tue, 19 Dec 2023 16:52:00 +0000 Subject: [PATCH 2/2] Adds content for threat models (#46939) Co-authored-by: James Fletcher <42464962+jf205@users.noreply.github.com> Co-authored-by: Felicity Chapman Co-authored-by: Sam Browning <106113886+sabrowning1@users.noreply.github.com> --- ...g-your-advanced-setup-for-code-scanning.md | 16 +++++++++++++++ ...ing-your-configuration-of-default-setup.md | 20 +++++++++++++++++-- ...analyzing-your-code-with-codeql-queries.md | 19 ++++++++++++++++++ .../codeql-cli-threat-models-java.yml | 6 ++++++ data/features/codeql-threat-models-java.yml | 6 ++++++ .../code-scanning/beta-threat-models-cli.md | 9 +++++++++ .../code-scanning/beta-threat-models.md | 9 +++++++++ 7 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 data/features/codeql-cli-threat-models-java.yml create mode 100644 data/features/codeql-threat-models-java.yml create mode 100644 data/reusables/code-scanning/beta-threat-models-cli.md create mode 100644 data/reusables/code-scanning/beta-threat-models.md diff --git a/content/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.md b/content/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.md index 906e898f1a02..f3f770832b7b 100644 --- a/content/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.md +++ b/content/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.md @@ -335,6 +335,12 @@ If your codebase depends on a library or framework that is not recognized by the {% data reusables.code-scanning.beta-model-packs %} +{% ifversion codeql-threat-models-java %} + +### Using {% data variables.product.prodname_codeql %} model packs + +{% endif %} + To add one or more published {% data variables.product.prodname_codeql %} model packs, specify them inside the `with: packs:` entry within the `uses: {% data reusables.actions.action-codeql-action-init %}` section of the workflow. Within `packs` you specify one or more packages to use and, optionally, which version to download. Where you don't specify a version, the latest version is downloaded. If you want to use packages that are not publicly available, you need to set the `GITHUB_TOKEN` environment variable to a secret that has access to the packages. For more information, see "[AUTOTITLE](/actions/security-guides/automatic-token-authentication)" and "[AUTOTITLE](/actions/security-guides/encrypted-secrets)." ``` yaml copy @@ -545,6 +551,16 @@ packs: {% endraw %} {% endif %} +{% ifversion codeql-threat-models-java %} + +### Extending {% data variables.product.prodname_codeql %} coverage with threat models + +{% data reusables.code-scanning.beta-threat-models %} + +The default threat model includes remote sources of untrusted data. You can extend the {% data variables.product.prodname_codeql %} threat model to include local sources of untrusted data (for example: command-line arguments, environment variables, file systems, and databases) by specifying `threat-models: local` in a custom configuration file. If you extend the threat model, the default threat model will also be used. + +{% endif %} + ### Specifying additional queries You specify additional queries in a `queries` array. Each element of the array contains a `uses` parameter with a value that identifies a single query file, a directory containing query files, or a query suite definition file. diff --git a/content/code-security/code-scanning/managing-your-code-scanning-configuration/editing-your-configuration-of-default-setup.md b/content/code-security/code-scanning/managing-your-code-scanning-configuration/editing-your-configuration-of-default-setup.md index ce280f81b1b4..048111f8a335 100644 --- a/content/code-security/code-scanning/managing-your-code-scanning-configuration/editing-your-configuration-of-default-setup.md +++ b/content/code-security/code-scanning/managing-your-code-scanning-configuration/editing-your-configuration-of-default-setup.md @@ -15,7 +15,9 @@ topics: After running an initial analysis of your code with default setup, you may need to make changes to your configuration to better meet your code security needs. For existing configurations of default setup, you can edit{% ifversion code-scanning-without-workflow-310 %}: - Which languages default setup will analyze. -- {% endif %} The query suite run during analysis. For more information on the available query suites, see "[AUTOTITLE](/code-security/code-scanning/managing-your-code-scanning-configuration/codeql-query-suites)." +- {% endif %} The query suite run during analysis. For more information on the available query suites, see "[AUTOTITLE](/code-security/code-scanning/managing-your-code-scanning-configuration/codeql-query-suites)."{% ifversion codeql-threat-models-java %} +- The threat models (beta) to use for analysis. Your choice of threat model determines which sources of tainted data are treated as a risk to your application. During the beta, threat models are supported only by Java analysis. For more information about threat models, see "[Including local sources of tainted data in default setup](#including-local-sources-of-tainted-data-in-default-setup)." +{% endif %} {% ifversion codeql-model-packs-java %} @@ -35,7 +37,9 @@ If you need to change any other aspects of your {% data variables.product.prodna 1. In the "{% data variables.product.prodname_codeql %} analysis" row of the "{% data variables.product.prodname_code_scanning_caps %}" section, select {% octicon "kebab-horizontal" aria-label="Menu" %}, then click {% octicon "gear" aria-hidden="true" %} **View {% data variables.product.prodname_codeql %} configuration**. 1. In the "{% data variables.product.prodname_codeql %} default configuration" window, click {% octicon "pencil" aria-hidden="true" %} **Edit**. 1. Optionally, in the "Languages" section, select or deselect languages for analysis. -1. Optionally, in the "Query suites" section, select a different query suite to run against your code. +1. Optionally, in the "Query suite" row of the "Scan settings" section, select a different query suite to run against your code.{% ifversion codeql-threat-models-java %} +1. (Beta) Optionally, in the "Threat model" row of the "Scan settings" section, select **Remote and local sources**. +{% endif %} 1. To update your configuration, as well as run an initial analysis of your code with the new configuration, click **Save changes**. All future analyses will use your new configuration. {% else %} @@ -50,6 +54,18 @@ If you need to change any other aspects of your {% data variables.product.prodna 1. To update your configuration of default setup, click **Enable {% data variables.product.prodname_codeql %}**. {% endif %} +{% ifversion codeql-threat-models-java %} + +## Including local sources of tainted data in default setup + +{% data reusables.code-scanning.beta-threat-models %} + +If your codebase only considers remote network requests to be potential sources of tainted data, then we recommend using the default threat model. If your codebase considers sources other than network requests to potentially contain tainted data, then you can use threat models to add these additional sources to your {% data variables.product.prodname_codeql %} analysis. During the beta, you can add local sources (for example: command-line arguments, environment variables, file systems, and databases) that your codebase may consider to be additional sources of tainted data. + +You can edit the threat model used in a default setup configuration. For more information, see "[Customizing your existing configuration of default setup](#customizing-your-existing-configuration-of-default-setup)." + +{% endif %} + {% ifversion codeql-model-packs-java %} ## Extending {% data variables.product.prodname_codeql %} coverage with {% data variables.product.prodname_codeql %} model packs in default setup diff --git a/content/code-security/codeql-cli/getting-started-with-the-codeql-cli/analyzing-your-code-with-codeql-queries.md b/content/code-security/codeql-cli/getting-started-with-the-codeql-cli/analyzing-your-code-with-codeql-queries.md index 98cee11b2199..7e21626253b5 100644 --- a/content/code-security/codeql-cli/getting-started-with-the-codeql-cli/analyzing-your-code-with-codeql-queries.md +++ b/content/code-security/codeql-cli/getting-started-with-the-codeql-cli/analyzing-your-code-with-codeql-queries.md @@ -75,6 +75,7 @@ You must specify ``, `--format`, and `--output`. You can specify addit | --download | {% octicon "x" aria-label="Optional" %} | Use if some of your {% data variables.product.prodname_codeql %} query packs are not yet on disk and need to be downloaded before running queries.{% endif %} | --threads | {% octicon "x" aria-label="Optional" %} | Use if you want to use more than one thread to run queries. The default value is `1`. You can specify more threads to speed up query execution. To set the number of threads to the number of logical processors, specify `0`. | --verbose | {% octicon "x" aria-label="Optional" %} | Use to get more detailed information about the analysis process and diagnostic data from the database creation process. +| --threat-model | {% octicon "x" aria-label="Optional" %} | (Beta) Use to add threat models to configure additional sources in your {% data variables.product.prodname_codeql %} analysis. During the beta, threat models are supported only by Java analysis. For more information, see "[AUTOTITLE](/code-security/codeql-cli/codeql-cli-manual/database-analyze#--threat-modelname)." | {% note %} @@ -296,6 +297,24 @@ For more information, see "[AUTOTITLE](/code-security/codeql-cli/using-the-advan For information about creating custom query suites, see "[AUTOTITLE](/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/creating-codeql-query-suites)." +{% ifversion codeql-cli-threat-models-java %} + +### Including model packs to add potential sources of tainted data + +{% data reusables.code-scanning.beta-threat-models-cli %} + +You can configure threat models in a {% data variables.product.prodname_code_scanning %} analysis. For more information, see "[Customizing library models for Java and Kotlin](https://codeql.github.com/docs/codeql-language-guides/customizing-library-models-for-java-and-kotlin)" in the {% data variables.product.prodname_codeql %} documentation. + +```shell +$ codeql database analyze /codeql-dbs/my-company --format=sarif-latest \ + --threat-model=local \ + --output=/temp/my-company.sarif codeql/java-queries +``` + +In this example, the relevant queries in the standard query pack `codeql/java-queries` will use the `local` threat model as well as the default threat model for `remote` dataflow sources. You should use the `local` threat model if you consider data from local sources (for example: file systems, command-line arguments, databases, and environment variables) to be potential sources of tainted data for your codebase. + +{% endif %} + ## Results You can save analysis results in a number of different formats, including SARIF and CSV. diff --git a/data/features/codeql-cli-threat-models-java.yml b/data/features/codeql-cli-threat-models-java.yml new file mode 100644 index 000000000000..e84844e20160 --- /dev/null +++ b/data/features/codeql-cli-threat-models-java.yml @@ -0,0 +1,6 @@ +# Reference: #12431. +# Documentation for CodeQL threat models for CodeQL CLI +versions: + fpt: '*' + ghec: '*' + ghes: '>=3.12' diff --git a/data/features/codeql-threat-models-java.yml b/data/features/codeql-threat-models-java.yml new file mode 100644 index 000000000000..000ec5acc597 --- /dev/null +++ b/data/features/codeql-threat-models-java.yml @@ -0,0 +1,6 @@ +# Reference: #12431. +# Documentation for CodeQL threat models +versions: + fpt: '*' + ghec: '*' + ghes: '>=3.13' diff --git a/data/reusables/code-scanning/beta-threat-models-cli.md b/data/reusables/code-scanning/beta-threat-models-cli.md new file mode 100644 index 000000000000..70b0ac92ba06 --- /dev/null +++ b/data/reusables/code-scanning/beta-threat-models-cli.md @@ -0,0 +1,9 @@ +{% ifversion codeql-cli-threat-models-java %} + +{% note %} + +**Note:** Threat models are currently in beta and subject to change. During the beta, threat models are supported only by Java analysis. + +{% endnote %} + +{% endif %} diff --git a/data/reusables/code-scanning/beta-threat-models.md b/data/reusables/code-scanning/beta-threat-models.md new file mode 100644 index 000000000000..f5d7dc2578b4 --- /dev/null +++ b/data/reusables/code-scanning/beta-threat-models.md @@ -0,0 +1,9 @@ +{% ifversion codeql-threat-models-java %} + +{% note %} + +**Note:** Threat models are currently in beta and subject to change. During the beta, threat models are supported only by Java analysis. + +{% endnote %} + +{% endif %}