Skip to content

Commit

Permalink
Merge branch 'master' into action_alias_publisher
Browse files Browse the repository at this point in the history
  • Loading branch information
cognifloyd authored Feb 3, 2023
2 parents 6c81f4f + 8d2304e commit 76891f3
Show file tree
Hide file tree
Showing 266 changed files with 8,882 additions and 129 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/lint.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ on:
jobs:
# Lint checks which don't depend on any service containes, etc. to be running.
lint-checks:
name: 'Lint Checks (pants runs: shellcheck, bandit, black, flake8)'
name: 'Lint Checks (pants runs: shellcheck, bandit, black, flake8, pylint)'
runs-on: ubuntu-20.04

env:
Expand Down Expand Up @@ -57,7 +57,7 @@ jobs:
./scripts/github/install-apt-packages-use-cache.sh
- name: Initialize Pants and its GHA caches
uses: pantsbuild/actions/init-pants@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb
uses: pantsbuild/actions/init-pants@v2
# This action adds an env var to make pants use both pants.ci.toml & pants.toml.
# This action also creates 3 GHA caches (1 is optional).
# - `pants-setup` has the bootsrapped pants install
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pants.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
submodules: 'true'

- name: Initialize Pants and its GHA caches
uses: pantsbuild/actions/init-pants@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb
uses: pantsbuild/actions/init-pants@v2
# This action adds an env var to make pants use both pants.ci.toml & pants.toml.
# This action also creates 3 GHA caches (1 is optional).
# - `pants-setup` has the bootsrapped pants install
Expand Down
114 changes: 114 additions & 0 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
---
# This Test workflow uses pants
name: Test

on:
push:
branches:
# only on merges to master branch
- master
# and version branches, which only include minor versions (eg: v3.4)
- v[0-9]+.[0-9]+
tags:
# also version tags, which include bugfix releases (eg: v3.4.0)
- v[0-9]+.[0-9]+.[0-9]+
pull_request:
type: [opened, reopened, edited]
branches:
# Only for PRs targeting those branches
- master
- v[0-9]+.[0-9]+
#schedule:
# # run every night at midnight
# - cron: '0 0 * * *'

jobs:
test:
name: '${{ matrix.name }} - Python ${{ matrix.python-version-short }}'
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
# NOTE: We need to use full Python version as part of Python deps cache key otherwise
# setup virtualenv step will fail.
include:
- name: 'Test (pants runs: pytest)'
python-version-short: '3.6'
python-version: '3.6.13'
- name: 'Test (pants runs: pytest)'
python-version-short: '3.8'
python-version: '3.8.10'

services:
mongo:
image: mongo:4.4
ports:
- 27017:27017

env:
COLUMNS: '120'

steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
# a test uses a submodule, and pants needs access to it to calculate deps.
submodules: 'true'

- name: 'Set up Python (${{ matrix.python-version }})'
uses: actions/setup-python@v2
with:
python-version: '${{ matrix.python-version }}'


#- name: Cache APT Dependencies
# id: cache-apt-deps
# uses: actions/cache@v2
# with:
# path: |
# ~/apt_cache
# key: ${{ runner.os }}-apt-v7-${{ hashFiles('scripts/github/apt-packages.txt') }}
# restore-keys: |
# ${{ runner.os }}-apt-v7-
- name: Install APT Depedencies
env:
CACHE_HIT: 'false' # cache doesn't work
#CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}}
run: |
# install dev dependencies for Python YAML and LDAP packages
# https://github.com/StackStorm/st2-auth-ldap
./scripts/github/install-apt-packages-use-cache.sh
- name: Initialize Pants and its GHA caches
uses: pantsbuild/actions/init-pants@v2
# This action adds an env var to make pants use both pants.ci.toml & pants.toml.
# This action also creates 3 GHA caches (1 is optional).
# - `pants-setup` has the bootsrapped pants install
# - `pants-named-caches` has pip/wheel and PEX caches
# - `pants-lmdb-store` has the fine-grained process cache.
# If we ever use a remote cache, then we can drop this.
# Otherwise, we may need an additional workflow or job to delete old caches
# if they are not expiring fast enough, and we hit the GHA 10GB per repo max.
with:
base-branch: master
# To ignore a bad cache, bump the cache* integer.
gha-cache-key: cache0-py${{ matrix.python-version }}
# This hash should include all of our lockfiles so that the pip/pex caches
# get invalidated on any transitive dependency update.
named-caches-hash: ${{ hashFiles('requirements.txt') }}
# enable the optional lmdb_store cache since we're not using remote caching.
cache-lmdb-store: 'true'

- name: Test
# We do not support running pytest everywhere yet. When we do it will be simply:
# ./pants test ::
# Until then, we need to manually adjust this command line to test what we can.
run: |
./pants test pylint_plugins/:: pants-plugins/::
- name: Upload pants log
uses: actions/upload-artifact@v2
with:
name: pants-log-py${{ matrix.python-version }}
path: .pants.d/pants.log
if: always() # We want the log even on failures.
1 change: 1 addition & 0 deletions ADOPTERS.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ This is an alphabetical list of known [StackStorm](https://stackstorm.com/) adop
* [Adobe](https://www.adobe.com/) - Multinational computer software company. After evaluating both SaltStack and Rundeck, Adobe chose StackStorm towards their journey to self-healing infrastructure. As a result, SRE team could resolve thousands of alerts and fix 70% of the outages automatically without human intervention. [[ DevOpsDays Notes ](https://threadreaderapp.com/thread/1098901714567081984.html)] [[ DevOpsCon Talk ](https://devopscon.io/monitoring-traceability-diagnostics/workflow-engines-our-journey-towards-a-self-healing-infrastructure/)]
* [Bitovi](https://www.bitovi.com/) - Consulting company, implemented an Automation solution based on StackStorm API with HA capabilities and custom UI for a Fortune top 10 organization. [[ Blog ](https://www.bitovi.com/blog/stackstorm-solves-devops-automation-for-enterprise-client)] [[ Case study ](https://stackstorm.com/case-study-bitovi/)]
* [DMM.com](https://dmm-corp.com/en/) - Large content provider in Japan. StackStorm is used in Operations helping to maintain online services and development at scale. [[ Case study ](https://stackstorm.com/case-study-dmm/)]
* [DigitalOcean](https://www.digitalocean.com/about) - DigitalOcean simplifies cloud computing so builders can spend more time creating software that changes the world. Internally, StackStorm is used as a consistent frontend to our numerous operational tools, and it also plays the part of the orchestration and automation engine driving the machine lifecycle of our vast fleet of machines spread across the globe.
* [Dimension Data](https://www.dimensiondata.com/en/about-us) - Global systems integrator and IT services provider, using StackStorm for Datacenter Orchestration as well as Infrastructure, Networking, Security Automation for their large clients and government projects. [[ Case study ](https://stackstorm.com/case-study-dimension-data/)]
* [Encore](https://www.encore.tech/) - Data Center, Cloud Computing, IT solutions company ​leverages StackStorm in enterprise scale IT infrastructure for VM & server provisioning, automation, network diagnosis, configuration and orchestration​ on customers' public and private clouds. [[ Blog ](https://encoretechnologies.github.io/blog/2018/03/stackstorm-changed-our-lives/)] [[ Case study ](https://stackstorm.com/case-study-encore/)]
* [Fastly](https://www.fastly.com) - Edge Cloud Platform, implemented StackStorm as part of a bigger global network automation architecture aimed at providing an interface to network operations and traffic engineering changes triggered both manually or in response to events on hundreds of devices spread across dozens of sites. [[ Blog ](https://www.fastly.com/blog/network-automation-helps-support-worlds-biggest-live-streaming-moments)]
Expand Down
17 changes: 16 additions & 1 deletion BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,24 @@ python_requirements(
"//:reqs#zake",
]
},
# make sure anything that uses st2-auth-ldap gets the st2auth constant
"st2-auth-ldap": {
"dependencies": [
"st2auth/st2auth/backends/constants.py",
]
},
},
)

target(
name="auth_backends",
dependencies=[
"//:reqs#st2-auth-backend-flat-file",
"//:reqs#st2-auth-ldap",
],
)

python_test_utils(
name="test_utils0",
name="test_utils",
skip_pylint=True,
)
13 changes: 10 additions & 3 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,22 @@ in development

Added
~~~~~
* Move `git clone` to `user_home/.st2packs` #5845

* Error on `st2ctl status` when running in Kubernetes. #5851
Contributed by @mamercad

* Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
working on StackStorm, improve our security posture, and improve CI reliability thanks in part
to pants' use of PEX lockfiles. This is not a user-facing addition.
#5778 #5789 #5817 #5795 #5830
#5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850
#5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874
Contributed by @cognifloyd

* Added publisher to ActionAlias to enable streaming ActionAlias create/update/delete events.
Contributed @ubaumann
* Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805

* Added publisher to ActionAlias to enable streaming ActionAlias create/update/delete events. #5763
Contributed by @ubaumann


3.8.0 - November 18, 2022
Expand Down
8 changes: 2 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -460,11 +460,7 @@ generate-api-spec: requirements .generate-api-spec
@echo
@echo "================== Generate openapi.yaml file ===================="
@echo
echo "# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY" > st2common/st2common/openapi.yaml
echo "# Edit st2common/st2common/openapi.yaml.j2 and then run" >> st2common/st2common/openapi.yaml
echo "# make .generate-api-spec" >> st2common/st2common/openapi.yaml
echo "# to generate the final spec file" >> st2common/st2common/openapi.yaml
. $(VIRTUALENV_DIR)/bin/activate; python st2common/bin/st2-generate-api-spec --config-file conf/st2.dev.conf >> st2common/st2common/openapi.yaml
. $(VIRTUALENV_DIR)/bin/activate; python st2common/bin/st2-generate-api-spec --config-file conf/st2.dev.conf > st2common/st2common/openapi.yaml

.PHONY: circle-lint-api-spec
circle-lint-api-spec:
Expand Down Expand Up @@ -568,7 +564,7 @@ clean: .cleanpycs
compilepy3:
@echo "======================= compile ========================"
@echo "------- Compile all .py files (syntax check test - Python 3) ------"
python3 -m compileall -f -q -x 'virtualenv|virtualenv-osx|virtualenv-py3|.tox|.git|.venv-st2devbox|./st2tests/st2tests/fixtures/packs/test' .
python3 -m compileall -f -q -x 'virtualenv|virtualenv-osx|virtualenv-py3|.tox|.git|.venv-st2devbox|./st2tests/st2tests/fixtures/packs/test|./pants-plugins' .

.PHONY: .cleanpycs
.cleanpycs:
Expand Down
4 changes: 2 additions & 2 deletions OWNERS.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ Being part of Technical Steering Committee (TSC) [@StackStorm/maintainers](https
- Puppet, StackStorm Exchange.
* Carlos ([@nzlosh](https://github.com/nzlosh)) <<[email protected]>>
- Packaging, Systems, Chatops, Errbot, Community, Discussions, StackStorm Exchange.
* JP Bourget ([@punkrokk](https://github.com/punkrokk)) <<[email protected]>>
- Systems, deb/rpm, Deployments, Community, StackStorm Exchange, SecOps, CircleCI.
* Khushboo Bhatia ([@khushboobhatia01](https://github.com/khushboobhatia01)), _VMware_ <<[email protected]>>
- StackStorm Core, Workflows.
* Marcel Weinberg ([@winem](https://github.com/winem)), _CoreMedia_ <<[email protected]>>
Expand All @@ -57,6 +55,7 @@ They're not part of the TSC voting process, but appreciated for their contributi
* Anand Patel ([@arms11](https://github.com/arms11)), _VMware_ - Docker, Kubernetes.
* Harsh Nanchahal ([@hnanchahal](https://github.com/hnanchahal)), _Starbucks_ - Core, Docker, Kubernetes.
* Hiroyasu Ohyama ([@userlocalhost](https://github.com/userlocalhost)) - Orquesta, Workflows, st2 Japan Community. [Case Study](https://stackstorm.com/case-study-dmm/).
* Mark Mercado ([@mamercad](https://github.com/mamercad)), _DigitalOcean_ - Ansible, Docker, K8s, StackStorm Exchange. [StackStorm Adoption](https://github.com/StackStorm/st2/pull/5836).
* Rick Kauffman ([@xod442](https://github.com/xod442)), _HPE_ - Community, HOWTOs, Blogs, Publications, Docker.
* Sheshagiri Rao Mallipedhi ([@sheshagiri](https://github.com/sheshagiri)) - Docker, Core, StackStorm Exchange.
* Shital Raut ([@shital-orchestral](https://github.com/shital-orchestral)), _Orchestral.ai_ - Web UI.
Expand Down Expand Up @@ -84,6 +83,7 @@ Thank you, Friends!
* Jinping Han ([@jinpingh](https://github.com/jinpingh)) - ex Stormer. Community, Core, Tests, Pack Dependencies.
* Johan Dahlberg ([@johandahlberg](https://github.com/johandahlberg)) - Using st2 for Bioinformatics/Science project, providing feedback & contributions in Ansible, Community, Workflows. [Case Study](https://stackstorm.com/case-study-scilifelab/).
* Johan Hermansson ([@johanherman](https://github.com/johanherman)) - Using st2 for Bioinformatics/Science project, feedback & contributions in Ansible, Community, Workflows. [Case Study](https://stackstorm.com/case-study-scilifelab/).
* JP Bourget ([@punkrokk](https://github.com/punkrokk)) - Systems, deb/rpm, Deployments, Community, StackStorm Exchange, SecOps, CircleCI. Used ST2 for Security Orchestration.
* Jon Middleton ([@jjm](https://github.com/jjm)) - StackStorm Exchange, Core, Discussions.
* Lakshmi Kannan ([@lakshmi-kannan](https://github.com/lakshmi-kannan)) - early Stormer. Initial Core platform architecture, scalability, reliability, Team Leadership during the project hard times.
* Lindsay Hill ([@LindsayHill](https://github.com/LindsayHill)) - ex StackStorm product manager that made a significant impact building an ecosystem we see today.
Expand Down
59 changes: 59 additions & 0 deletions conf/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
file(
name="st2client_sample_config",
source="st2rc.sample.ini",
)

sample_conf( # defined in pants-plugins/sample_conf
name="st2.conf.sample",
source="st2.conf.sample",
dependencies=[
"tools/config_gen.py",
],
)

file(
name="logrotate",
source="logrotate.conf",
)

file(
name="nginx_sample_config",
source="nginx/st2.conf",
)

file(
name="st2_kvstore_demo_crypto_key",
source="st2_kvstore_demo.crypto.key.json",
)

files(
name="st2_tests_conf",
sources=[
"st2.tests*.conf",
],
dependencies=[
"st2auth/conf:htpasswd",
"st2actions/conf:logging",
"st2reactor/conf:logging",
"st2tests/conf:other_logging_conf",
],
)

file(
name="st2_dev_conf",
source="st2.dev.conf",
dependencies=[
":st2_kvstore_demo_crypto_key",
"st2auth/conf:htpasswd",
"st2actions/conf:logging",
"st2api/conf:logging",
"st2auth/conf:logging",
"st2reactor/conf:logging",
"st2stream/conf:logging",
],
)

file(
name="st2_package_conf",
source="st2.package.conf",
)
3 changes: 3 additions & 0 deletions contrib/chatops/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
pack_metadata(
name="metadata",
)
11 changes: 10 additions & 1 deletion contrib/chatops/tests/BUILD
Original file line number Diff line number Diff line change
@@ -1 +1,10 @@
python_tests()
files(
name="fixtures",
sources=["fixtures/*.json"],
)

python_tests(
name="tests",
dependencies=[":fixtures"],
skip_pylint=True,
)
8 changes: 7 additions & 1 deletion contrib/core/BUILD
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
python_sources()
pack_metadata(
name="metadata",
)

python_requirements(
name="reqs",
Expand All @@ -7,3 +9,7 @@ python_requirements(
# https://github.com/pantsbuild/pants/pull/17390
module_mapping={"mail-parser": ["mailparser"]},
)

python_sources(
dependencies=[":metadata"],
)
14 changes: 13 additions & 1 deletion contrib/core/tests/BUILD
Original file line number Diff line number Diff line change
@@ -1 +1,13 @@
python_tests()
python_tests(
skip_pylint=True,
overrides={
"test_action_sendmail.py": {
"dependencies": [
# contrib/core is symlinked to st2tests/st2tests/fixtures/packs/core
# so pants thinks "mail-parser" is defined twice, making it ambiguous.
# Use contrib/core as the canonical copy.
"contrib/core:reqs#mail-parser",
],
}
},
)
3 changes: 3 additions & 0 deletions contrib/debug/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
pack_metadata(
name="metadata",
)
3 changes: 3 additions & 0 deletions contrib/default/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
pack_metadata(
name="metadata",
)
18 changes: 18 additions & 0 deletions contrib/examples/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
pack_metadata(
name="metadata",
)

# Using `python_requirements()` here results in:
# ">1 target exports this module, so it is ambiguous which to use"
# This error refers to the "requests" module.
# So, we explicitly list deps instead.

python_requirement(
name="beautifulsoup4",
requirements=["beautifulsoup4"],
)

python_requirement(
name="flask",
requirements=["flask"],
)
4 changes: 3 additions & 1 deletion contrib/examples/tests/BUILD
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
python_tests()
python_tests(
skip_pylint=True,
)
3 changes: 3 additions & 0 deletions contrib/hello_st2/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
pack_metadata(
name="metadata",
)
4 changes: 4 additions & 0 deletions contrib/linux/BUILD
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
pack_metadata(
name="metadata",
)

python_requirements(
name="reqs",
)
Loading

0 comments on commit 76891f3

Please sign in to comment.