-
Notifications
You must be signed in to change notification settings - Fork 475
892 lines (836 loc) · 42.9 KB
/
build.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
name: Build and test GAM
on:
push:
pull_request:
schedule:
- cron: '37 22 * * *'
permissions:
contents: read
id-token: write
defaults:
run:
shell: bash
working-directory: src
env:
OPENSSL_CONFIG_OPTS: no-fips --api=3.0.0
OPENSSL_INSTALL_PATH: ${{ github.workspace }}/bin/ssl
OPENSSL_SOURCE_PATH: ${{ github.workspace }}/src/openssl
PYTHON_INSTALL_PATH: ${{ github.workspace }}/bin/python
PYTHON_SOURCE_PATH: ${{ github.workspace }}/src/cpython
jobs:
build:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-20.04
jid: 1
goal: build
arch: x86_64
openssl_archs: linux-x86_64
- os: [self-hosted, linux, arm64]
jid: 2
goal: build
arch: aarch64
openssl_archs: linux-aarch64
- os: ubuntu-20.04
jid: 3
goal: build
arch: x86_64
openssl_archs: linux-x86_64
staticx: yes
- os: [self-hosted, linux, arm64]
jid: 4
goal: build
arch: aarch64
openssl_archs: linux-aarch64
staticx: yes
- os: macos-12
jid: 5
goal: build
arch: x86_64
openssl_archs: darwin64-x86_64
- os: [self-hosted, macOS, ARM64]
jid: 7
goal: build
arch: aarch64
openssl_archs: darwin64-arm64
- os: [self-hosted, macOS, ARM64]
jid: 12
goal: build
arch: universal2
openssl_archs: darwin64-arm64 darwin64-x86_64
- os: windows-2022
jid: 6
goal: build
arch: Win64
openssl_archs: VC-WIN64A
- os: ubuntu-22.04
goal: test
python: "3.8"
jid: 9
arch: x86_64
- os: ubuntu-22.04
goal: test
python: "3.9"
jid: 10
arch: x86_64
- os: ubuntu-22.04
goal: test
python: "3.10"
jid: 11
arch: x86_64
- os: ubuntu-22.04
goal: test
python: "3.11"
jid: 12
arch: x86_64
steps:
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 0
- id: auth
name: Authenticate to Google Cloud
uses: google-github-actions/auth@v1
with:
workload_identity_provider: projects/297925809119/locations/global/workloadIdentityPools/gha-pool/providers/gha-provider
service_account: github-actions-testing-for-gam@gam-project-wyo-lub-ivl.iam.gserviceaccount.com
- name: Cache multiple paths
if: matrix.goal == 'build'
uses: actions/cache@v3
id: cache-python-ssl
with:
path: |
bin.tar.xz
src/cpython
key: gam-${{ matrix.jid }}-202311118
- name: Untar Cache archive
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit == 'true'
working-directory: ${{ github.workspace }}
run: |
tar xvvf bin.tar.xz
- name: Use pre-compiled Python for testing
if: matrix.python != ''
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
allow-prereleases: true
- name: common variables for all runs
env:
arch: ${{ matrix.arch }}
JID: ${{ matrix.jid }}
ACTIONS_CACHE: ${{ steps.cache-python-ssl.outputs.cache-hit }}
ACTIONS_GOAL: ${{ matrix.goal }}
run: |
echo "arch=${arch}" >> $GITHUB_ENV
echo "JID=${JID}" >> $GITHUB_ENV
echo "ACTIONS_CACHE=${ACTIONS_CACHE}" >> $GITHUB_ENV
echo "ACTIONS_GOAL=${ACTIONS_GOAL}" >> $GITHUB_ENV
curl_version=$(curl --version | head -n 1 | awk '{ print $2 }')
echo "cURL is ${curl_version}"
if [ "$curl_version" == "7.68.0" ]; then
export curl_retry="--retry 5 --retry-connrefused"
else
export curl_retry="--retry 5 --retry-all-errors"
fi
echo "curl_retry=${curl_retry}" >> $GITHUB_ENV
# GAMCFGDIR should be recreated on every run
GAMCFGDIR="${RUNNER_TEMP}/.gam"
if [ "$arch" == "Win64" ]; then
GAMCFGDIR=$(cygpath -u "$GAMCFGDIR")
fi
echo "GAMCFGDIR=${GAMCFGDIR}" >> $GITHUB_ENV
echo "GAMCFGDIR is: ${GAMCFGDIR}"
- name: Set env variables for test
if: matrix.goal == 'test'
run: |
export PYTHON=$(which python3)
export PIP=$(which pip3)
export gam="${PYTHON} -m gam"
export gampath="$(readlink -e .)"
echo -e "PYTHON: ${PYTHON}\nPIP: ${PIP}\gam: ${gam}\ngampath: ${gampath}"
echo "PYTHON=${PYTHON}" >> $GITHUB_ENV
echo "PIP=${PIP}" >> $GITHUB_ENV
echo "gam=${gam}" >> $GITHUB_ENV
echo "gampath=${gampath}" >> $GITHUB_ENV
- name: Install necessary Github-hosted Linux packages
if: runner.os == 'Linux' && runner.arch == 'X64'
run: |
echo "RUNNING: apt update..."
sudo apt-get -qq --yes update
sudo apt-get -qq --yes install swig libpcsclite-dev libxslt1-dev
- name: MacOS install tools
if: runner.os == 'macOS' && runner.arch == 'x86_64'
run: |
# Install latest Rust
curl $curl_retry -fsS -o rust.sh https://sh.rustup.rs
bash ./rust.sh -y
source $HOME/.cargo/env
# needed for Rust to compile cryptography Python package for universal2
# not needed since MacOS ARM and universal will be on self-hosted
#rustup target add aarch64-apple-darwin
- name: Windows Configure VCode
uses: ilammy/msvc-dev-cmd@v1
if: runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
with:
arch: ${{ matrix.arch }}
- name: Set Env Variables for build
if: matrix.goal == 'build'
env:
openssl_archs: ${{ matrix.openssl_archs }}
staticx: ${{ matrix.staticx }}
run: |
echo "We are running on ${RUNNER_OS}"
LD_LIBRARY_PATH="${OPENSSL_INSTALL_PATH}/lib:${PYTHON_INSTALL_PATH}/lib:/usr/local/lib"
if [[ "${arch}" == "Win64" ]]; then
PYEXTERNALS_PATH="amd64"
PYBUILDRELEASE_ARCH="x64"
GAM_ARCHIVE_ARCH="x86_64"
WIX_ARCH="x64"
CHOC_OPS=""
elif [[ "${arch}" == "Win32" ]]; then
PYEXTERNALS_PATH="win32"
PYBUILDRELEASE_ARCH="Win32"
GAM_ARCHIVE_ARCH="x86"
WIX_ARCH="x86"
CHOC_OPS="--forcex86"
fi
if [[ "${RUNNER_OS}" == "macOS" ]]; then
#brew install coreutils
#brew install bash
MAKE=make
MAKEOPT="-j$(sysctl -n hw.logicalcpu)"
PERL=perl
echo "MACOSX_DEPLOYMENT_TARGET=10.15" >> $GITHUB_ENV
echo "PYTHON=${PYTHON_INSTALL_PATH}/bin/python3" >> $GITHUB_ENV
#echo "PIP_ARGS=--no-binary=:all:" >> $GITHUB_ENV
elif [[ "${RUNNER_OS}" == "Linux" ]]; then
MAKE=make
MAKEOPT="-j$(nproc)"
PERL=perl
echo "PYTHON=${PYTHON_INSTALL_PATH}/bin/python3" >> $GITHUB_ENV
elif [[ "${RUNNER_OS}" == "Windows" ]]; then
MAKE=nmake
MAKEOPT=""
PERL="c:\strawberry\perl\bin\perl.exe"
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${PYTHON_SOURCE_PATH}/PCbuild/${PYEXTERNALS_PATH}"
echo "PYTHON=${PYTHON_SOURCE_PATH}/PCbuild/${PYEXTERNALS_PATH}/python.exe" >> $GITHUB_ENV
echo "GAM_ARCHIVE_ARCH=${GAM_ARCHIVE_ARCH}" >> $GITHUB_ENV
echo "WIX_ARCH=${WIX_ARCH}" >> $GITHUB_ENV
fi
echo "We'll run make with: ${MAKEOPT}"
echo "staticx=${staticx}" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
echo "MAKE=${MAKE}" >> $GITHUB_ENV
echo "MAKEOPT=${MAKEOPT}" >> $GITHUB_ENV
echo "PERL=${PERL}" >> $GITHUB_ENV
echo "PYEXTERNALS_PATH=${PYEXTERNALS_PATH}" >> $GITHUB_ENV
echo "PYBUILDRELEASE_ARCH=${PYBUILDRELEASE_ARCH}" >> $GITHUB_ENV
echo "openssl_archs=${openssl_archs}" >> $GITHUB_ENV
- name: Get latest stable OpenSSL source
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
mkdir -vp "${GITHUB_WORKSPACE}/src"
cd "${GITHUB_WORKSPACE}/src"
git clone https://github.com/openssl/openssl.git
cd "${OPENSSL_SOURCE_PATH}"
export LATEST_STABLE_TAG=$(git tag --list openssl-* | grep -v alpha | grep -v beta | sort -Vr | head -n1)
echo "Checking out version ${LATEST_STABLE_TAG}"
git checkout "${LATEST_STABLE_TAG}"
export COMPILED_OPENSSL_VERSION=${LATEST_STABLE_TAG:8} # Trim the openssl- prefix
echo "COMPILED_OPENSSL_VERSION=${COMPILED_OPENSSL_VERSION}" >> $GITHUB_ENV
if ([ "${RUNNER_OS}" == "macOS" ] && [ "$arch" == "universal2" ]); then
for openssl_arch in $openssl_archs; do
ssldir="${OPENSSL_SOURCE_PATH}-${openssl_arch}"
mkdir -v "${ssldir}"
cp -vrf ${OPENSSL_SOURCE_PATH}/* "${ssldir}/"
done
rm -vrf "${OPENSSL_SOURCE_PATH}"
else
mv -v "${OPENSSL_SOURCE_PATH}" "${OPENSSL_SOURCE_PATH}-${openssl_archs}"
fi
- name: Windows NASM Install
uses: ilammy/setup-nasm@v1
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
- name: Config OpenSSL
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
for openssl_arch in $openssl_archs; do
cd "${GITHUB_WORKSPACE}/src/openssl-${openssl_arch}"
# --libdir=lib is needed so Python can find OpenSSL libraries
"${PERL}" ./Configure "${openssl_arch}" --libdir=lib --prefix="${OPENSSL_INSTALL_PATH}" $OPENSSL_CONFIG_OPTS
done
- name: Rename GNU link on Windows
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
shell: bash
run: mv /usr/bin/link /usr/bin/gnulink
- name: Make OpenSSL
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
for openssl_arch in $openssl_archs; do
cd "${GITHUB_WORKSPACE}/src/openssl-${openssl_arch}"
$MAKE "${MAKEOPT}"
done
- name: Install OpenSSL
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
if ([ "${RUNNER_OS}" == "macOS" ] && [ "$arch" == "universal2" ]); then
for openssl_arch in $openssl_archs; do
cd "${GITHUB_WORKSPACE}/src/openssl-${openssl_arch}"
# install_sw saves us ages processing man pages :-)
$MAKE install_sw
mv "${OPENSSL_INSTALL_PATH}" "${GITHUB_WORKSPACE}/bin/ssl-${openssl_arch}"
done
mkdir -vp "${OPENSSL_INSTALL_PATH}/lib"
mkdir -vp "${OPENSSL_INSTALL_PATH}/bin"
for archlib in libcrypto.3.dylib libssl.3.dylib libcrypto.a libssl.a; do
lipo -create "${GITHUB_WORKSPACE}/bin/ssl-darwin64-x86_64/lib/${archlib}" \
"${GITHUB_WORKSPACE}/bin/ssl-darwin64-arm64/lib/${archlib}" \
-output "${GITHUB_WORKSPACE}/bin/ssl/lib/${archlib}"
done
mv ${GITHUB_WORKSPACE}/bin/ssl-darwin64-x86_64/include ${GITHUB_WORKSPACE}/bin/ssl/
lipo -create "${GITHUB_WORKSPACE}/bin/ssl-darwin64-x86_64/bin/openssl" \
"${GITHUB_WORKSPACE}/bin/ssl-darwin64-arm64/bin/openssl" \
-output "${GITHUB_WORKSPACE}/bin/ssl/bin/openssl"
rm -rf ${GITHUB_WORKSPACE}/bin/ssl-darwin64-x86_64
rm -rf ${GITHUB_WORKSPACE}/bin/ssl-darwin64-arm64
echo "LDFLAGS=-L${OPENSSL_INSTALL_PATH}/lib" >> $GITHUB_ENV
echo "CRYPTOGRAPHY_SUPPRESS_LINK_FLAGS=1" >> $GITHUB_ENV
echo "CFLAGS=-I${OPENSSL_INSTALL_PATH}/include -arch arm64 -arch x86_64 ${CFLAGS}" >> $GITHUB_ENV
echo "ARCHFLAGS=-arch x86_64 -arch arm64" >> $GITHUB_ENV
else
cd "${GITHUB_WORKSPACE}/src/openssl-${openssl_archs}"
# install_sw saves us ages processing man pages :-)
$MAKE install_sw
fi
- name: Run OpenSSL
if: matrix.goal == 'build'
run: |
"${OPENSSL_INSTALL_PATH}/bin/openssl" version
"${OPENSSL_INSTALL_PATH}/bin/openssl" version -f
file "${OPENSSL_INSTALL_PATH}/bin/openssl"
- name: Get latest stable Python source
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
cd "${GITHUB_WORKSPACE}/src"
git clone https://github.com/python/cpython.git
cd "${PYTHON_SOURCE_PATH}"
# Pin Windows to 3.11.6 for the moment
# if [[ "${RUNNER_OS}" == "Windows" ]]; then
# export LATEST_STABLE_TAG="v3.11.6"
# else
export LATEST_STABLE_TAG=$(git tag --list | grep -v a | grep -v rc | grep -v b | sort -Vr | head -n1)
# fi
git checkout "${LATEST_STABLE_TAG}"
export COMPILED_PYTHON_VERSION=${LATEST_STABLE_TAG:1} # Trim the "v" prefix
echo "COMPILED_PYTHON_VERSION=${COMPILED_PYTHON_VERSION}" >> $GITHUB_ENV
- name: Mac/Linux Configure Python
if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
cd "${PYTHON_SOURCE_PATH}"
if ([ "${RUNNER_OS}" == "macOS" ] && [ "$arch" == "universal2" ]); then
extra_args=( "--enable-universalsdk" "--with-universal-archs=universal2" )
else
extra_args=( )
fi
./configure --with-openssl="${OPENSSL_INSTALL_PATH}" \
--prefix="${PYTHON_INSTALL_PATH}" \
--enable-shared \
--with-ensurepip=upgrade \
--enable-optimizations \
--with-lto \
"${extra_args[@]}"
- name: Windows Get External Python deps
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
shell: powershell
run: |
cd "${env:PYTHON_SOURCE_PATH}"
PCBuild\get_externals.bat
- name: Windows overwrite external OpenSSL with local
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
shell: powershell
run: |
cd "${env:PYTHON_SOURCE_PATH}"
$env:OPENSSL_EXT_PATH = "$(Get-Item externals\openssl-bin-* | Select -exp FullName)\"
echo "External OpenSSL was downloaded to ${env:OPENSSL_EXT_PATH}"
Remove-Item -recurse -force "${env:OPENSSL_EXT_PATH}*"
# Emulate what this script does:
# https://github.com/python/cpython/blob/main/PCbuild/openssl.vcxproj
$env:OPENSSL_EXT_TARGET_PATH = "${env:OPENSSL_EXT_PATH}${env:PYEXTERNALS_PATH}"
echo "Copying our OpenSSL to ${env:OPENSSL_EXT_TARGET_PATH}"
mkdir "${env:OPENSSL_EXT_TARGET_PATH}\include\openssl\"
Copy-Item -Path "${env:GITHUB_WORKSPACE}/src/openssl-${env:openssl_archs}\LICENSE.txt" -Destination "${env:OPENSSL_EXT_TARGET_PATH}\LICENSE" -Verbose
cp -v "$env:OPENSSL_INSTALL_PATH\lib\*" "${env:OPENSSL_EXT_TARGET_PATH}"
cp -v "$env:OPENSSL_INSTALL_PATH\bin\*" "${env:OPENSSL_EXT_TARGET_PATH}"
cp -v "$env:OPENSSL_INSTALL_PATH\include\openssl\*" "${env:OPENSSL_EXT_TARGET_PATH}\include\openssl\"
cp -v "$env:OPENSSL_INSTALL_PATH\include\openssl\applink.c" "${env:OPENSSL_EXT_TARGET_PATH}\include\"
- name: Windows Install sphinx-build
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
shell: powershell
run: |
pip install --upgrade pip
pip install --upgrade sphinx
sphinx-build --version
- name: Windows Config/Build Python
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
shell: powershell
run: |
cd "${env:PYTHON_SOURCE_PATH}"
# We need out custom openssl.props which uses OpenSSL 3 DLL names
Copy-Item -Path "${env:GITHUB_WORKSPACE}\src\tools\openssl.props" -Destination PCBuild\ -Verbose
echo "Building for ${env:PYBUILDRELEASE_ARCH}..."
PCBuild\build.bat -m --pgo -c Release -p "${env:PYBUILDRELEASE_ARCH}"
- name: Mac/Linux Build Python
if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
cd "${PYTHON_SOURCE_PATH}"
echo "Running: ${MAKE} ${MAKEOPT}"
$MAKE $MAKEOPT
- name: Mac/Linux Install Python
if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
cd "${PYTHON_SOURCE_PATH}"
$MAKE altinstall
$MAKE bininstall
export PATH="${PATH}:${PYTHON_INSTALL_PATH}/bin"
echo "PATH=${PATH}" >> $GITHUB_ENV
echo "PATH: ${PATH}"
- name: Run Python
run: |
"${PYTHON}" -V
- name: Upgrade pip, wheel, etc
run: |
curl $curl_retry -O https://bootstrap.pypa.io/get-pip.py
"${PYTHON}" get-pip.py
"${PYTHON}" -m pip install --upgrade pip
"${PYTHON}" -m pip install --upgrade wheel
"${PYTHON}" -m pip install --upgrade setuptools
- name: Install pip requirements
run: |
echo "before anything..."
"${PYTHON}" -m pip list
if ([ "${RUNNER_OS}" == "macOS" ] && [ "$arch" == "universal2" ]); then
# cffi is a dep of cryptography and doesn't ship
# a universal2 wheel so we must build one ourself :-/
export CFLAGS="-arch x86_64 -arch arm64"
export ARCHFLAGS="-arch x86_64 -arch arm64"
"${PYTHON}" -m pip install --upgrade --force-reinstall --no-binary :all: \
--no-cache-dir --no-deps --use-pep517 \
--use-feature=no-binary-enable-wheel-cache \
cffi
echo "before cryptography..."
"${PYTHON}" -m pip list
# cryptography has a universal2 wheel but getting it installed
# on x86-64 MacOS is a royal pain in the keester.
"${PYTHON}" -m pip download --only-binary :all: \
--dest . \
--no-cache \
--no-deps \
--platform macosx_10_15_universal2 \
cryptography
"${PYTHON}" -m pip install --force-reinstall --no-deps cryptography*.whl
echo "after cryptography..."
"${PYTHON}" -m pip list
"${PYTHON}" -m pip install --upgrade --no-binary :all: -r requirements.txt
else
"${PYTHON}" -m pip install --upgrade -r requirements.txt
echo "after requirements..."
"${PYTHON}" -m pip list
"${PYTHON}" -m pip install --force-reinstall --no-deps --upgrade cryptography
fi
echo "after everything..."
"${PYTHON}" -m pip list
- name: Install PyInstaller
if: matrix.goal == 'build'
run: |
git clone https://github.com/pyinstaller/pyinstaller.git
cd pyinstaller
export latest_release=$(git tag --list | grep -v dev | grep -v rc | sort -Vr | head -n1)
#V6.0.0 causes errors on staticx
if [[ "${staticx}" == "yes" ]]; then
git checkout "v5.13.2"
elif [[ "${RUNNER_OS}" == "Windows" ]]; then
git checkout "v5.13.2"
elif [[ "${RUNNER_OS}" == "macOS" ]]; then
git checkout "v5.13.2"
else
git checkout "${latest_release}"
fi
# remove pre-compiled bootloaders so we fail if bootloader compile fails
rm -rvf PyInstaller/bootloader/*-*/*
cd bootloader
case "${arch}" in
"Win32")
export PYINSTALLER_BUILD_ARGS="--target-arch=32bit"
;;
"Win64")
export PYINSTALLER_BUILD_ARGS="--target-arch=64bit"
;;
esac
echo "PyInstaller build arguments: ${PYINSTALLER_BUILD_ARGS}"
"${PYTHON}" ./waf all $PYINSTALLER_BUILD_ARGS
cd ..
echo "---- Installing PyInstaller ----"
"${PYTHON}" -m pip install .
- name: Build GAM with PyInstaller
if: matrix.goal != 'test'
run: |
if [[ "${staticx}" == "yes" ]]; then
export distpath="./dist/gam"
export gampath="${distpath}"
else
export distpath="./dist"
export gampath="${distpath}/gam"
fi
mkdir -p -v "${gampath}"
if [[ "${RUNNER_OS}" == "macOS" ]]; then
export gampath=$($PYTHON -c "import os; print(os.path.realpath('$gampath'))")
elif [[ "${RUNNER_OS}" == "Windows" ]]; then
# Work around issue where PyInstaller picks up python3.dll from other Python versions
# https://github.com/pyinstaller/pyinstaller/issues/7102
export PATH="/usr/bin"
else
export gampath=$(realpath "${gampath}")
fi
export gam="${gampath}/gam"
echo "gampath=${gampath}" >> $GITHUB_ENV
echo "gam=${gam}" >> $GITHUB_ENV
echo -e "GAM: ${gam}\nGAMPATH: ${gampath}"
# TEMP force everything back to one file.
export PYINSTALLER_BUILD_ONEFILE="yes"
export distpath="./dist/gam"
export gampath="${distpath}"
"${PYTHON}" -m PyInstaller --clean --noconfirm --distpath="${distpath}" gam.spec
cat build/gam/warn-gam.txt
- name: Copy extra package files
if: matrix.goal == 'build'
run: |
cp -v cacerts.pem $gampath
cp -v LICENSE $gampath
cp -v GamCommands.txt $gampath
cp -v GamUpdate.txt $gampath
if [[ "${RUNNER_OS}" == "Windows" ]]; then
cp -v gam-setup.bat $gampath
fi
- name: Install StaticX
if: matrix.staticx == 'yes'
run: |
"${PYTHON}" -m pip install --upgrade patchelf-wrapper
"${PYTHON}" -m pip install --upgrade staticx
- name: Make StaticX
if: matrix.staticx == 'yes'
run: |
case $RUNNER_ARCH in
X64)
ldlib=/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
;;
ARM64)
ldlib=/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1
;;
esac
echo "ldlib=${ldlib}"
$PYTHON -m staticx -l "${ldlib}" "${gam}" "${gam}-staticx"
rm -v "${gam}"
mv -v "${gam}-staticx" "${gam}"
- name: Basic Tests all jobs
id: basictests
run: |
$PYTHON -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer || if [ $? != 5 ]; then exit $?; fi # exit 5 is no tests
$gam version extended nooffseterror
export GAMVERSION=$($gam version simple)
echo "GAM Version ${GAMVERSION}"
echo "GAMVERSION=${GAMVERSION}" >> $GITHUB_ENV
- name: Linux/MacOS package
if: runner.os != 'Windows' && matrix.goal == 'build'
run: |
if [[ "${RUNNER_OS}" == "macOS" ]]; then
GAM_ARCHIVE="gam-${GAMVERSION}-macos-${arch}.tar.xz"
elif [[ "${RUNNER_OS}" == "Linux" ]]; then
if [[ "${staticx}" == "yes" ]]; then
libver="legacy"
else
libver="glibc$(ldd --version | awk '/ldd/{print $NF}')"
fi
GAM_ARCHIVE="gam-${GAMVERSION}-linux-$(arch)-$libver}.tar.xz"
fi
tar -C dist/ --create --verbose --exclude-from "${GITHUB_WORKSPACE}/.github/actions/package_exclusions.txt" --file $GAM_ARCHIVE --xz gam
- name: Windows package
if: runner.os == 'Windows' && matrix.goal != 'test'
run: |
cd dist/
GAM_ARCHIVE="../gam-${GAMVERSION}-windows-${GAM_ARCHIVE_ARCH}.zip"
/c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam "-xr@${GITHUB_WORKSPACE}/.github/actions/package_exclusions.txt" -bb3
cd ..
/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch "${WIX_ARCH}" gam.wxs
/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/light.exe -ext /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/WixUIExtension.dll gam.wixobj -o "gam-${GAMVERSION}-windows-${GAM_ARCHIVE_ARCH}.msi" || true;
rm -v -f *.wixpdb
- name: Basic Tests build jobs only
if: matrix.goal != 'test' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
export voutput=$($gam version extended nooffseterror)
export python_line=$(echo -e "${voutput}" | grep "Python ")
export python_arr=($python_line)
export this_python=${python_arr[1]}
if [[ "${this_python}" != "${COMPILED_PYTHON_VERSION}" ]]; then
echo "ERROR: Tried to compile Python ${COMPILED_PYTHON_VERSION} but ended up with ${this_python}"
exit 1
fi
export openssl_line=$(echo -e "${voutput}" | grep "OpenSSL ")
export openssl_arr=($openssl_line)
export this_openssl="${openssl_arr[1]}"
if [[ "${this_openssl}" != "${COMPILED_OPENSSL_VERSION}" ]]; then
echo "ERROR: Tried to compile OpenSSL ${COMPILED_OPENSSL_VERSION} but ended up with ${this_openssl}"
exit 1
fi
echo "We successfully compiled Python ${this_python} and OpenSSL ${this_openssl}"
- name: Live API tests push only
if: github.event_name == 'push' || github.event_name == 'schedule'
env:
PASSCODE: ${{ secrets.PASSCODE }}
run: |
if ([ "${RUNNER_OS}" == "macOS" ] && which gpg > /dev/null); then
brew install gnupg
fi
source ../.github/actions/decrypt.sh ../.github/actions/creds.tar.xz.gpg creds.tar.xz "${GAMCFGDIR}"
mv -v "${GAMCFGDIR}/oauth2.txt-gam-gha-${JID}" "${GAMCFGDIR}/oauth2.txt"
rm -v $GAMCFGDIR/oauth2.txt-gam*
export gam_user="gam-gha-${JID}@pdl.jaylee.us"
echo "gam_user=${gam_user}" >> $GITHUB_ENV
$gam config customer_id "C03uzfv2s" save
$gam config domain "pdl.jaylee.us" save
$gam config admin_email "${gam_user}" save
$gam config enable_dasa false save
$gam oauth info
$gam oauth refresh
$gam config enable_dasa true save
$gam create signjwtserviceaccount
$gam checkconn
$gam user "$gam_user" check serviceaccount
$gam info domain
$gam info user
export tstamp=$($PYTHON -c "import time; print(time.time_ns())")
export newbase="gha_test_${JID}_${tstamp}"
export newuser="${newbase}@pdl.jaylee.us"
export newgroup="${newbase}[email protected]"
export newalias="${newbase}[email protected]"
export newbuilding="${newbase}-building"
export newresource="${newbase}-resource"
export newou="aaaGithub Actions/${newbase}"
# cleanup old runs
$gam config enable_dasa false save
$gam config csv_output_row_filter "name:regex:gha_test_${JID}_" print vaultholds || if [ $? != 55 ]; then exit $?; fi | $gam csv - gam delete vaulthold "id:~~holdId~~" matter "id:~~matterId~~"
$gam config enable_dasa true save
$gam config csv_output_row_filter "name:regex:gha_test_${JID}_" print features | $gam csv - gam delete feature ~name
$gam config csv_output_row_filter "name:regex:^gha_test_${JID}_" user $gam_user print shareddrives asadmin | $gam csv - gam user $gam_user delete shareddrive ~id nukefromorbit
$gam print users query "gha.jid=$JID" | $gam csv - gam delete user ~primaryEmail
$gam config csv_output_row_filter "name:regex:^gha_test_${JID}_" print ous fromparent "aaaGithub Actions" | $gam csv - gam delete ou ~orgUnitId
$gam config csv_output_row_filter "email:regex:^gha_test_${JID}_" print cigroups | $gam csv - gam delete cigroup ~email
$gam config csv_output_row_filter "resourceId:regex:^gha_test_${JID}_" print resources | $gam csv - gam delete resource ~resourceId
$gam config csv_output_row_filter "buildingId:regex:^gha_test_${JID}_" print buildings | $gam csv - gam delete building ~buildingId
$gam config csv_output_row_filter "Emails.1.address:regex:^gha_test-${JID}_" print contacts | $gam csv - gam delete contact ~ContactID
echo "Creating OrgUnit ${newou}"
$gam create ou "${newou}"
export GAM_THREADS=5
echo email > sample.csv;
for i in {1..10}; do
echo "${newbase}-bulkuser-$i" >> sample.csv;
done
driveid=$($gam user $gam_user add shareddrive "${newbase}" returnidonly)
echo "Created shared drive ${driveid}"
$gam create user $newuser firstname GHA lastname $JID displayname "Github Actions ${JID}" password random ou "${newou}" recoveryphone 12125121110 recoveryemail [email protected] gha.jid $JID languages en+,en-GB-
$gam user $newuser update photo https://dummyimage.com/400x600/000/fff
$gam user $newuser get photo
$gam user $newuser delete photo
$gam create alias $newalias user $newuser
$gam create group $newgroup name "GHA $JID group" description "This is a description" isarchived true
$gam user $gam_user sendemail recipient $newuser subject "test message $newbase" message "GHA test message"
$gam user $gam_user sendemail recipient [email protected] subject "test ${tstamp}" message "test message"
$gam config enable_dasa false save
$gam create contact firstname GHA lastname "$JID" email work "${newbase}@example.com" primary
$gam print contacts
$gam user $newuser add license workspaceenterpriseplus
$gam print privileges
$gam config enable_dasa true save
$gam update cigroup $newgroup security memberrestriction 'member.type == 1 || member.customer_id == groupCustomerId()'
$gam info cigroup $newgroup
$gam update group $newgroup add owner $gam_user
$gam update group $newgroup add member $newuser
$gam config enable_dasa false save
$gam create admin $newuser _GROUPS_EDITOR_ROLE CUSTOMER # condition nonsecuritygroup
$gam create admin $newgroup _HELP_DESK_ADMIN_ROLE org_unit "${newou}"
$gam config csv_output_row_filter "assignedToUser:regex:${newuser}" print admins | $gam csv - gam delete admin "~roleAssignmentId"
$gam config csv_output_row_filter "assignedToGroup:regex:${newgroup}" print admins | $gam csv - gam delete admin "~roleAssignmentId"
$gam config enable_dasa false save
$gam csv sample.csv gam create user ~~email~~ firstname "GHA Bulk" lastname ~~email~~ gha.jid $JID ou "${newou}"
$gam csv sample.csv gam update user ~~email~~ recoveryphone 12125121110 recoveryemail [email protected] password random displayname "GitHub Actions Bulk ${JID}"
$gam csv sample.csv gam update user ~~email~~ recoveryphone "" recoveryemail ""
$gam config enable_dasa false save
$gam csv sample.csv gam user ~email add license workspaceenterpriseplus
$gam config enable_dasa true save
$gam csv sample.csv gam user $gam_user sendemail recipient [email protected] subject "test message $newbase" message "GHA test message"
$gam csv sample.csv gam update group $newgroup add member ~email
$gam info group $newgroup
$gam info cigroup $newgroup membertree
# confirm mailbox is provisoned before continuing
$gam user $newuser waitformailbox
$gam user $newuser imap on
$gam user $newuser show imap
$gam user $newuser show delegates
#$gam user $newuser add contactdelegate "${newbase}-bulkuser-1"
#$gam user $newuser print contactdelegates
export biohazard=$(echo -e '\xe2\x98\xa3')
$gam user $newuser label "$biohazard unicode biohazard $biohazard"
$gam user $newuser show labels
$gam user $newuser show labels > labels.txt
$gam user $gam_user importemail subject "GHA import $newbase" message "This is a test import" labels IMPORTANT,UNREAD,INBOX,STARRED
$gam user $gam_user insertemail subject "GHA insert $newbase" file gam.py labels INBOX,UNREAD # yep body is gam code
$gam user $gam_user sendemail subject "GHA send $gam_user $newbase" file gam.py recipient [email protected]
$gam user $gam_user draftemail subject "GHA draft $newbase" message "Draft message test"
$gam csvfile sample.csv:email waitformailbox
$gam user $newuser delegate to "${newbase}-bulkuser-1" || if [ $? != 50 ]; then exit $?; fi # expect a 50 return code (delegation failed)
$gam users "$gam_user $newbase-bulkuser-1 $newbase-bulkuser-2 $newbase-bulkuser-3" delete messages query in:anywhere maxtodelete 99999 doit || if [ $? != 60 ]; then exit $?; fi # expect a 60 return code (no messages)
$gam users "$newbase-bulkuser-4 $newbase-bulkuser-5 $newbase-bulkuser-6" trash messages query in:anywhere maxtotrash 99999 doit || if [ $? != 60 ]; then exit $?; fi # expect a 60 return code (no messages)
$gam users "$newbase-bulkuser-7 $newbase-bulkuser-8 $newbase-bulkuser-9" modify messages query in:anywhere maxtomodify 99999 addlabel IMPORTANT addlabel STARRED doit || if [ $? != 60 ]; then exit $?; fi # expect a 60 return code (no messages)
$gam user $newuser delete label --ALL_LABELS--
$gam config csv_output_row_filter "name:regex:gha-test-${JID}" print features | $gam csv - gam delete feature ~name
$gam create feature name VC-$newbase
$gam create feature name Whiteboard-$newbase
$gam create building "My Building - $newbase" id $newbuilding floors 1,2,3,4,5,6,7,8,9,10,11,12,14,15 description "No 13th floor here..."
$gam create resource $newresource "Resource Calendar $tstamp" capacity 25 features Whiteboard-$newbase,VC-$newbase building $newbuilding floor 15 type Room
$gam info resource $newresource
$gam user $newuser add drivefile drivefilename "TPS Reports" mimetype gfolder
$gam user $newuser show filelist
$gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete ~id # clear ACLs
$gam calendar $gam_user add read domain
$gam calendar $gam_user add freebusy default
$gam calendar $gam_user add editor $newuser
$gam calendar $gam_user showacl
$gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete ~id
$gam calendar $gam_user addevent summary "GHA test event" start +1h end +2h attendee $newgroup hangoutsmeet guestscanmodify true sendupdates all
$gam calendar $gam_user printevents after -0d
$gam config enable_dasa false save
matterid=uid:$($gam create vaultmatter name "GHA matter $newbase" description "test matter" collaborators $newuser returnidonly)
$gam create vaulthold matter $matterid name "GHA hold $newbase" corpus mail accounts $newuser
$gam print vaultmatters matterstate open
$gam print vaultholds matter $matterid
$gam print vaultcount matter $matterid corpus mail everyone todrive tdnobrowser
$gam create vaultexport matter $matterid name "GHA export $newbase" corpus mail accounts $newuser
$gam print exports matter $matterid | $gam csv - gam info export $matterid id:~~id~~
$gam config enable_dasa true save
$gam csv sample.csv gam user ~email add calendar id:$newresource
$gam delete resource $newresource
$gam delete feature Whiteboard-$newbase
$gam delete feature VC-$newbase
$gam delete building $newbuilding
$gam delete group $newgroup
$gam config enable_dasa false save
echo start
$gam user $newuser delete license workspaceenterpriseplus
echo finish
$gam config enable_dasa true save
$gam whatis $newuser || if [ $? != 20 ]; then exit $?; fi # expect a 20 return code (is a user)
$gam user $gam_user show tokens
$gam config enable_dasa false save
download_dir="${RUNNER_TEMP}/TEMP_DELETE_ME"
mkdir -v "$download_dir"
$gam print exports matter $matterid | $gam csv - gam download export $matterid id:~~id~~ targetfolder "$download_dir"
rm -rvf "$download_dir"
$gam delete hold "GHA hold $newbase" matter $matterid
$gam update matter $matterid action close
$gam update matter $matterid action delete
# shakes off vault hold on user so we can delete
$gam print users query "email:${newuser}" orgunitpath | $gam csv - gam update user ~primaryEmail ou ~orgUnitPath
$gam user $newuser show holds || if [ $? != 55 ]; then exit $?; fi # expect a 55 return code
export sn="$JID$JID$JID$JID-$(openssl rand -base64 32 | sed 's/[^a-zA-Z0-9]//g')"
$gam create device serialnumber $sn devicetype android
$gam config enable_dasa true save
$gam print users query "gha.jid=$JID" | $gam csv - gam delete user ~primaryEmail || if [ $? != 50 ]; then exit $?; fi # expect a 50 return code (vault hold on user)
$gam delete contacts emailmatchpattern "^${newbase}@example.com$"
$gam print mobile
$gam print devices
$gam print browsers
$gam print cros allfields orderby serialnumber
$gam show crostelemetry storagepercentonly
$gam report usageparameters customer
$gam report usage customer parameters gmail:num_emails_sent,accounts:num_1day_logins
$gam report customer todrive tdnobrowser
#$gam report users fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive tdnobrowser
$gam report users todrive tdnobrowser
$gam report admin start -3d todrive tdnobrowser
$gam print devices nopersonaldevices nodeviceusers filter "serial:$JID$JID$JID$JID-" | $gam csv - gam delete device id ~name
$gam config enable_dasa false save
$gam print userinvitations
$gam print userinvitations | $gam csv - gam send userinvitation ~name
$gam config enable_dasa false save
$gam create caalevel "zzz_${newbase}" basic condition ipsubnetworks 1.1.1.1/32,2.2.2.2/32 endcondition
$gam print caalevels
$gam delete caalevel "zzz_${newbase}"
$gam user $gam_user add drivefile localfile gam.py parentid "${driveid}"
$gam user $gam_user update shareddrive "${driveid}" ou "${newou}"
$gam user $gam_user show shareddrives asadmin
$gam user $gam_user update shareddrive "${driveid}" ou "aaaGithub Actions" # so we can delete our OU...
$gam user $gam_user delete shareddrive "${driveid}" nukefromorbit
echo "printer model count:"
ssoprofile=$($gam create inboundssoprofile name "El Goog ${newbase}" loginurl https://www.google.com logouturl https://www.google.com changepasswordurl https://www.google.com entityid ElGoog return_name_only)
$gam create inboundssocredential profile "id:${ssoprofile}" generate_key
#$gam create inboundssoassignment profile "id:${ssoprofile}" orgunit "${newou}" mode SAML_SSO
#$gam delete inboundssoassignment "orgunit:${newou}"
$gam delete inboundssoprofile "id:${ssoprofile}"
$gam print printermodels | wc -l
$gam print printers
printerid=$($gam create printer displayname "${newbase}" uri ipp://localhost:631 driverless description "made by $(gam_user)" ou "${newou}" nodetails | awk '{print substr($2, 1, length($2)-1)}')
$gam info printer "$printerid"
$gam delete printer "$printerid"
$gam delete ou "${newou}"
- name: Archive production artifacts
uses: actions/upload-artifact@v3
if: (github.event_name == 'push' || github.event_name == 'schedule') && matrix.goal != 'test'
with:
name: gam-binaries
path: |
src/*.tar.xz
src/*.zip
src/*.msi
- name: Tar Cache archive
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
working-directory: ${{ github.workspace }}
run: |
tar cJvvf bin.tar.xz bin/
publish:
if: github.event_name == 'push'
runs-on: ubuntu-latest
needs: build
permissions:
contents: write
packages: write
pull-requests: read
steps:
- uses: actions/checkout@v3
with:
persist-credentials: false
fetch-depth: 0
- name: Download artifacts
uses: actions/download-artifact@v3
- name: VirusTotal Scan
uses: crazy-max/ghaction-virustotal@v3
with:
vt_api_key: ${{ secrets.VT_API_KEY }}
files: |
gam-binaries/*
- name: Set datetime version string
id: dateversion
run: |
export dateversion="$(date +'%Y%m%d.%H%M%S')"
echo "Date version: ${dateversion}"
echo "dateversion=${dateversion}" >> $GITHUB_OUTPUT
- uses: "marvinpinto/action-automatic-releases@latest"
name: Publish draft release
with:
repo_token: "${{ secrets.GITHUB_TOKEN }}"
automatic_release_tag: "${{ steps.dateversion.outputs.dateversion }}"
prerelease: false
draft: true
files: |
gam-binaries/*