Skip to content

Commit

Permalink
Merge pull request #289 from Capsize-Games/develop
Browse files Browse the repository at this point in the history
Develop
  • Loading branch information
w4ffl35 authored Jul 27, 2023
2 parents a8c3a4a + 882205d commit f44379c
Show file tree
Hide file tree
Showing 110 changed files with 10,914 additions and 6,317 deletions.
5 changes: 4 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ WORKDIR /app
RUN pip install --upgrade pip
RUN pip install --upgrade setuptools
RUN pip install --upgrade wheel
RUN pip install requests aihandler cmake
RUN pip install requests cmake
RUN pip uninstall torch torchvision -y
RUN pip install torch torchvision --index-url https://download.pytorch.org/whl/cu118 --upgrade

Expand All @@ -76,3 +76,6 @@ COPY dobuild.py dobuild.py
COPY build.sh build.sh
COPY build.airunner.linux.prod.spec build.airunner.linux.prod.spec
COPY setup.py setup.py

FROM build_files as install_diffusers
RUN pip install https://github.com/w4ffl35/diffusers/archive/refs/tags/v0.19.0-convert_from_ckpt_patch.zip
21 changes: 21 additions & 0 deletions build-socket-server.dev.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
./venv/bin/python -m PyInstaller --log-level=INFO --noconfirm ./build.airunner-socket-server.linux.dev.spec
cp -R ./venv/lib/python3.10/site-packages/timm ./dist/airunner-socket-server/
cp ./venv/lib/python3.10/site-packages/torch/lib/libtorch_cuda_linalg.so ./dist/airunner-socket-server/
cp -R ./venv/lib/python3.10/site-packages/Pillow-9.5.0.dist-info ./dist/airunner-socket-server/

# remove redundant files
rm -rf ./dist/airunner-socket-server/nvfuser
rm -rf ./dist/airunner-socket-server/functorch
rm -rf ./dist/airunner-socket-server/pydantic
rm -rf ./dist/airunner-socket-server/tensorflow/python/data/experimental
rm -f ./dist/airunner-socket-server/matplotlib/_qhull.cpython-310-x86_64-linux-gnu.so
rm -f ./dist/airunner-socket-server/JIT/_C.so
rm -f ./dist/airunner-socket-server/libtorch_cuda_linalg.so
rm -f ./dist/airunner-socket-server/libtorch.so
rm -f ./dist/airunner-socket-server/libtorch_cuda.so
rm -f ./dist/airunner-socket-server/libnvfuser_codegen.so
rm -f ./dist/airunner-socket-server/libc10_cuda.so
rm -f ./dist/airunner-socket-server/libtorch_python.so
rm -f ./dist/airunner-socket-server/libtorch_cpu.so
rm -f ./dist/airunner-socket-server/libc10.so
rm -f ./dist/airunner-socket-server/libshm.so
173 changes: 173 additions & 0 deletions build.airunner-socket-server.linux.dev.spec
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
# -*- mode: python ; coding: utf-8 -*-
import os
import shutil
from PyInstaller.utils.hooks import copy_metadata, collect_data_files
import sys ; sys.setrecursionlimit(sys.getrecursionlimit() * 5)
os.environ["AIRUNNER_ENVIRONMENT"] = "prod"
libraries = [
"/usr/lib/x86_64-linux-gnu/wine-development/",
"./venv/lib/python3.10/site-packages/h5py.libs/",
"./venv/lib/python3.10/site-packages/scipy.libs/",
"./venv/lib/python3.10/site-packages/tokenizers.libs/",
"./venv/lib/python3.10/site-packages/Pillow.libs/",
"./venv/lib/python3.10/site-packages/opencv_python.libs/",
"./venv/lib/python3.10/site-packages/torchaudio/lib/",
"./venv/lib/python3.10/site-packages/torch/lib/",
"/usr/lib/python3.10",
"/usr/lib/x86_64-linux-gnu/",
"/usr/local/lib/",
"/usr/local/lib/python3.10",
"/usr/local/lib/python3.10/dist-packages"
]
os.environ["LD_LIBRARY_PATH"] = ":".join(libraries)
block_cipher = None
DEBUGGING = True
EXCLUDE_BINARIES = True
EXE_NAME = "airunner-socket-server" # used when creating a binary instead of a folder
EXE_STRIP = False
EXE_UPX = True
EXE_RUNTIME_TMP_DIR = None
COLLECT_NAME = 'airunner-socket-server'
COLLECT_STRIP = False
COLLECT_UPX = True
datas = []
datas += copy_metadata('tqdm')
datas += copy_metadata('regex')
datas += copy_metadata('requests')
datas += copy_metadata('packaging')
datas += copy_metadata('filelock')
datas += copy_metadata('numpy')
datas += copy_metadata('tokenizers')
datas += copy_metadata('transformers')
datas += copy_metadata('rich')
datas += copy_metadata('sympy')
datas += copy_metadata('opencv-python')
datas += collect_data_files("torch", include_py_files=True)
datas += collect_data_files("torchvision", include_py_files=True)
datas += collect_data_files("JIT", include_py_files=True)
datas += collect_data_files("pytorch_lightning", include_py_files=True)
datas += collect_data_files("lightning_fabric", include_py_files=True)
datas += collect_data_files("transformers", include_py_files=True)
datas += collect_data_files("sympy", include_py_files=True)
datas += collect_data_files("controlnet_aux", include_py_files=True)
a = Analysis(
[
f'./src/airunner/aihandler/socket_server.py',
],
pathex=[
"./venv/lib/python3.10/site-packages/",
"./venv/lib/python3.10/site-packages/torch/lib",
"./venv/lib/python3.10/site-packages/tokenizers",
"./venv/lib/python3.10/site-packages/tensorflow",
"/usr/lib/x86_64-linux-gnu/",
],
binaries=[
('./venv/lib/python3.10/site-packages/nvidia/cudnn/lib/libcudnn_ops_infer.so.8', '.'),
('./venv/lib/python3.10/site-packages/nvidia/cudnn/lib/libcudnn_cnn_infer.so.8', '.'),
('/usr/lib/x86_64-linux-gnu/libgstgl-1.0.so.0', '.'),
],
datas=datas,
hiddenimports=[
"JIT",
"tqdm",
"diffusers",
"transformers",
"nvidia",
"torch",
"torchvision",
"torchvision.io",
"logging",
"logging.config",
"einops",
"omegaconf",
"contextlib",
"itertools",
"pytorch_lightning",
"huggingface_hub.hf_api",
"huggingface_hub.repository",
"inspect",
"psutil",
"matplotlib",
"numpy",
"PIL._tkinter_finder",
"sympy",
"opencv-python",
],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[
"tcl",
"tcl8",
"cmake",
"cryptography",
"email-validator",
"google",
"google-auth",
"google-auth-oauthlib",
"google-pasta",
"Jinja2",
"lightning-cloud",
"Markdown",
"markdown-it-py",
"MarkupSafe",
"mdurl",
"ninja",
"nvidia-pyindex",
"pytz",
"tensorboard",
"tensorboard-data-server",
"tensorboard-plugin-wit",
"unattended-upgrades",
"watchfiles",
"wcwidth",
"websocket-client",
"websockets",
],
win_no_prefer_redirects=False,
win_private_assemblies=False,
cipher=block_cipher,
noarchive=False,
)
pyz = PYZ(
a.pure,
a.zipped_data,
cipher=block_cipher
)
exe = EXE(
pyz,
a.scripts,
[],
exclude_binaries=EXCLUDE_BINARIES,
name=EXE_NAME,
debug=DEBUGGING,
strip=EXE_STRIP,
upx=EXE_UPX,
runtime_tmpdir=EXE_RUNTIME_TMP_DIR,
console=DEBUGGING
)
coll = COLLECT(
exe,
a.binaries,
a.zipfiles,
a.datas,
strip=COLLECT_STRIP,
upx=COLLECT_UPX,
upx_exclude=[],
name=COLLECT_NAME
)

# copy files for distribution
shutil.copytree('./src/airunner/data', './dist/airunner-socket-server/data')

# copy sd config files
os.makedirs('./dist/airunner/diffusers/pipelines/stable_diffusion', exist_ok=True)
shutil.copyfile('./src/airunner/v1.yaml', './dist/airunner-socket-server/v1.yaml')
shutil.copyfile('./src/airunner/v2.yaml', './dist/airunner-socket-server/v2.yaml')
shutil.copyfile('./src/airunner/sd_xl_base.yaml', './dist/airunner-socket-server/sd_xl_base.yaml')
shutil.copyfile('./src/airunner/sd_xl_refiner.yaml', './dist/airunner-socket-server/sd_xl_refiner.yaml')

shutil.copyfile(
f'./venv/lib/python3.10/site-packages/JIT/__pycache__/random.cpython-310.pyc',
f'./dist/airunner-socket-server/random.pyc'
)
4 changes: 2 additions & 2 deletions build.airunner.linux.dev.spec
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ COLLECT_NAME = 'airunner'
COLLECT_STRIP = False
COLLECT_UPX = True
datas = []
datas += copy_metadata('aihandler')
datas += copy_metadata('tqdm')
datas += copy_metadata('regex')
datas += copy_metadata('requests')
Expand Down Expand Up @@ -70,7 +69,6 @@ a = Analysis(
],
datas=datas,
hiddenimports=[
"aihandler",
"airunner",
"airunner.extensions",
"JIT",
Expand Down Expand Up @@ -174,6 +172,8 @@ shutil.copyfile('./src/airunner/src/splashscreen.png', './dist/airunner/src/spla
os.makedirs('./dist/airunner/diffusers/pipelines/stable_diffusion', exist_ok=True)
shutil.copyfile('./src/airunner/v1.yaml', './dist/airunner/v1.yaml')
shutil.copyfile('./src/airunner/v2.yaml', './dist/airunner/v2.yaml')
shutil.copyfile('./src/airunner/sd_xl_base.yaml', './dist/airunner/sd_xl_base.yaml')
shutil.copyfile('./src/airunner/sd_xl_refiner.yaml', './dist/airunner/sd_xl_refiner.yaml')

shutil.copyfile(
f'./venv/lib/python3.10/site-packages/JIT/__pycache__/random.cpython-310.pyc',
Expand Down
5 changes: 2 additions & 3 deletions build.airunner.linux.prod.spec
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ COLLECT_NAME = 'airunner'
COLLECT_STRIP = False
COLLECT_UPX = True
datas = []
datas += copy_metadata('aihandler')
datas += copy_metadata('tqdm')
datas += copy_metadata('regex')
datas += copy_metadata('requests')
Expand Down Expand Up @@ -60,7 +59,6 @@ a = Analysis(
"/usr/local/lib/python3.10/dist-packages/",
"/usr/local/lib/python3.10/dist-packages/torch/lib",
"/usr/local/lib/python3.10/dist-packages/tokenizers",
"/usr/local/lib/python3.10/dist-packages/tensorflow",
"/usr/lib/x86_64-linux-gnu/",
],
binaries=[
Expand All @@ -70,7 +68,6 @@ a = Analysis(
],
datas=datas,
hiddenimports=[
"aihandler",
"airunner",
"airunner.extensions",
"JIT",
Expand Down Expand Up @@ -174,6 +171,8 @@ shutil.copyfile('/app/airunner/src/airunner/src/splashscreen.png', '/app/dist/ai
os.makedirs('/dist/airunner/diffusers/pipelines/stable_diffusion', exist_ok=True)
shutil.copyfile('/app/airunner/src/airunner/v1.yaml', '/app/dist/airunner/v1.yaml')
shutil.copyfile('/app/airunner/src/airunner/v2.yaml', '/app/dist/airunner/v2.yaml')
shutil.copyfile('/app/airunner/src/airunner/sd_xl_base.yaml', '/app/dist/airunner/sd_xl_base.yaml')
shutil.copyfile('/app/airunner/src/airunner/sd_xl_refiner.yaml', '/app/dist/airunner/sd_xl_refiner.yaml')

shutil.copyfile(
f'/usr/local/lib/python3.10/dist-packages/JIT/__pycache__/random.cpython-310.pyc',
Expand Down
10 changes: 8 additions & 2 deletions build.airunner.windows.dev.spec
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ COLLECT_STRIP = False
COLLECT_UPX = True

datas = []
datas += copy_metadata('aihandler')
datas += copy_metadata('tqdm')
datas += copy_metadata('regex')
datas += copy_metadata('requests')
Expand Down Expand Up @@ -68,7 +67,6 @@ a = Analysis(
],
datas=datas,
hiddenimports=[
"aihandler",
"airunner",
"airunner.extensions",
"JIT",
Expand Down Expand Up @@ -196,6 +194,14 @@ shutil.copyfile(
f'{ROOT}/src/airunner/v2.yaml',
f'{DIST}/v2.yaml'
)
shutil.copyfile(
f'{ROOT}/src/airunner/sd_xl_base.yaml',
f'{DIST}/sd_xl_base.yaml'
)
shutil.copyfile(
f'{ROOT}/src/airunner/sd_xl_refiner.yaml',
f'{DIST}/sd_xl_refiner.yaml'
)
shutil.copyfile(
f'.\\setup.py',
f'{DIST}/setup.py'
Expand Down
16 changes: 8 additions & 8 deletions build.airunner.windows.prod.spec
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ COLLECT_STRIP = False
COLLECT_UPX = True

datas = []
datas += copy_metadata('aihandler')
datas += copy_metadata('tqdm')
datas += copy_metadata('regex')
datas += copy_metadata('requests')
Expand All @@ -33,14 +32,12 @@ datas += copy_metadata('tokenizers')
datas += copy_metadata('transformers')
datas += copy_metadata('rich')
datas += copy_metadata('sympy')
datas += copy_metadata('tensorflow')
datas += copy_metadata('scipy')
datas += collect_data_files("torch", include_py_files=True)
datas += collect_data_files("torchvision", include_py_files=True)
datas += collect_data_files("pytorch_lightning", include_py_files=True)
datas += collect_data_files("lightning_fabric", include_py_files=True)
datas += collect_data_files("transformers", include_py_files=True)
datas += collect_data_files("tensorflow", include_py_files=True)
datas += collect_data_files("sympy", include_py_files=True)
datas += collect_data_files("controlnet_aux", include_py_files=True)
a = Analysis(
Expand All @@ -50,11 +47,8 @@ a = Analysis(
pathex=[
"C:\\Python310\\Lib\\site-packages",
"C:\\Python310\\Lib\\site-packages\\tokenizers",
"C:\\Python310\\Lib\\site-packages\\tensorflow_io_gcs_filesystem\\core\\python\\ops",
"C:\\Python310\\Lib\\site-packages\\tensorflow\\python\\data\\experimental\\service\\",
"C:\\Python310\\Lib\\site-packages\\torch\\lib",
"C:\\Python310\\Lib\\site-packages\\PyQt6",
"C:\\Users\\root\\AppData\\Local\\Programs\\Python\\Python310\\Lib\\site-packages\\tensorflow\\python\\data\\experimental\\service\\",
"C:\\Users\\root\\AppData\\Local\\Programs\\Python\\Python310\\",
],
binaries=[
Expand All @@ -63,11 +57,9 @@ a = Analysis(
("C:\\Python310\\vcruntime140_1.dll", "."),
("C:\\windows\\syswow64\\msvcp140.dll", "."),
("C:\\api-ms-win-shcore-scaling-l1-1-1.dll.so", "."),
("C:\\Python310\\Lib\\site-packages\\tensorflow\\python\\util\\_pywrap_utils.pyd", "."),
],
datas=datas,
hiddenimports=[
"aihandler",
"airunner",
"airunner.extensions",
"accelerate",
Expand Down Expand Up @@ -194,6 +186,14 @@ shutil.copyfile(
f'{ROOT}/src/airunner/v2.yaml',
f'{DIST}/v2.yaml'
)
shutil.copyfile(
f'{ROOT}/src/airunner/sd_xl_base.yaml',
f'{DIST}/sd_xl_base.yaml'
)
shutil.copyfile(
f'{ROOT}/src/airunner/sd_xl_refiner.yaml',
f'{DIST}/sd_xl_refiner.yaml'
)
shutil.copyfile(
f'Z:\\app\\setup.py',
f'{DIST}/setup.py'
Expand Down
2 changes: 1 addition & 1 deletion build.windows.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,4 @@ def clone(repo):
os.chdir("Z:\\app\\airunner")
os.system("C:\\cmd\\git.exe checkout master")
os.system("C:\\cmd\\git.exe pull")
os.system("C:\\Python310\\python.exe -m pip install . --no-deps")
os.system("C:\\Python310\\python.exe -m pip install .")
2 changes: 1 addition & 1 deletion dobuild.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,4 @@ def clone(repo):
os.chdir("/app/airunner")
os.system("git checkout master")
os.system("git pull")
os.system("python3 -m pip install . --no-deps")
os.system("python3 -m pip install .")
Loading

0 comments on commit f44379c

Please sign in to comment.