-
Notifications
You must be signed in to change notification settings - Fork 1
/
schism.yaml
102 lines (94 loc) · 4.15 KB
/
schism.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
# Dimensions of the final video
width: 854
height: 468
# Frame rate of the final video
# source video at 29.97fps, we skip every 2nd frame to reduce the amount of frames we have to generate (can just fill it in with RIFE afterwards)
frames_per_second: 14.985
output_path: H:\latent\out\schism
# A directory for intermediary results, and misc. other models that are required
persistence_dir: persistence/
# MISC notes:
# - Load SD2 depth2img model in webUI first (all operations here are img2img)
# Torch device to use
torch_device: cuda
scenes:
- # The prompt to generate images from
prompt: a beautiful painting by alex grey, visionary art
# How long this prompt should last at the frame rate specified above
duration: 446s
# Interpolation between the previous and this scene. does not add to the duration of the scene.
interpolation: 0s
# Which mechanism to use to generate frames. Must be defined with parameters and type below
mechanism: api
# Optionally, some parameters of a mechanism may be modified at the scene-level. For example the number of steps to sample for in a diffusion model.
mechanism_parameters:
strength_schedule: clamp(0,1, 0.62-(beat*0.23)+(is_turbo_step*0.14))
mechanisms:
- # Name of this parameter set within the config. Can be referenced in scenes
name: api
# Name of the mechanism to use
type: api
# Arbitrary parameters the text2video mechanism can take. These are always implementation specific
mechanism_parameters:
host: http://localhost:7860
seed: 2812101937
# Statically stay on the original seed
subseed_strength: 0
# How many turbo steps (lower amount of sampling steps) to run in-between "full sized" frames
turbo_steps: 0
# How many sampling steps to run for a turbo frame
turbo_sampling_steps: 25
# Noise schedule (how much noise to add to in-between frames), can be a function or static
noise_schedule: 0.00
steps: 25
# Euler a, LMS, ...
sampler: LMS
scale: 9.5
# This is denoising strength here, lower = better flow
strength_schedule: (min(1.0, 0.76-(beat*0.5)+(is_turbo_step*0.12)))
# Video just dumped from youtube
video_init: H:\latent\in\TOOL - Schism-MM62wjLrgmA.mkv
video_init_offset: 0s
# Intentionally drop every 2nd frame to reduce the result frame rate
video_init_skip_frames: 1
# Type of animation to apply between frames, if any
animation: 3D
# Parameters for the animation
animation_parameters:
translation_x: 0
translation_y: 0
translation_z: 0
# depth model parameters
near_plane: 200
far_plane: 10000
fov: 40
sampling_mode: bicubic
padding_mode: border
# Other multi-modal context
additional_context:
input_mechanisms:
# Spectral input mechanism: Analyzes an audio file using bandpass filters,
# passes a variable into the functions in the `0..1` range that indicates the amplitude of the signal within the filter.
# Normalized to cover the full range.
- type: "spectral"
mechanism_parameters:
file: H:\latent\in\TOOL - Schism-MM62wjLrgmA.mkv
offset: 0
atempo: 1
filters:
# These are very rough estimates without analysing and could probably be improved.
- variable_name: kick # The name of the variable we will use in the animation function
f_center: 54 # Central frequency the filter will respond to (54Hz)
f_width: 40 # The filter will pass signals from 34Hz to 74Hz
order: 6 # The filter slope will be 7*6=42dB per octave
- variable_name: snare # The name of the variable we will use in the animation function
f_center: 300 # Central frequency the filter will respond to (54Hz)
f_width: 40 # The filter will pass signals from 34Hz to 74Hz
order: 8 # The filter slope will be 7*6=42dB per octave
custom_functions:
- # The name under which the results of the function can be used in other functions
variable_name: beat
prev_values: 3
function: |
kick+snare
# Store