-
-
Notifications
You must be signed in to change notification settings - Fork 962
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Restore manganis optimizations (#3195)
* Create a crate for constant serialization of config structs for manganis * use SerializeConst for the image asset builder * switch to a declarative macro for assets * add serializable options for each asset type * implement the new manganis macro * optimize assets in the CLI * Fix assets with dioxus formattable * reduce fuzzing test limits * use the new syntax in examples * fix avif support * Fix manganis doc tests * cache asset optimization * Split out asset and bundled asset * make hash pointer width independent * Fix manganis macro docs * add constvec::is_empty method to fix clippy lint * remove nasm feature * simplify test_rsplit_once test so typos passes * revert example change from clippy fix * always accept unix paths * fix windows path seperator * fix folder asset hash * Optimize assets in a blocking task * Fix asset options docs * Document Asset and BundledAsset * simplify the linker macro a bit * add more docs to AssetParser expansion * add image format helpers * Split by both unix and windows path separators and take the smallest one
- Loading branch information
Showing
64 changed files
with
6,250 additions
and
721 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| use std::path::Path; | ||
|
|
||
| use anyhow::Context; | ||
| use lightningcss::{ | ||
| printer::PrinterOptions, | ||
| stylesheet::{MinifyOptions, ParserOptions, StyleSheet}, | ||
| }; | ||
| use manganis_core::CssAssetOptions; | ||
|
|
||
| pub(crate) fn process_css( | ||
| css_options: &CssAssetOptions, | ||
| source: &Path, | ||
| output_path: &Path, | ||
| ) -> anyhow::Result<()> { | ||
| let css = std::fs::read_to_string(source)?; | ||
|
|
||
| let css = if css_options.minified() { | ||
| minify_css(&css) | ||
| } else { | ||
| css | ||
| }; | ||
|
|
||
| std::fs::write(output_path, css).with_context(|| { | ||
| format!( | ||
| "Failed to write css to output location: {}", | ||
| output_path.display() | ||
| ) | ||
| })?; | ||
|
|
||
| Ok(()) | ||
| } | ||
|
|
||
| pub(crate) fn minify_css(css: &str) -> String { | ||
| let mut stylesheet = StyleSheet::parse(css, ParserOptions::default()).unwrap(); | ||
| stylesheet.minify(MinifyOptions::default()).unwrap(); | ||
| let printer = PrinterOptions { | ||
| minify: true, | ||
| ..Default::default() | ||
| }; | ||
| let res = stylesheet.to_css(printer).unwrap(); | ||
| res.code | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| use anyhow::Context; | ||
| use manganis_core::{AssetOptions, CssAssetOptions, ImageAssetOptions, JsAssetOptions}; | ||
| use std::path::Path; | ||
|
|
||
| use super::{ | ||
| css::process_css, folder::process_folder, image::process_image, js::process_js, | ||
| json::process_json, | ||
| }; | ||
|
|
||
| /// Process a specific file asset with the given options reading from the source and writing to the output path | ||
| pub(crate) fn process_file_to( | ||
| options: &AssetOptions, | ||
| source: &Path, | ||
| output_path: &Path, | ||
| ) -> anyhow::Result<()> { | ||
| // If the file already exists, then we must have a file with the same hash | ||
| // already. The hash has the file contents and options, so if we find a file | ||
| // with the same hash, we probably already created this file in the past | ||
| if output_path.exists() { | ||
| return Ok(()); | ||
| } | ||
| if let Some(parent) = output_path.parent() { | ||
| if !parent.exists() { | ||
| std::fs::create_dir_all(parent)?; | ||
| } | ||
| } | ||
| match options { | ||
| AssetOptions::Unknown => match source.extension().map(|e| e.to_string_lossy()).as_deref() { | ||
| Some("css") => { | ||
| process_css(&CssAssetOptions::new(), source, output_path)?; | ||
| } | ||
| Some("js") => { | ||
| process_js(&JsAssetOptions::new(), source, output_path)?; | ||
| } | ||
| Some("json") => { | ||
| process_json(source, output_path)?; | ||
| } | ||
| Some("jpg" | "jpeg" | "png" | "webp" | "avif") => { | ||
| process_image(&ImageAssetOptions::new(), source, output_path)?; | ||
| } | ||
| None if source.is_dir() => { | ||
| process_folder(source, output_path)?; | ||
| } | ||
| Some(_) | None => { | ||
| let source_file = std::fs::File::open(source)?; | ||
| let mut reader = std::io::BufReader::new(source_file); | ||
| let output_file = std::fs::File::create(output_path)?; | ||
| let mut writer = std::io::BufWriter::new(output_file); | ||
| std::io::copy(&mut reader, &mut writer).with_context(|| { | ||
| format!( | ||
| "Failed to write file to output location: {}", | ||
| output_path.display() | ||
| ) | ||
| })?; | ||
| } | ||
| }, | ||
| AssetOptions::Css(options) => { | ||
| process_css(options, source, output_path)?; | ||
| } | ||
| AssetOptions::Js(options) => { | ||
| process_js(options, source, output_path)?; | ||
| } | ||
| AssetOptions::Image(options) => { | ||
| process_image(options, source, output_path)?; | ||
| } | ||
| AssetOptions::Folder(_) => { | ||
| process_folder(source, output_path)?; | ||
| } | ||
| _ => { | ||
| tracing::warn!("Unknown asset options: {:?}", options); | ||
| } | ||
| } | ||
|
|
||
| Ok(()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| use std::path::Path; | ||
|
|
||
| use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; | ||
|
|
||
| use super::file::process_file_to; | ||
|
|
||
| /// Process a folder, optimizing and copying all assets into the output folder | ||
| pub fn process_folder(source: &Path, output_folder: &Path) -> anyhow::Result<()> { | ||
| // Create the folder | ||
| std::fs::create_dir_all(output_folder)?; | ||
|
|
||
| // Then optimize children | ||
| let files: Vec<_> = std::fs::read_dir(source) | ||
| .into_iter() | ||
| .flatten() | ||
| .flatten() | ||
| .collect(); | ||
|
|
||
| files.par_iter().try_for_each(|file| { | ||
| let file = file.path(); | ||
| let metadata = file.metadata()?; | ||
| let output_path = output_folder.join(file.strip_prefix(source)?); | ||
| if metadata.is_dir() { | ||
| process_folder(&file, &output_path) | ||
| } else { | ||
| process_file_minimal(&file, &output_path) | ||
| } | ||
| })?; | ||
|
|
||
| Ok(()) | ||
| } | ||
|
|
||
| /// Optimize a file without changing any of its contents significantly (e.g. by changing the extension) | ||
| fn process_file_minimal(input_path: &Path, output_path: &Path) -> anyhow::Result<()> { | ||
| process_file_to( | ||
| &manganis_core::AssetOptions::Unknown, | ||
| input_path, | ||
| output_path, | ||
| )?; | ||
| Ok(()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| use image::{DynamicImage, EncodableLayout}; | ||
| use std::{ | ||
| io::{BufWriter, Write}, | ||
| path::Path, | ||
| }; | ||
|
|
||
| pub(crate) fn compress_jpg(image: DynamicImage, output_location: &Path) -> anyhow::Result<()> { | ||
| let mut comp = mozjpeg::Compress::new(mozjpeg::ColorSpace::JCS_EXT_RGBX); | ||
| let width = image.width() as usize; | ||
| let height = image.height() as usize; | ||
|
|
||
| comp.set_size(width, height); | ||
| let mut comp = comp.start_compress(Vec::new())?; // any io::Write will work | ||
|
|
||
| comp.write_scanlines(image.to_rgba8().as_bytes())?; | ||
|
|
||
| let jpeg_bytes = comp.finish()?; | ||
|
|
||
| let file = std::fs::File::create(output_location)?; | ||
| let w = &mut BufWriter::new(file); | ||
| w.write_all(&jpeg_bytes)?; | ||
| Ok(()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| use std::path::Path; | ||
|
|
||
| use anyhow::Context; | ||
| use jpg::compress_jpg; | ||
| use manganis_core::{ImageAssetOptions, ImageFormat, ImageSize}; | ||
| use png::compress_png; | ||
|
|
||
| mod jpg; | ||
| mod png; | ||
|
|
||
| pub(crate) fn process_image( | ||
| image_options: &ImageAssetOptions, | ||
| source: &Path, | ||
| output_path: &Path, | ||
| ) -> anyhow::Result<()> { | ||
| let mut image = image::ImageReader::new(std::io::Cursor::new(&*std::fs::read(source)?)) | ||
| .with_guessed_format()? | ||
| .decode(); | ||
|
|
||
| if let Ok(image) = &mut image { | ||
| if let ImageSize::Manual { width, height } = image_options.size() { | ||
| *image = image.resize_exact(width, height, image::imageops::FilterType::Lanczos3); | ||
| } | ||
| } | ||
|
|
||
| match (image, image_options.format()) { | ||
| (image, ImageFormat::Png) => { | ||
| compress_png(image?, output_path); | ||
| } | ||
| (image, ImageFormat::Jpg) => { | ||
| compress_jpg(image?, output_path)?; | ||
| } | ||
| (Ok(image), ImageFormat::Avif) => { | ||
| if let Err(error) = image.save(output_path) { | ||
| tracing::error!("Failed to save avif image: {} with path {}. You must have the avif feature enabled to use avif assets", error, output_path.display()); | ||
| } | ||
| } | ||
| (Ok(image), ImageFormat::Webp) => { | ||
| if let Err(err) = image.save(output_path) { | ||
| tracing::error!("Failed to save webp image: {}. You must have the avif feature enabled to use webp assets", err); | ||
| } | ||
| } | ||
| (Ok(image), _) => { | ||
| image.save(output_path)?; | ||
| } | ||
| // If we can't decode the image or it is of an unknown type, we just copy the file | ||
| _ => { | ||
| let source_file = std::fs::File::open(source)?; | ||
| let mut reader = std::io::BufReader::new(source_file); | ||
| let output_file = std::fs::File::create(output_path)?; | ||
| let mut writer = std::io::BufWriter::new(output_file); | ||
| std::io::copy(&mut reader, &mut writer).with_context(|| { | ||
| format!( | ||
| "Failed to write image to output location: {}", | ||
| output_path.display() | ||
| ) | ||
| })?; | ||
| } | ||
| } | ||
|
|
||
| Ok(()) | ||
| } |
Oops, something went wrong.