Tests fixing and disabling.

src/engine.rs

@@ -7,6 +7,7 @@ use crate::regex_manager::RegexManagerDiscardPolicy;
 use crate::request::Request;
 use crate::resources::{Resource, ResourceStorage};
 
+#[allow(unused_imports)]
 pub use crate::engine_serializer::Serialize;
 
 use std::collections::HashSet;

src/filters/fb_network.rs

@@ -86,6 +86,11 @@ impl<'a> FlatNetworkFiltersListBuilder<'a> {
             None
         };
 
+        let raw_line = network_filter
+            .raw_line
+            .as_ref()
+            .map(|v| self.builder.create_string(v.as_str()));
+
         let filter = fb::NetworkFilter::create(
             &mut self.builder,
             &fb::NetworkFilterArgs {
@@ -96,6 +101,7 @@ impl<'a> FlatNetworkFiltersListBuilder<'a> {
                 opt_not_domains: opt_not_domains,
                 hostname: hostname,
                 tag: tag,
+                raw_line: raw_line,
             },
         );
 
@@ -247,6 +253,11 @@ impl<'a> FlatNetworkFilter<'a> {
     pub fn patterns(&self) -> FlatPatterns {
         FlatPatterns::new(self.fb_filter.patterns())
     }
+
+    #[inline(always)]
+    pub fn raw_line(&self) -> Option<String> {
+        self.fb_filter.raw_line().map(|v| v.to_string())
+    }
 }
 
 impl<'a> NetworkFilterMaskHelper for FlatNetworkFilter<'a> {

src/flat/fb_network_filter.fbs

@@ -1,26 +1,27 @@
-// flatc --rust --gen-object-api -o src/flat/ src/flat/network_filter.fbs
-
-namespace fb;
-
-table NetworkFilter {
-  mask: uint32;  // NetworkFilterMask (network.rs)
-
-  // These arrays contain sorted(asc) indecies in the |unique_domains_hashes|
-  // instead of hashes itself. It saves a lot of memory because there
-  // aren't many unique hashes.
-  opt_domains: [uint16];
-  opt_not_domains: [uint16];
-
-  patterns: [string];
-  modifier_option: string;
-  hostname: string;
-
-  tag: string;
-}
-
-table NetworkFilterList {
-  network_filters: [NetworkFilter] (required);
-  unique_domains_hashes: [uint64] (required);
-}
-
-root_type NetworkFilterList;
+// flatc --rust --gen-object-api -o src/flat/ src/flat/network_filter.fbs
+
+namespace fb;
+
+table NetworkFilter {
+  mask: uint32;  // NetworkFilterMask (network.rs)
+
+  // These arrays contain sorted(asc) indecies in the |unique_domains_hashes|
+  // instead of hashes itself. It saves a lot of memory because there
+  // aren't many unique hashes.
+  opt_domains: [uint16];
+  opt_not_domains: [uint16];
+
+  patterns: [string];
+  modifier_option: string;
+  hostname: string;
+
+  tag: string;
+  raw_line: string;
+}
+
+table NetworkFilterList {
+  network_filters: [NetworkFilter] (required);
+  unique_domains_hashes: [uint64] (required);
+}
+
+root_type NetworkFilterList;

src/flat/fb_network_filter_generated.rs

@@ -42,6 +42,7 @@ pub mod fb {
         pub const VT_MODIFIER_OPTION: flatbuffers::VOffsetT = 12;
         pub const VT_HOSTNAME: flatbuffers::VOffsetT = 14;
         pub const VT_TAG: flatbuffers::VOffsetT = 16;
+        pub const VT_RAW_LINE: flatbuffers::VOffsetT = 18;
 
         #[inline]
         pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
@@ -58,6 +59,9 @@ pub mod fb {
             args: &'args NetworkFilterArgs<'args>,
         ) -> flatbuffers::WIPOffset<NetworkFilter<'bldr>> {
             let mut builder = NetworkFilterBuilder::new(_fbb);
+            if let Some(x) = args.raw_line {
+                builder.add_raw_line(x);
+            }
             if let Some(x) = args.tag {
                 builder.add_tag(x);
             }
@@ -90,6 +94,7 @@ pub mod fb {
             let modifier_option = self.modifier_option().map(|x| x.to_string());
             let hostname = self.hostname().map(|x| x.to_string());
             let tag = self.tag().map(|x| x.to_string());
+            let raw_line = self.raw_line().map(|x| x.to_string());
             NetworkFilterT {
                 mask,
                 opt_domains,
@@ -98,6 +103,7 @@ pub mod fb {
                 modifier_option,
                 hostname,
                 tag,
+                raw_line,
             }
         }
 
@@ -183,6 +189,16 @@ pub mod fb {
                     .get::<flatbuffers::ForwardsUOffset<&str>>(NetworkFilter::VT_TAG, None)
             }
         }
+        #[inline]
+        pub fn raw_line(&self) -> Option<&'a str> {
+            // Safety:
+            // Created from valid Table for this object
+            // which contains a valid value in this slot
+            unsafe {
+                self._tab
+                    .get::<flatbuffers::ForwardsUOffset<&str>>(NetworkFilter::VT_RAW_LINE, None)
+            }
+        }
     }
 
     impl flatbuffers::Verifiable for NetworkFilter<'_> {
@@ -218,6 +234,11 @@ pub mod fb {
                     false,
                 )?
                 .visit_field::<flatbuffers::ForwardsUOffset<&str>>("tag", Self::VT_TAG, false)?
+                .visit_field::<flatbuffers::ForwardsUOffset<&str>>(
+                    "raw_line",
+                    Self::VT_RAW_LINE,
+                    false,
+                )?
                 .finish();
             Ok(())
         }
@@ -232,6 +253,7 @@ pub mod fb {
         pub modifier_option: Option<flatbuffers::WIPOffset<&'a str>>,
         pub hostname: Option<flatbuffers::WIPOffset<&'a str>>,
         pub tag: Option<flatbuffers::WIPOffset<&'a str>>,
+        pub raw_line: Option<flatbuffers::WIPOffset<&'a str>>,
     }
     impl<'a> Default for NetworkFilterArgs<'a> {
         #[inline]
@@ -244,6 +266,7 @@ pub mod fb {
                 modifier_option: None,
                 hostname: None,
                 tag: None,
+                raw_line: None,
             }
         }
     }
@@ -309,6 +332,13 @@ pub mod fb {
                 .push_slot_always::<flatbuffers::WIPOffset<_>>(NetworkFilter::VT_TAG, tag);
         }
         #[inline]
+        pub fn add_raw_line(&mut self, raw_line: flatbuffers::WIPOffset<&'b str>) {
+            self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(
+                NetworkFilter::VT_RAW_LINE,
+                raw_line,
+            );
+        }
+        #[inline]
         pub fn new(
             _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
         ) -> NetworkFilterBuilder<'a, 'b, A> {
@@ -335,6 +365,7 @@ pub mod fb {
             ds.field("modifier_option", &self.modifier_option());
             ds.field("hostname", &self.hostname());
             ds.field("tag", &self.tag());
+            ds.field("raw_line", &self.raw_line());
             ds.finish()
         }
     }
@@ -348,6 +379,7 @@ pub mod fb {
         pub modifier_option: Option<String>,
         pub hostname: Option<String>,
         pub tag: Option<String>,
+        pub raw_line: Option<String>,
     }
     impl Default for NetworkFilterT {
         fn default() -> Self {
@@ -359,6 +391,7 @@ pub mod fb {
                 modifier_option: None,
                 hostname: None,
                 tag: None,
+                raw_line: None,
             }
         }
     }
@@ -377,6 +410,7 @@ pub mod fb {
             let modifier_option = self.modifier_option.as_ref().map(|x| _fbb.create_string(x));
             let hostname = self.hostname.as_ref().map(|x| _fbb.create_string(x));
             let tag = self.tag.as_ref().map(|x| _fbb.create_string(x));
+            let raw_line = self.raw_line.as_ref().map(|x| _fbb.create_string(x));
             NetworkFilter::create(
                 _fbb,
                 &NetworkFilterArgs {
@@ -387,6 +421,7 @@ pub mod fb {
                     modifier_option,
                     hostname,
                     tag,
+                    raw_line,
                 },
             )
         }

src/lib.rs

@@ -20,7 +20,7 @@ pub mod blocker;
 #[cfg(feature = "content-blocking")]
 pub mod content_blocking;
 pub mod cosmetic_filter_cache;
-#[cfg(not(feature = "flatbuffers"))]
+#[cfg(not(feature = "flatbuffers"))] // No serialization for flatbuffers yet.
 mod data_format;
 mod engine;
 mod engine_serializer;

src/network_filter_list.rs

@@ -277,7 +277,7 @@ pub struct FlatNetworkFilterList {
 }
 
 impl NetworkFilterListTrait for FlatNetworkFilterList {
-    fn new(filters: Vec<NetworkFilter>, _optimize: bool) -> Self {
+    fn new(filters: Vec<NetworkFilter>, optimize: bool) -> Self {
         // Compute tokens for all filters
         let filter_tokens: Vec<_> = filters
             .into_iter()
@@ -294,7 +294,9 @@ impl NetworkFilterListTrait for FlatNetworkFilterList {
         let mut optimizable = HashMap::<Hash, Vec<NetworkFilter>>::new();
         {
             for (network_filter, multi_tokens) in filter_tokens {
-                let index = if !optimizer::is_filter_optimizable_by_patterns(&network_filter) {
+                let index = if !optimize
+                    || !optimizer::is_filter_optimizable_by_patterns(&network_filter)
+                {
                     Some(flat_builder.add(&network_filter))
                 } else {
                     None
@@ -325,13 +327,20 @@ impl NetworkFilterListTrait for FlatNetworkFilterList {
             }
         }
 
-        for (token, v) in optimizable {
-            let optimized = optimizer::optimize_by_groupping_patterns(v);
+        if optimize {
+            for (token, v) in optimizable {
+                let optimized = optimizer::optimize_by_groupping_patterns(v);
 
-            for filter in optimized {
-                let index = flat_builder.add(&filter);
-                insert_dup(&mut filter_map, token, index);
+                for filter in optimized {
+                    let index = flat_builder.add(&filter);
+                    insert_dup(&mut filter_map, token, index);
+                }
             }
+        } else {
+            debug_assert!(
+                optimizable.is_empty(),
+                "Should be empty if optimization is off"
+            );
         }
 
         let flatbuffer_memory = flat_builder.finish();
@@ -393,7 +402,7 @@ impl NetworkFilterListTrait for FlatNetworkFilterList {
                         return Some(CheckResult {
                             filter_mask: filter.mask,
                             modifier_option: filter.modifier_option(),
-                            raw_line: None,
+                            raw_line: filter.raw_line(),
                         });
                     }
                 }
@@ -436,7 +445,7 @@ impl NetworkFilterListTrait for FlatNetworkFilterList {
                         filters.push(CheckResult {
                             filter_mask: filter.mask,
                             modifier_option: filter.modifier_option(),
-                            raw_line: None,
+                            raw_line: filter.raw_line(),
                         });
                     }
                 }

tests/legacy_harness.rs

@@ -314,6 +314,7 @@ mod legacy_test_filters {
     }
 }
 
+#[cfg(not(feature = "flatbuffers"))] // No serialization for flatbuffers yet.
 mod legacy_check_match {
     use adblock::request::Request;
     use adblock::{Engine, Serialize};
@@ -888,6 +889,7 @@ mod legacy_misc_tests {
         assert_eq!(filter.hostname, Some(String::from("test.com")));
     }
 
+    #[cfg(not(feature = "flatbuffers"))] // No serialization for flatbuffers yet.
     #[test]
     fn serialization_tests() {
         let engine = Engine::from_rules_parametrised(

tests/live.rs

@@ -354,6 +354,7 @@ fn check_live_redirects() {
     }
 }
 
+#[cfg(not(feature = "flatbuffers"))] // No serialization for flatbuffers yet.
 #[test]
 /// Ensure that two different engines loaded from the same textual filter set serialize to
 /// identical buffers.
@@ -367,6 +368,7 @@ fn stable_serialization() {
     assert_eq!(ser1, ser2);
 }
 
+#[cfg(not(feature = "flatbuffers"))] // No serialization for flatbuffers yet.
 #[test]
 /// Ensure that one engine's serialization result can be exactly reproduced by another engine after
 /// deserializing from it.

tests/ublock-coverage.rs

@@ -169,6 +169,7 @@ fn check_specifics_default() {
     }
 }
 
+#[cfg(not(feature = "flatbuffers"))] // No serialization for flatbuffers yet.
 #[test]
 fn check_basic_works_after_deserialization() {
     let engine = get_blocker_engine();

tests/unit/blocker.rs

@@ -1410,13 +1410,14 @@ mod blocker_tests {
             enable_optimizations: true,
         };
 
-        let mut blocker = Blocker::new(Vec::new(), &blocker_options);
-        let resources = Default::default();
-
-        blocker
-            .add_filter(NetworkFilter::parse("@@*ad_banner.png", true, Default::default()).unwrap())
+        let mut filter_set = crate::lists::FilterSet::new(true);
+        filter_set
+            .add_filter("@@*ad_banner.png", Default::default())
             .unwrap();
 
+        let blocker = Blocker::new(filter_set.network_filters, &blocker_options);
+        let resources = Default::default();
+
         let request = Request::new(
             "http://example.com/ad_banner.png",
             "https://example.com",
@@ -1435,15 +1436,13 @@ mod blocker_tests {
             enable_optimizations: true,
         };
 
-        let mut blocker = Blocker::new(Vec::new(), &blocker_options);
-
-        blocker
-            .add_filter(
-                NetworkFilter::parse("@@||example.com$generichide", true, Default::default())
-                    .unwrap(),
-            )
+        let mut filter_set = crate::lists::FilterSet::new(true);
+        filter_set
+            .add_filter("@@||example.com$generichide", Default::default())
             .unwrap();
 
+        let blocker = Blocker::new(filter_set.network_filters, &blocker_options);
+
         assert!(blocker.check_generic_hide(
             &Request::new("https://example.com", "https://example.com", "other").unwrap()
         ));