Merge d5d5c0b into sapling-pr-archive-shadaj

.cargo/config.toml

@@ -1,12 +1,8 @@
 [build]
 rustflags = [
-    "-Zproc-macro-backtrace",
     # Flag to make build.rs scripts generate docs. Should only be used in this repository
     # internally, not by dependants.
     '--cfg=HYDROFLOW_GENERATE_DOCS',
-    # https://github.com/rust-lang/rust-clippy/issues/10087
-    ## TODO(mingwei): Need rust-analyzer support:
-    # "-Aclippy::uninlined-format-args",
 ]
 
 [target.x86_64-apple-darwin]

.github/workflows/ci.yml

@@ -36,13 +36,13 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        rust_release: [pinned-nightly, latest-nightly]
+        rust_release: [pinned-nightly, latest-stable]
         exclude:
           # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is
           # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will
           # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998
           - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }}
-          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }}
+          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-stable' }}
 
     env:
       CARGO_TERM_COLOR: always
@@ -55,12 +55,12 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Install nightly toolchain
+      - name: Install toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
-          override: ${{ matrix.rust_release == 'latest-nightly' }}
+          toolchain: stable
+          override: ${{ matrix.rust_release == 'latest-stable' }}
           components: rustfmt, clippy
 
       - name: Run sccache-cache
@@ -96,25 +96,25 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        rust_release: [pinned-nightly, latest-nightly]
+        rust_release: [pinned-nightly, latest-stable]
         exclude:
           # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is
           # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will
           # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998
           - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }}
-          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }}
+          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-stable' }}
 
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Install nightly toolchain
+      - name: Install toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
+          toolchain: stable
           target: wasm32-unknown-unknown
-          override: ${{ matrix.rust_release == 'latest-nightly' }}
+          override: ${{ matrix.rust_release == 'latest-stable' }}
 
       - name: Check hydroflow_lang
         uses: actions-rs/cargo@v1
@@ -132,13 +132,13 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        rust_release: [pinned-nightly, latest-nightly]
+        rust_release: [pinned-nightly, latest-stable]
         exclude:
           # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is
           # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will
           # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998
           - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }}
-          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }}
+          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-stable' }}
           - os: ${{ (github.event_name != 'pull_request' && 'nothing') || 'windows-latest' }}
 
     env:
@@ -152,12 +152,12 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Install nightly toolchain
+      - name: Install toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
-          override: ${{ matrix.rust_release == 'latest-nightly' }}
+          toolchain: stable
+          override: ${{ matrix.rust_release == 'latest-stable' }}
 
       - name: Run sccache-cache
         if: matrix.rust_release == 'pinned-nightly'
@@ -182,6 +182,10 @@ jobs:
 
       - name: Run cargo nextest on all targets
         run: cargo nextest run --no-fail-fast --features python --features deploy --all-targets
+        env:
+          # On stable the output messages will often not match pinned-nightly, so we set to 'overwrite'.
+          TRYBUILD: ${{ matrix.rust_release == 'pinned-nightly' && 'wip' || 'overwrite' }}
+          HYDROFLOW_EXPECT_WARNINGS: ${{ matrix.rust_release == 'pinned-nightly' && 'noop' || 'ignore' }}
 
       - name: Run doctests
         run: cargo test --no-fail-fast --features python --features deploy --doc
@@ -217,7 +221,7 @@ jobs:
           cd python_tests
           pip install -r requirements.txt
           RUST_BACKTRACE=1 pytest
-      
+
       - name: Run Python tests (Windows)
         if: ${{ matrix.os == 'windows-latest' }}
         run: |
@@ -235,25 +239,25 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        rust_release: [pinned-nightly, latest-nightly]
+        rust_release: [pinned-nightly, latest-stable]
         exclude:
           # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is
           # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will
           # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998
           - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }}
-          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }}
+          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-stable' }}
 
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Install nightly toolchain
+      - name: Install toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
+          toolchain: stable
           target: wasm32-unknown-unknown
-          override: ${{ matrix.rust_release == 'latest-nightly' }}
+          override: ${{ matrix.rust_release == 'latest-stable' }}
 
       - name: Get wasm-bindgen version
         id: wasm-bindgen-version
@@ -366,11 +370,10 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Install nightly toolchain
+      - name: Install toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
 
       - name: Checkout gh-pages
         shell: bash

benches/benches/fork_join.rs

@@ -86,9 +86,7 @@ fn benchmark_hydroflow(c: &mut Criterion) {
 fn benchmark_hydroflow_surface(c: &mut Criterion) {
     c.bench_function("fork_join/hydroflow/surface", |b| {
         b.iter(|| {
-            let mut hf = hydroflow_syntax! {
-                source_iter(0..NUM_INTS) -> import!("fork_join_20.hf") -> for_each(|x| { black_box(x); });
-            };
+            let mut hf = include!("fork_join_20.hf");
             hf.run_available();
         })
     });

benches/build.rs

@@ -19,17 +19,21 @@ pub fn fork_join() -> std::io::Result<()> {
     let file = File::create(path)?;
     let mut write = BufWriter::new(file);
 
-    writeln!(write, "a0 = mod -> tee();")?;
-
+    writeln!(write, "hydroflow_syntax! {{")?;
+    writeln!(write, "a0 = source_iter(0..NUM_INTS) -> tee();")?;
     for i in 0..NUM_OPS {
         if i > 0 {
             writeln!(write, "a{} = union() -> tee();", i)?;
         }
         writeln!(write, "a{} -> filter(|x| x % 2 == 0) -> a{};", i, i + 1)?;
         writeln!(write, "a{} -> filter(|x| x % 2 == 1) -> a{};", i, i + 1)?;
     }
-
-    writeln!(write, "a{} = union() -> mod;", NUM_OPS)?;
+    writeln!(
+        write,
+        "a{} = union() -> for_each(|x| {{ black_box(x); }});",
+        NUM_OPS
+    )?;
+    writeln!(write, "}}")?;
 
     write.flush()?;
 

build_docs.bash

@@ -1,8 +1,8 @@
 set -e
 
-PLATFORM=${1:-"x86_64-linux-gnu-ubuntu-16.04"}
+PLATFORM=${1:-"x86_64-linux-gnu-ubuntu-20.04"}
 
-wget -qO- https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang+llvm-13.0.0-$PLATFORM.tar.xz | tar xJ
+wget -qO- https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/clang+llvm-18.1.8-$PLATFORM.tar.xz | tar xJ
 
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 

hydroflow/Cargo.toml

@@ -11,9 +11,8 @@ description = "Hydro's low-level dataflow runtime and IR"
 workspace = true
 
 [features]
-default = [ "macros", "nightly", "debugging" ]
+default = [ "macros", "debugging" ]
 
-nightly = [ "hydroflow_macro", "hydroflow_macro/diagnostics" ]
 macros = [ "hydroflow_macro", "hydroflow_datalog" ]
 hydroflow_macro = [ "dep:hydroflow_macro" ]
 hydroflow_datalog = [ "dep:hydroflow_datalog" ]
@@ -29,14 +28,6 @@ required-features = [ "nightly" ]
 name = "python_udf"
 required-features = [ "python" ]
 
-[[example]]
-name = "modules_outer_join"
-required-features = [ "debugging" ]
-
-[[example]]
-name = "modules_triple_cross_join"
-required-features = [ "debugging" ]
-
 [dependencies]
 bincode = "1.3.1"
 byteorder = "1.3.2"