Merge branch 'main' into repeat-n

.cargo/config.toml

@@ -1,12 +1,8 @@
 [build]
 rustflags = [
-    "-Zproc-macro-backtrace",
     # Flag to make build.rs scripts generate docs. Should only be used in this repository
     # internally, not by dependants.
     '--cfg=HYDROFLOW_GENERATE_DOCS',
-    # https://github.com/rust-lang/rust-clippy/issues/10087
-    ## TODO(mingwei): Need rust-analyzer support:
-    # "-Aclippy::uninlined-format-args",
 ]
 
 [target.x86_64-apple-darwin]

.github/workflows/ci.yml

@@ -36,13 +36,13 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        rust_release: [pinned-nightly, latest-nightly]
+        rust_release: [pinned-nightly, latest-stable]
         exclude:
           # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is
           # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will
           # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998
           - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }}
-          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }}
+          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-stable' }}
 
     env:
       CARGO_TERM_COLOR: always
@@ -55,12 +55,12 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Install nightly toolchain
+      - name: Install toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
-          override: ${{ matrix.rust_release == 'latest-nightly' }}
+          toolchain: stable
+          override: ${{ matrix.rust_release == 'latest-stable' }}
           components: rustfmt, clippy
 
       - name: Run sccache-cache
@@ -96,25 +96,25 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        rust_release: [pinned-nightly, latest-nightly]
+        rust_release: [pinned-nightly, latest-stable]
         exclude:
           # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is
           # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will
           # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998
           - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }}
-          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }}
+          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-stable' }}
 
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Install nightly toolchain
+      - name: Install toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
+          toolchain: stable
           target: wasm32-unknown-unknown
-          override: ${{ matrix.rust_release == 'latest-nightly' }}
+          override: ${{ matrix.rust_release == 'latest-stable' }}
 
       - name: Check hydroflow_lang
         uses: actions-rs/cargo@v1
@@ -132,13 +132,13 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        rust_release: [pinned-nightly, latest-nightly]
+        rust_release: [pinned-nightly, latest-stable]
         exclude:
           # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is
           # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will
           # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998
           - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }}
-          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }}
+          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-stable' }}
           - os: ${{ (github.event_name != 'pull_request' && 'nothing') || 'windows-latest' }}
 
     env:
@@ -152,12 +152,12 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Install nightly toolchain
+      - name: Install toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
-          override: ${{ matrix.rust_release == 'latest-nightly' }}
+          toolchain: stable
+          override: ${{ matrix.rust_release == 'latest-stable' }}
 
       - name: Run sccache-cache
         if: matrix.rust_release == 'pinned-nightly'
@@ -182,6 +182,10 @@ jobs:
 
       - name: Run cargo nextest on all targets
         run: cargo nextest run --no-fail-fast --features python --features deploy --all-targets
+        env:
+          # On stable the output messages will often not match pinned-nightly, so we set to 'overwrite'.
+          TRYBUILD: ${{ matrix.rust_release == 'pinned-nightly' && 'wip' || 'overwrite' }}
+          HYDROFLOW_EXPECT_WARNINGS: ${{ matrix.rust_release == 'pinned-nightly' && 'noop' || 'ignore' }}
 
       - name: Run doctests
         run: cargo test --no-fail-fast --features python --features deploy --doc
@@ -217,7 +221,7 @@ jobs:
           cd python_tests
           pip install -r requirements.txt
           RUST_BACKTRACE=1 pytest
-      
+
       - name: Run Python tests (Windows)
         if: ${{ matrix.os == 'windows-latest' }}
         run: |
@@ -235,25 +239,25 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        rust_release: [pinned-nightly, latest-nightly]
+        rust_release: [pinned-nightly, latest-stable]
         exclude:
           # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is
           # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will
           # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998
           - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }}
-          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }}
+          - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-stable' }}
 
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Install nightly toolchain
+      - name: Install toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
+          toolchain: stable
           target: wasm32-unknown-unknown
-          override: ${{ matrix.rust_release == 'latest-nightly' }}
+          override: ${{ matrix.rust_release == 'latest-stable' }}
 
       - name: Get wasm-bindgen version
         id: wasm-bindgen-version
@@ -366,11 +370,10 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v3
 
-      - name: Install nightly toolchain
+      - name: Install toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
 
       - name: Checkout gh-pages
         shell: bash

benches/benches/fork_join.rs

@@ -86,9 +86,7 @@ fn benchmark_hydroflow(c: &mut Criterion) {
 fn benchmark_hydroflow_surface(c: &mut Criterion) {
     c.bench_function("fork_join/hydroflow/surface", |b| {
         b.iter(|| {
-            let mut hf = hydroflow_syntax! {
-                source_iter(0..NUM_INTS) -> import!("fork_join_20.hf") -> for_each(|x| { black_box(x); });
-            };
+            let mut hf = include!("fork_join_20.hf");
             hf.run_available();
         })
     });

benches/build.rs

@@ -19,17 +19,21 @@ pub fn fork_join() -> std::io::Result<()> {
     let file = File::create(path)?;
     let mut write = BufWriter::new(file);
 
-    writeln!(write, "a0 = mod -> tee();")?;
-
+    writeln!(write, "hydroflow_syntax! {{")?;
+    writeln!(write, "a0 = source_iter(0..NUM_INTS) -> tee();")?;
     for i in 0..NUM_OPS {
         if i > 0 {
             writeln!(write, "a{} = union() -> tee();", i)?;
         }
         writeln!(write, "a{} -> filter(|x| x % 2 == 0) -> a{};", i, i + 1)?;
         writeln!(write, "a{} -> filter(|x| x % 2 == 1) -> a{};", i, i + 1)?;
     }
-
-    writeln!(write, "a{} = union() -> mod;", NUM_OPS)?;
+    writeln!(
+        write,
+        "a{} = union() -> for_each(|x| {{ black_box(x); }});",
+        NUM_OPS
+    )?;
+    writeln!(write, "}}")?;
 
     write.flush()?;
 

docs/src/pages/people.js

@@ -12,6 +12,7 @@ import mingwei from "./people-img/mingwei.jpeg"
 import natacha from "./people-img/natacha.jpeg"
 import shadaj from "./people-img/shadaj.png"
 import lucky from "./people-img/lucky.jpeg"
+import rohit from "./people-img/rohit.jpg"
 import hydroTurtle from "./people-img/hydro-turtle.png"
 
 import styles from './people.module.css';
@@ -73,12 +74,7 @@ export default function Home() {
               <PersonCard name={"Alvin Cheung"} role={"Faculty"} url={"https://people.eecs.berkeley.edu/~akcheung"} img={akcheung}></PersonCard>
               <PersonCard name={"Natacha Crooks"} role={"Faculty"} url={"https://nacrooks.github.io"} img={natacha}></PersonCard>
               <PersonCard name={"Joe Hellerstein"} role={"Faculty"} url={"https://dsf.berkeley.edu/jmh"} img={jmh}></PersonCard>
-            </div>
-
-            <div className={styles["subtitle"]}>Postdocs</div>
-            <div className={styles["personGroup"]}>
-              <PersonCard name={"Tiemo Bang"} role={"Postdoc"} url={"https://scholar.google.com/citations?user=HDK0KRYAAAAJ&hl=en"} img={hydroTurtle}></PersonCard>
-              <PersonCard name={"Mae Milano"} role={"Postdoc"} url={"http://www.languagesforsyste.ms"} img={mae}></PersonCard>
+              <PersonCard name={"Mae Milano"} role={"Faculty"} url={"http://www.languagesforsyste.ms"} img={mae}></PersonCard>
             </div>
 
             <div className={styles["subtitle"]}>Graduate Students & Research Engineers</div>
@@ -88,7 +84,14 @@ export default function Home() {
               <PersonCard name={"Shadaj Laddad"} role={"PhD Student"} url={"https://www.shadaj.me"} img={shadaj}></PersonCard>
               <PersonCard name={"Conor Power"} role={"PhD Student"} url={"https://www.linkedin.com/in/conorpower23"} img={conor}></PersonCard>
               <PersonCard name={"Mingwei Samuel"} role={"Research Engineer"} url={"https://github.com/MingweiSamuel"} img={mingwei}></PersonCard>
+              <PersonCard name={"Rohit Kulshreshtha"} role={"Research Engineer"} url={"https://www.linkedin.com/in/lucky-k-59020457/"} img={rohit}></PersonCard>
+            </div>
+
+            <div className={styles["subtitle"]}>Alumni</div>
+            <div className={styles["personGroup"]}>
+              <PersonCard name={"Tiemo Bang"} role={"Postdoc"} url={"https://scholar.google.com/citations?user=HDK0KRYAAAAJ&hl=en"} img={hydroTurtle}></PersonCard>
               <PersonCard name={"Lucky Katahanas"} role={"Research Engineer"} url={"https://www.linkedin.com/in/lucky-k-59020457/"} img={lucky}></PersonCard>
+              <PersonCard name={"Justin Jaffray"} role={"Research Engineer"} url={"https://www.linkedin.com/in/justinjaffray/"} img={hydroTurtle}></PersonCard>
             </div>
           </div>
         </div>

hydroflow/Cargo.toml

@@ -11,9 +11,8 @@ description = "Hydro's low-level dataflow runtime and IR"
 workspace = true
 
 [features]
-default = [ "macros", "nightly", "debugging" ]
+default = [ "macros", "debugging" ]
 
-nightly = [ "hydroflow_macro", "hydroflow_macro/diagnostics" ]
 macros = [ "hydroflow_macro", "hydroflow_datalog" ]
 hydroflow_macro = [ "dep:hydroflow_macro" ]
 hydroflow_datalog = [ "dep:hydroflow_datalog" ]
@@ -29,14 +28,6 @@ required-features = [ "nightly" ]
 name = "python_udf"
 required-features = [ "python" ]
 
-[[example]]
-name = "modules_outer_join"
-required-features = [ "debugging" ]
-
-[[example]]
-name = "modules_triple_cross_join"
-required-features = [ "debugging" ]
-
 [dependencies]
 bincode = "1.3.1"
 byteorder = "1.3.2"