Releasing Logos2

After months of waiting for a response from @maciejhirsz, regarding the access to publishing to crates.io, I think it is better to move on and publish a new crate, logos2 (and related).

The libraries' name is the same, but crates have a `2` appended at the end. 

This will allow to community to continue working on Logos, and publishing new releases when possible.

.github/workflows/pages.yml

@@ -23,7 +23,7 @@ concurrency:
   group: pages
   cancel-in-progress: true
 
-jobs:  
+jobs:
   # Build job
   build-book:
     runs-on: ubuntu-latest
@@ -33,7 +33,7 @@ jobs:
     - name: Setup mdBook
       uses: peaceiris/actions-mdbook@v1
       with:
-        mdbook-version: '0.4.28'
+        mdbook-version: 0.4.28
     - name: Build book
       run: mdbook build book
     - name: Upload artifact

.github/workflows/rustbench.yml

@@ -7,8 +7,8 @@ on:
 
 name: Benchmark
 
-permissions: 
-   pull-requests: write
+permissions:
+  pull-requests: write
 
 jobs:
   benchmark:

.github/workflows/rustlib.yml

@@ -29,13 +29,13 @@ jobs:
     strategy:
       matrix:
         rust:
-          - stable
-          - beta
-          - nightly
+        - stable
+        - beta
+        - nightly
         os:
-          - macos-latest
-          - ubuntu-latest
-          - windows-latest
+        - macos-latest
+        - ubuntu-latest
+        - windows-latest
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -55,5 +55,5 @@ jobs:
     - name: Check that code compiles
       uses: actions-rs/cargo@v1
       with:
-          command: test
-          args: --verbose
+        command: test
+        args: --verbose

Cargo.toml

@@ -1,6 +1,19 @@
+[profile]
+bench = {lto = true}
+release = {lto = true}
+
 [workspace]
 members = ["logos", "logos-cli", "logos-codegen", "logos-derive", "tests"]
 
-[profile]
-release = { lto = true }
-bench = { lto = true }
+[workspace.package]
+authors = ["Maciej Hirsz <[email protected]>"]
+categories = ["parsing", "text-processing"]
+description = "Create ridiculously fast Lexers"
+edition = "2021"
+homepage = "https://logos.maciej.codes/"
+keywords = ["lexer", "lexical", "tokenizer", "parser", "no_std"]
+license = "MIT OR Apache-2.0"
+readme = "README.md"
+repository = "https://github.com/maciejhirsz/logos"
+rust-version = "1.65.0"
+version = "0.13.0"

book/src/callbacks.md

@@ -66,4 +66,3 @@ Callbacks can be also used to do perform more specialized lexing in place
 where regular expressions are too limiting. For specifics look at
 [`Lexer::remainder`](https://docs.rs/logos/latest/logos/struct.Lexer.html#method.remainder) and
 [`Lexer::bump`](https://docs.rs/logos/latest/logos/struct.Lexer.html#method.bump).
-

book/src/contributing.md

@@ -10,10 +10,10 @@ There are many ways to help us, and here is a short list of some of them:
 + fixing an [BUG](https://github.com/maciejhirsz/logos/labels/bug), by providing
   a patch (or suggesting in the comments how one could fix it);
 + correcting some typos in the documentation, the book, or anywhere else;
-+ raising an issue about a problem (i.e., 
++ raising an issue about a problem (i.e.,
   [opening an issue](https://github.com/maciejhirsz/logos/issues/new) on GitHub);
-+ proposing new features (either with 
-  [an issue](https://github.com/maciejhirsz/logos/issues/new) or 
++ proposing new features (either with
+  [an issue](https://github.com/maciejhirsz/logos/issues/new) or
   [a pull request](https://github.com/maciejhirsz/logos/pulls) on GitHub);
 + or improving the documentation (either in the crate or in the book).
 

book/src/examples.md

@@ -2,6 +2,6 @@
 
 The following examples are ordered by increasing level of complexity.
 
-**[Brainfuck interpreter](./examples/brainfuck.md)**: Lexers are very powerful tools for parsing code programs into meaningful instructions. We show you how you can build an interpreter for the Brainfuck programming language under 100 lines of code! 
+**[Brainfuck interpreter](./examples/brainfuck.md)**: Lexers are very powerful tools for parsing code programs into meaningful instructions. We show you how you can build an interpreter for the Brainfuck programming language under 100 lines of code!
 
-**[JSON parser](./examples/json.md)**: We present a JSON parser written with Logos that does nice error reporting when invalid values are encountered. 
+**[JSON parser](./examples/json.md)**: We present a JSON parser written with Logos that does nice error reporting when invalid values are encountered.

book/src/examples/brainfuck.md

@@ -12,7 +12,7 @@ All other characters must be ignored.
 
 Once the tokens are obtained, a Brainfuck interpreter can be easily created using a [Finite-state machine](https://en.wikipedia.org/wiki/Finite-state_machine). For the sake of simpliciy, we collected all the tokens into one vector called `operations`.
 
-Now, creating an interpreter becomes straightforward[^1]: 
+Now, creating an interpreter becomes straightforward[^1]:
 ```rust,no_run,noplayground
 {{#include ../../../logos/examples/brainfuck.rs:fsm}}
 ```

book/src/getting-started.md

@@ -68,5 +68,3 @@ for result in Token::lexer("Create ridiculously fast Lexers.") {
     }
 }
 ```
-
-

logos-cli/Cargo.toml

@@ -1,23 +1,29 @@
-[package]
+[[bin]]
 name = "logos-cli"
-version = "0.13.0"
-license = "MIT OR Apache-2.0"
-description = "Create ridiculously fast Lexers"
-repository = "https://github.com/maciejhirsz/logos"
-documentation = "https://docs.rs/logos-derive"
-keywords = ["lexer", "lexical", "tokenizer", "parser", "no_std"]
-categories = ["parsing", "text-processing"]
-readme = "../README.md"
-edition = "2018"
+path = "src/main.rs"
 
 [dependencies]
 anyhow = "1.0.57"
-clap = { version = "3.1.18", features = ["derive"] }
+clap = {version = "3.1.18", features = ["derive"]}
 fs-err = "2.7.0"
-logos-codegen = { version = "0.13.0", path = "../logos-codegen" }
+logos-codegen2 = {version = "0.13.0", path = "../logos-codegen"}
 proc-macro2 = "1.0.39"
 
 [dev-dependencies]
 assert_cmd = "2.0.4"
 assert_fs = "1.0.7"
 predicates = "2.1.1"
+
+[package]
+name = "logos-cli2"
+authors.workspace = true
+categories.categories = true
+description.workspace = true
+edition.workspace = true
+homepage.workspace = true
+keywords.workspace = true
+license.workspace = true
+readme.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true

logos-cli/tests/data/input.rs

@@ -2,4 +2,4 @@
 enum Token {
     #[regex("a-z")]
     Letter,
-}
+}

logos-cli/tests/data/output.rs

@@ -1 +1 @@
-# [derive (Debug , Clone , Copy , PartialEq)] enum Token { Letter , }impl < 's > :: logos :: Logos < 's > for Token { type Error = () ; type Extras = () ; type Source = str ; fn lex (lex : & mut :: logos :: Lexer < 's , Self >) { use :: logos :: internal :: { LexerInternal , CallbackResult } ; type Lexer < 's > = :: logos :: Lexer < 's , Token > ; fn _end < 's > (lex : & mut Lexer < 's >) { lex . end () } fn _error < 's > (lex : & mut Lexer < 's >) { lex . bump_unchecked (1) ; lex . error () ; } macro_rules ! _fast_loop { ($ lex : ident , $ test : ident , $ miss : expr) => { while let Some (arr) = $ lex . read :: < & [u8 ; 16] > () { if $ test (arr [0]) { if $ test (arr [1]) { if $ test (arr [2]) { if $ test (arr [3]) { if $ test (arr [4]) { if $ test (arr [5]) { if $ test (arr [6]) { if $ test (arr [7]) { if $ test (arr [8]) { if $ test (arr [9]) { if $ test (arr [10]) { if $ test (arr [11]) { if $ test (arr [12]) { if $ test (arr [13]) { if $ test (arr [14]) { if $ test (arr [15]) { $ lex . bump_unchecked (16) ; continue ; } $ lex . bump_unchecked (15) ; return $ miss ; } $ lex . bump_unchecked (14) ; return $ miss ; } $ lex . bump_unchecked (13) ; return $ miss ; } $ lex . bump_unchecked (12) ; return $ miss ; } $ lex . bump_unchecked (11) ; return $ miss ; } $ lex . bump_unchecked (10) ; return $ miss ; } $ lex . bump_unchecked (9) ; return $ miss ; } $ lex . bump_unchecked (8) ; return $ miss ; } $ lex . bump_unchecked (7) ; return $ miss ; } $ lex . bump_unchecked (6) ; return $ miss ; } $ lex . bump_unchecked (5) ; return $ miss ; } $ lex . bump_unchecked (4) ; return $ miss ; } $ lex . bump_unchecked (3) ; return $ miss ; } $ lex . bump_unchecked (2) ; return $ miss ; } $ lex . bump_unchecked (1) ; return $ miss ; } return $ miss ; } while $ lex . test ($ test) { $ lex . bump_unchecked (1) ; } $ miss } ; } # [inline] fn goto1_x < 's > (lex : & mut Lexer < 's >) { lex . set (Ok (Token :: Letter)) ; } # [inline] fn goto3_at1_with3 < 's > (lex : & mut Lexer < 's >) { match lex . read_at :: < & [u8 ; 2usize] > (1usize) { Some (b"-z") => { lex . bump_unchecked (3usize) ; goto1_x (lex) } , _ => _error (lex) , } } # [inline] fn goto4 < 's > (lex : & mut Lexer < 's >) { let arr = match lex . read :: < & [u8 ; 3usize] > () { Some (arr) => arr , None => return _end (lex) , } ; match arr [0] { b'a' => goto3_at1_with3 (lex) , _ => _error (lex) , } } goto4 (lex) } }
+# [derive (Debug , Clone , Copy , PartialEq)] enum Token { Letter , }impl < 's > :: logos :: Logos < 's > for Token { type Error = () ; type Extras = () ; type Source = str ; fn lex (lex : & mut :: logos :: Lexer < 's , Self >) { use :: logos :: internal :: { LexerInternal , CallbackResult } ; type Lexer < 's > = :: logos :: Lexer < 's , Token > ; fn _end < 's > (lex : & mut Lexer < 's >) { lex . end () } fn _error < 's > (lex : & mut Lexer < 's >) { lex . bump_unchecked (1) ; lex . error () ; } macro_rules ! _fast_loop { ($ lex : ident , $ test : ident , $ miss : expr) => { while let Some (arr) = $ lex . read :: < & [u8 ; 16] > () { if $ test (arr [0]) { if $ test (arr [1]) { if $ test (arr [2]) { if $ test (arr [3]) { if $ test (arr [4]) { if $ test (arr [5]) { if $ test (arr [6]) { if $ test (arr [7]) { if $ test (arr [8]) { if $ test (arr [9]) { if $ test (arr [10]) { if $ test (arr [11]) { if $ test (arr [12]) { if $ test (arr [13]) { if $ test (arr [14]) { if $ test (arr [15]) { $ lex . bump_unchecked (16) ; continue ; } $ lex . bump_unchecked (15) ; return $ miss ; } $ lex . bump_unchecked (14) ; return $ miss ; } $ lex . bump_unchecked (13) ; return $ miss ; } $ lex . bump_unchecked (12) ; return $ miss ; } $ lex . bump_unchecked (11) ; return $ miss ; } $ lex . bump_unchecked (10) ; return $ miss ; } $ lex . bump_unchecked (9) ; return $ miss ; } $ lex . bump_unchecked (8) ; return $ miss ; } $ lex . bump_unchecked (7) ; return $ miss ; } $ lex . bump_unchecked (6) ; return $ miss ; } $ lex . bump_unchecked (5) ; return $ miss ; } $ lex . bump_unchecked (4) ; return $ miss ; } $ lex . bump_unchecked (3) ; return $ miss ; } $ lex . bump_unchecked (2) ; return $ miss ; } $ lex . bump_unchecked (1) ; return $ miss ; } return $ miss ; } while $ lex . test ($ test) { $ lex . bump_unchecked (1) ; } $ miss } ; } # [inline] fn goto1_x < 's > (lex : & mut Lexer < 's >) { lex . set (Ok (Token :: Letter)) ; } # [inline] fn goto3_at1_with3 < 's > (lex : & mut Lexer < 's >) { match lex . read_at :: < & [u8 ; 2usize] > (1usize) { Some (b"-z") => { lex . bump_unchecked (3usize) ; goto1_x (lex) } , _ => _error (lex) , } } # [inline] fn goto4 < 's > (lex : & mut Lexer < 's >) { let arr = match lex . read :: < & [u8 ; 3usize] > () { Some (arr) => arr , None => return _end (lex) , } ; match arr [0] { b'a' => goto3_at1_with3 (lex) , _ => _error (lex) , } } goto4 (lex) } }

logos-codegen/Cargo.toml

@@ -1,24 +1,29 @@
-[package]
-name = "logos-codegen"
-version = "0.13.0"
-authors = ["Maciej Hirsz <[email protected]>"]
-license = "MIT OR Apache-2.0"
-description = "Implementation details for logos-codegen and logos-derive. Not for public consumption."
-repository = "https://github.com/maciejhirsz/logos"
-documentation = "https://docs.rs/logos-derive"
-keywords = ["lexer", "lexical", "tokenizer", "parser", "no_std"]
-categories = ["parsing", "text-processing"]
-readme = "../README.md"
-edition = "2018"
-
 [dependencies]
 beef = "0.5.0"
 fnv = "1.0.6"
-syn = { version = "2.0.13", features = ["full"] }
+lazy_static = "1.4.0"
+proc-macro2 = "1.0.9"
 quote = "1.0.3"
 proc-macro2 = "1.0.9"
 regex-syntax = "0.6"
 lazy_static = "1.4.0"
 
 [dev-dependencies]
 pretty_assertions = "0.6.1"
+
+[lib]
+name = "logos_codegen"
+
+[package]
+name = "logos-codegen2"
+authors.workspace = true
+categories.categories = true
+description.workspace = true
+edition.workspace = true
+homepage.workspace = true
+keywords.workspace = true
+license.workspace = true
+readme.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true

logos-derive/Cargo.toml

@@ -1,19 +1,20 @@
-[package]
-name = "logos-derive"
-version = "0.13.0"
-authors = ["Maciej Hirsz <[email protected]>"]
-license = "MIT OR Apache-2.0"
-description = "Create ridiculously fast Lexers"
-repository = "https://github.com/maciejhirsz/logos"
-documentation = "https://docs.rs/logos-derive"
-keywords = ["lexer", "lexical", "tokenizer", "parser", "no_std"]
-categories = ["parsing", "text-processing"]
-readme = "../README.md"
-edition = "2018"
+[dependencies]
+logos-codegen2 = {version = "0.13.0", path = "../logos-codegen"}
 
 [lib]
 name = "logos_derive"
 proc-macro = true
 
-[dependencies]
-logos-codegen = { version = "0.13.0", path = "../logos-codegen" }
+[package]
+name = "logos-derive2"
+authors.workspace = true
+categories.categories = true
+description.workspace = true
+edition.workspace = true
+homepage.workspace = true
+keywords.workspace = true
+license.workspace = true
+readme.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true

logos/Cargo.toml

@@ -1,43 +1,13 @@
-[package]
-name = "logos"
-version = "0.13.0"
-authors = ["Maciej Hirsz <[email protected]>"]
-license = "MIT OR Apache-2.0"
-description = "Create ridiculously fast Lexers"
-repository = "https://github.com/maciejhirsz/logos"
-documentation = "https://docs.rs/logos"
-keywords = ["lexer", "lexical", "tokenizer", "parser", "no_std"]
-categories = ["parsing", "text-processing"]
-readme = "../README.md"
-edition = "2021"
-rust-version = "1.65.0"
-
-[package.metadata.docs.rs]
-all-features = true
-cargo-args = ["-Zunstable-options", "-Zrustdoc-scrape-examples"]
-rustdoc-args = ["--cfg", "docsrs"]
-
 [dependencies]
-logos-derive = { version = "0.13.0", path = "../logos-derive", optional = true }
+logos-derive2 = {version = "0.13.0", path = "../logos-derive", optional = true}
 
 [dev-dependencies]
-ariadne = { version = "0.2.0", features = ["auto-color"] }
-
-[features]
-default = ["export_derive", "std"]
-
-# Should the crate use the standard library?
-std = []
-
-# Re-exports the `Logos` derive macro, so that end user only needs to
-# import this crate and `use logos::Logos` to get both the trait and
-# derive proc macro.
-export_derive = ["logos-derive"]
+ariadne = {version = "0.2.0", features = ["auto-color"]}
 
 [[example]]
+doc-scrape-examples = true  # Only needed once, because requires dev-dependencies
 name = "brainfuck"
 path = "examples/brainfuck.rs"
-doc-scrape-examples = true  # Only needed once, because requires dev-dependencies
 
 [[example]]
 name = "custom_error"
@@ -50,3 +20,34 @@ path = "examples/extras.rs"
 [[example]]
 name = "json"
 path = "examples/json.rs"
+
+[features]
+default = ["export_derive", "std"]
+# Re-exports the `Logos` derive macro, so that end user only needs to
+# import this crate and `use logos::Logos` to get both the trait and
+# derive proc macro.
+export_derive = ["logos-derive2"]
+# Should the crate use the standard library?
+std = []
+
+[lib]
+name = "logos"
+
+[package]
+name = "logos2"
+authors.workspace = true
+categories.categories = true
+description.workspace = true
+edition.workspace = true
+homepage.workspace = true
+keywords.workspace = true
+license.workspace = true
+
+[package.metadata.docs.rs]
+all-features = true
+cargo-args = ["-Zunstable-options", "-Zrustdoc-scrape-examples"]
+rustdoc-args = ["--cfg", "docsrs"]
+readme.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true

logos/examples/hello_world.bf

@@ -40,4 +40,4 @@ Pointer :   ^
 <.                      Cell #3 was set to 'o' from the end of 'Hello'
 +++.------.--------.    Cell #3 for 'rl' and 'd'
 >>+.                    Add 1 to Cell #5 gives us an exclamation point
->++.                    And finally a newline from Cell #6        
+>++.                    And finally a newline from Cell #6

tests/Cargo.toml

@@ -1,18 +1,15 @@
-[package]
-name = "tests"
-version = "0.0.0"
-authors = ["Maciej Hirsz <[email protected]>"]
-license = "MIT OR Apache-2.0"
-repository = "https://github.com/maciejhirsz/logos"
-edition = "2018"
+[[bench]]
+harness = false
+name = "bench"
 
 [dependencies]
-logos = { path = "../logos", default-features = false, features = ["std"] }
-logos-derive = { path = "../logos-derive" }
+logos-derive2 = {path = "../logos-derive"}
+logos2 = {path = "../logos", default-features = false, features = ["std"]}
 
 [dev-dependencies]
 criterion = "0.4"
 
-[[bench]]
-name = "bench"
-harness = false
+[package]
+name = "tests"
+publish = false
+version = "0.0.0"