move into new index module (#2259)

move core modules to index module

src/collector/facet_collector.rs

@@ -493,7 +493,7 @@ mod tests {
     use super::{FacetCollector, FacetCounts};
     use crate::collector::facet_collector::compress_mapping;
     use crate::collector::Count;
-    use crate::core::Index;
+    use crate::index::Index;
     use crate::query::{AllQuery, QueryParser, TermQuery};
     use crate::schema::{Facet, FacetOptions, IndexRecordOption, Schema, TantivyDocument};
     use crate::{IndexWriter, Term};

src/collector/tests.rs

@@ -2,7 +2,7 @@ use columnar::{BytesColumn, Column};
 
 use super::*;
 use crate::collector::{Count, FilterCollector, TopDocs};
-use crate::core::SegmentReader;
+use crate::index::SegmentReader;
 use crate::query::{AllQuery, QueryParser};
 use crate::schema::{Schema, FAST, TEXT};
 use crate::time::format_description::well_known::Rfc3339;

src/core/mod.rs

@@ -1,32 +1,14 @@
 mod executor;
-pub mod index;
-mod index_meta;
-mod inverted_index_reader;
 #[doc(hidden)]
 pub mod json_utils;
 pub mod searcher;
-mod segment;
-mod segment_component;
-mod segment_id;
-mod segment_reader;
-mod single_segment_index_writer;
 
 use std::path::Path;
 
 use once_cell::sync::Lazy;
 
 pub use self::executor::Executor;
-pub use self::index::{Index, IndexBuilder};
-pub use self::index_meta::{
-    IndexMeta, IndexSettings, IndexSortByField, Order, SegmentMeta, SegmentMetaInventory,
-};
-pub use self::inverted_index_reader::InvertedIndexReader;
 pub use self::searcher::{Searcher, SearcherGeneration};
-pub use self::segment::Segment;
-pub use self::segment_component::SegmentComponent;
-pub use self::segment_id::SegmentId;
-pub use self::segment_reader::{merge_field_meta_data, FieldMetadata, SegmentReader};
-pub use self::single_segment_index_writer::SingleSegmentIndexWriter;
 
 /// The meta file contains all the information about the list of segments and the schema
 /// of the index.

src/core/searcher.rs

@@ -3,7 +3,8 @@ use std::sync::Arc;
 use std::{fmt, io};
 
 use crate::collector::Collector;
-use crate::core::{Executor, SegmentReader};
+use crate::core::Executor;
+use crate::index::SegmentReader;
 use crate::query::{Bm25StatisticsProvider, EnableScoring, Query};
 use crate::schema::document::DocumentDeserialize;
 use crate::schema::{Schema, Term};

src/directory/mmap_directory.rs

@@ -673,7 +673,7 @@ mod tests {
             let num_segments = reader.searcher().segment_readers().len();
             assert!(num_segments <= 4);
             let num_components_except_deletes_and_tempstore =
-                crate::core::SegmentComponent::iterator().len() - 2;
+                crate::index::SegmentComponent::iterator().len() - 2;
             let max_num_mmapped = num_components_except_deletes_and_tempstore * num_segments;
             assert_eventually(|| {
                 let num_mmapped = mmap_directory.get_cache_info().mmapped.len();

src/core/index.rs → src/index/index.rs

@@ -6,24 +6,23 @@ use std::path::PathBuf;
 use std::sync::Arc;
 
 use super::segment::Segment;
-use super::IndexSettings;
-use crate::core::single_segment_index_writer::SingleSegmentIndexWriter;
-use crate::core::{
-    Executor, IndexMeta, SegmentId, SegmentMeta, SegmentMetaInventory, META_FILEPATH,
-};
+use super::segment_reader::merge_field_meta_data;
+use super::{FieldMetadata, IndexSettings};
+use crate::core::{Executor, META_FILEPATH};
 use crate::directory::error::OpenReadError;
 #[cfg(feature = "mmap")]
 use crate::directory::MmapDirectory;
 use crate::directory::{Directory, ManagedDirectory, RamDirectory, INDEX_WRITER_LOCK};
 use crate::error::{DataCorruption, TantivyError};
+use crate::index::{IndexMeta, SegmentId, SegmentMeta, SegmentMetaInventory};
 use crate::indexer::index_writer::{MAX_NUM_THREAD, MEMORY_BUDGET_NUM_BYTES_MIN};
 use crate::indexer::segment_updater::save_metas;
-use crate::indexer::IndexWriter;
+use crate::indexer::{IndexWriter, SingleSegmentIndexWriter};
 use crate::reader::{IndexReader, IndexReaderBuilder};
 use crate::schema::document::Document;
 use crate::schema::{Field, FieldType, Schema};
 use crate::tokenizer::{TextAnalyzer, TokenizerManager};
-use crate::{merge_field_meta_data, FieldMetadata, SegmentReader};
+use crate::SegmentReader;
 
 fn load_metas(
     directory: &dyn Directory,

src/core/index_meta.rs → src/index/index_meta.rs

@@ -7,7 +7,7 @@ use std::sync::Arc;
 use serde::{Deserialize, Serialize};
 
 use super::SegmentComponent;
-use crate::core::SegmentId;
+use crate::index::SegmentId;
 use crate::schema::Schema;
 use crate::store::Compressor;
 use crate::{Inventory, Opstamp, TrackedObject};
@@ -19,7 +19,7 @@ struct DeleteMeta {
 }
 
 #[derive(Clone, Default)]
-pub struct SegmentMetaInventory {
+pub(crate) struct SegmentMetaInventory {
     inventory: Inventory<InnerSegmentMeta>,
 }
 
@@ -408,7 +408,7 @@ impl fmt::Debug for IndexMeta {
 mod tests {
 
     use super::IndexMeta;
-    use crate::core::index_meta::UntrackedIndexMeta;
+    use crate::index::index_meta::UntrackedIndexMeta;
     use crate::schema::{Schema, TEXT};
     use crate::store::Compressor;
     #[cfg(feature = "zstd-compression")]

src/index/mod.rs

@@ -0,0 +1,22 @@
+//! # Index Module
+//!
+//! The `index` module in Tantivy contains core components to read and write indexes.
+//!
+//! It contains `Index` and `Segment`, where a `Index` consists of one or more `Segment`s.
+
+mod index;
+mod index_meta;
+mod inverted_index_reader;
+mod segment;
+mod segment_component;
+mod segment_id;
+mod segment_reader;
+
+pub use self::index::{Index, IndexBuilder};
+pub(crate) use self::index_meta::SegmentMetaInventory;
+pub use self::index_meta::{IndexMeta, IndexSettings, IndexSortByField, Order, SegmentMeta};
+pub use self::inverted_index_reader::InvertedIndexReader;
+pub use self::segment::Segment;
+pub use self::segment_component::SegmentComponent;
+pub use self::segment_id::SegmentId;
+pub use self::segment_reader::{FieldMetadata, SegmentReader};

src/core/segment.rs → src/index/segment.rs

@@ -2,9 +2,9 @@ use std::fmt;
 use std::path::PathBuf;
 
 use super::SegmentComponent;
-use crate::core::{Index, SegmentId, SegmentMeta};
 use crate::directory::error::{OpenReadError, OpenWriteError};
 use crate::directory::{Directory, FileSlice, WritePtr};
+use crate::index::{Index, SegmentId, SegmentMeta};
 use crate::schema::Schema;
 use crate::Opstamp;
 

src/core/segment_reader.rs → src/index/segment_reader.rs

@@ -6,11 +6,11 @@ use std::{fmt, io};
 use fnv::FnvHashMap;
 use itertools::Itertools;
 
-use crate::core::{InvertedIndexReader, Segment, SegmentComponent, SegmentId};
 use crate::directory::{CompositeFile, FileSlice};
 use crate::error::DataCorruption;
 use crate::fastfield::{intersect_alive_bitsets, AliveBitSet, FacetReader, FastFieldReaders};
 use crate::fieldnorm::{FieldNormReader, FieldNormReaders};
+use crate::index::{InvertedIndexReader, Segment, SegmentComponent, SegmentId};
 use crate::json_utils::json_path_sep_to_dot;
 use crate::schema::{Field, IndexRecordOption, Schema, Type};
 use crate::space_usage::SegmentSpaceUsage;
@@ -515,9 +515,9 @@ impl fmt::Debug for SegmentReader {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::core::Index;
+    use crate::index::Index;
     use crate::schema::{Schema, SchemaBuilder, Term, STORED, TEXT};
-    use crate::{DocId, FieldMetadata, IndexWriter};
+    use crate::{DocId, IndexWriter};
 
     #[test]
     fn test_merge_field_meta_data_same() {

src/indexer/index_writer.rs

@@ -9,10 +9,10 @@ use smallvec::smallvec;
 use super::operation::{AddOperation, UserOperation};
 use super::segment_updater::SegmentUpdater;
 use super::{AddBatch, AddBatchReceiver, AddBatchSender, PreparedCommit};
-use crate::core::{Index, Segment, SegmentComponent, SegmentId, SegmentMeta, SegmentReader};
 use crate::directory::{DirectoryLock, GarbageCollectionResult, TerminatingWrite};
 use crate::error::TantivyError;
 use crate::fastfield::write_alive_bitset;
+use crate::index::{Index, Segment, SegmentComponent, SegmentId, SegmentMeta, SegmentReader};
 use crate::indexer::delete_queue::{DeleteCursor, DeleteQueue};
 use crate::indexer::doc_opstamp_mapping::DocToOpstampMapping;
 use crate::indexer::index_writer_status::IndexWriterStatus;

src/indexer/log_merge_policy.rs

@@ -3,7 +3,7 @@ use std::cmp;
 use itertools::Itertools;
 
 use super::merge_policy::{MergeCandidate, MergePolicy};
-use crate::core::SegmentMeta;
+use crate::index::SegmentMeta;
 
 const DEFAULT_LEVEL_LOG_SIZE: f64 = 0.75;
 const DEFAULT_MIN_LAYER_SIZE: u32 = 10_000;
@@ -144,7 +144,7 @@ mod tests {
     use once_cell::sync::Lazy;
 
     use super::*;
-    use crate::core::{SegmentId, SegmentMeta, SegmentMetaInventory};
+    use crate::index::{SegmentId, SegmentMeta, SegmentMetaInventory};
     use crate::indexer::merge_policy::MergePolicy;
     use crate::schema;
     use crate::schema::INDEXED;

src/indexer/merge_policy.rs

@@ -1,7 +1,7 @@
 use std::fmt::Debug;
 use std::marker;
 
-use crate::core::{SegmentId, SegmentMeta};
+use crate::index::{SegmentId, SegmentMeta};
 
 /// Set of segment suggested for a merge.
 #[derive(Debug, Clone)]
@@ -39,7 +39,7 @@ impl MergePolicy for NoMergePolicy {
 pub mod tests {
 
     use super::*;
-    use crate::core::{SegmentId, SegmentMeta};
+    use crate::index::{SegmentId, SegmentMeta};
 
     /// `MergePolicy` useful for test purposes.
     ///

src/indexer/merger.rs

@@ -8,12 +8,12 @@ use common::ReadOnlyBitSet;
 use itertools::Itertools;
 use measure_time::debug_time;
 
-use crate::core::{Segment, SegmentReader};
 use crate::directory::WritePtr;
 use crate::docset::{DocSet, TERMINATED};
 use crate::error::DataCorruption;
 use crate::fastfield::{AliveBitSet, FastFieldNotAvailableError};
 use crate::fieldnorm::{FieldNormReader, FieldNormReaders, FieldNormsSerializer, FieldNormsWriter};
+use crate::index::{Segment, SegmentReader};
 use crate::indexer::doc_id_mapping::{MappingType, SegmentDocIdMapping};
 use crate::indexer::SegmentSerializer;
 use crate::postings::{InvertedIndexSerializer, Postings, SegmentPostings};
@@ -794,7 +794,7 @@ mod tests {
         BytesFastFieldTestCollector, FastFieldTestCollector, TEST_COLLECTOR_WITH_SCORE,
     };
     use crate::collector::{Count, FacetCollector};
-    use crate::core::Index;
+    use crate::index::Index;
     use crate::query::{AllQuery, BooleanQuery, EnableScoring, Scorer, TermQuery};
     use crate::schema::document::Value;
     use crate::schema::{

src/indexer/merger_sorted_index_test.rs

@@ -1,8 +1,8 @@
 #[cfg(test)]
 mod tests {
     use crate::collector::TopDocs;
-    use crate::core::Index;
     use crate::fastfield::AliveBitSet;
+    use crate::index::Index;
     use crate::query::QueryParser;
     use crate::schema::document::Value;
     use crate::schema::{
@@ -485,7 +485,7 @@ mod bench_sorted_index_merge {
 
     use test::{self, Bencher};
 
-    use crate::core::Index;
+    use crate::index::Index;
     use crate::indexer::merger::IndexMerger;
     use crate::schema::{NumericOptions, Schema};
     use crate::{IndexSettings, IndexSortByField, IndexWriter, Order};

src/indexer/mod.rs

@@ -25,6 +25,7 @@ mod segment_register;
 pub(crate) mod segment_serializer;
 pub(crate) mod segment_updater;
 pub(crate) mod segment_writer;
+pub(crate) mod single_segment_index_writer;
 mod stamper;
 
 use crossbeam_channel as channel;
@@ -34,13 +35,14 @@ pub use self::index_writer::IndexWriter;
 pub use self::log_merge_policy::LogMergePolicy;
 pub use self::merge_operation::MergeOperation;
 pub use self::merge_policy::{MergeCandidate, MergePolicy, NoMergePolicy};
+use self::operation::AddOperation;
 pub use self::operation::UserOperation;
 pub use self::prepared_commit::PreparedCommit;
 pub use self::segment_entry::SegmentEntry;
 pub(crate) use self::segment_serializer::SegmentSerializer;
 pub use self::segment_updater::{merge_filtered_segments, merge_indices};
 pub use self::segment_writer::SegmentWriter;
-use crate::indexer::operation::AddOperation;
+pub use self::single_segment_index_writer::SingleSegmentIndexWriter;
 
 /// Alias for the default merge policy, which is the `LogMergePolicy`.
 pub type DefaultMergePolicy = LogMergePolicy;
@@ -63,9 +65,10 @@ mod tests_mmap {
     use crate::aggregation::agg_result::AggregationResults;
     use crate::aggregation::AggregationCollector;
     use crate::collector::{Count, TopDocs};
+    use crate::index::FieldMetadata;
     use crate::query::{AllQuery, QueryParser};
     use crate::schema::{JsonObjectOptions, Schema, Type, FAST, INDEXED, STORED, TEXT};
-    use crate::{FieldMetadata, Index, IndexWriter, Term};
+    use crate::{Index, IndexWriter, Term};
 
     #[test]
     fn test_advance_delete_bug() -> crate::Result<()> {

src/indexer/segment_entry.rs

@@ -2,7 +2,7 @@ use std::fmt;
 
 use common::BitSet;
 
-use crate::core::{SegmentId, SegmentMeta};
+use crate::index::{SegmentId, SegmentMeta};
 use crate::indexer::delete_queue::DeleteCursor;
 
 /// A segment entry describes the state of

src/indexer/segment_manager.rs

@@ -3,8 +3,8 @@ use std::fmt::{self, Debug, Formatter};
 use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
 
 use super::segment_register::SegmentRegister;
-use crate::core::{SegmentId, SegmentMeta};
 use crate::error::TantivyError;
+use crate::index::{SegmentId, SegmentMeta};
 use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::SegmentEntry;
 

src/indexer/segment_register.rs

@@ -1,7 +1,7 @@
 use std::collections::{HashMap, HashSet};
 use std::fmt::{self, Debug, Display, Formatter};
 
-use crate::core::{SegmentId, SegmentMeta};
+use crate::index::{SegmentId, SegmentMeta};
 use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::segment_entry::SegmentEntry;
 
@@ -103,7 +103,7 @@ impl SegmentRegister {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::core::{SegmentId, SegmentMetaInventory};
+    use crate::index::{SegmentId, SegmentMetaInventory};
     use crate::indexer::delete_queue::*;
 
     fn segment_ids(segment_register: &SegmentRegister) -> Vec<SegmentId> {

src/indexer/segment_serializer.rs

@@ -1,8 +1,8 @@
 use common::TerminatingWrite;
 
-use crate::core::{Segment, SegmentComponent};
 use crate::directory::WritePtr;
 use crate::fieldnorm::FieldNormsSerializer;
+use crate::index::{Segment, SegmentComponent};
 use crate::postings::InvertedIndexSerializer;
 use crate::store::StoreWriter;
 

src/indexer/segment_updater.rs

@@ -9,11 +9,10 @@ use std::sync::{Arc, RwLock};
 use rayon::{ThreadPool, ThreadPoolBuilder};
 
 use super::segment_manager::SegmentManager;
-use crate::core::{
-    Index, IndexMeta, IndexSettings, Segment, SegmentId, SegmentMeta, META_FILEPATH,
-};
+use crate::core::META_FILEPATH;
 use crate::directory::{Directory, DirectoryClone, GarbageCollectionResult};
 use crate::fastfield::AliveBitSet;
+use crate::index::{Index, IndexMeta, IndexSettings, Segment, SegmentId, SegmentMeta};
 use crate::indexer::delete_queue::DeleteCursor;
 use crate::indexer::index_writer::advance_deletes;
 use crate::indexer::merge_operation::MergeOperationInventory;

src/indexer/segment_writer.rs

@@ -6,9 +6,9 @@ use tokenizer_api::BoxTokenStream;
 use super::doc_id_mapping::{get_doc_id_mapping_from_field, DocIdMapping};
 use super::operation::AddOperation;
 use crate::core::json_utils::index_json_values;
-use crate::core::Segment;
 use crate::fastfield::FastFieldsWriter;
 use crate::fieldnorm::{FieldNormReaders, FieldNormsWriter};
+use crate::index::Segment;
 use crate::indexer::segment_serializer::SegmentSerializer;
 use crate::postings::{
     compute_table_memory_size, serialize_postings, IndexingContext, IndexingPosition,