docs

FlagOpen · Nov 8, 2024 · 44a1880 · 44a1880
1 parent fd0346e
commit 44a1880
Show file tree

Hide file tree

Showing 115 changed files with 9,968 additions and 0 deletions.
diff --git a/docs/Makefile b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/README.md b/docs/README.md
@@ -0,0 +1,10 @@
+install the required pkgs:
+```
+pip install -r requirements.txt
+```
+
+
+to host the webpages locally:
+```
+python -m http.server
+```
diff --git a/docs/make.bat b/docs/make.bat
@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -0,0 +1,3 @@
+sphinx
+myst-nb
+furo
diff --git a/docs/source/API/abc.rst b/docs/source/API/abc.rst
@@ -0,0 +1,6 @@
+Abstract Class
+==============
+
+.. toctree::
+    abc/inference
+    abc/finetune
diff --git a/docs/source/API/abc/finetune.rst b/docs/source/API/abc/finetune.rst
@@ -0,0 +1,6 @@
+Finetune
+========
+
+.. toctree::
+    finetune/embedder
+    finetune/reranker
diff --git a/docs/source/API/abc/finetune/embedder.rst b/docs/source/API/abc/finetune/embedder.rst
@@ -0,0 +1,9 @@
+Embedder
+========
+
+.. toctree::
+    embedder/AbsArguments
+    embedder/AbsDataset
+    embedder/AbsModeling
+    embedder/AbsTrainer
+    embedder/AbsRunner
diff --git a/docs/source/API/abc/finetune/embedder/AbsArguments.rst b/docs/source/API/abc/finetune/embedder/AbsArguments.rst
@@ -0,0 +1,6 @@
+AbsArguments
+============
+
+.. autoclass:: FlagEmbedding.abc.finetune.reranker.AbsRerankerModelArguments
+
+.. autoclass:: FlagEmbedding.abc.finetune.reranker.AbsRerankerDataArguments
diff --git a/docs/source/API/abc/finetune/embedder/AbsDataset.rst b/docs/source/API/abc/finetune/embedder/AbsDataset.rst
@@ -0,0 +1,53 @@
+==========
+AbsDataset
+==========
+
+AbsEmbedderTrainDataset
+=======================
+
+.. autoclass:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderTrainDataset
+
+Methods
+-------
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderTrainDataset._load_dataset
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderTrainDataset._shuffle_text
+
+AbsEmbedderCollator
+===================
+
+.. autoclass:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderCollator
+
+AbsEmbedderSameDatasetTrainDataset
+==================================
+
+.. autoclass:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderSameDatasetTrainDataset
+
+Methods
+-------
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderSameDatasetTrainDataset.refresh_epoch
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderSameDatasetTrainDataset._load_dataset
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderSameDatasetTrainDataset._get_file_batch_size
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderSameDatasetTrainDataset._get_train_group_size
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderSameDatasetTrainDataset._create_batch_data
+
+AbsEmbedderSameDatasetCollator
+==============================
+
+.. autoclass:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderSameDatasetCollator
+
+EmbedderTrainerCallbackForDataRefresh
+=====================================
+
+.. autoclass:: FlagEmbedding.abc.finetune.embedder.EmbedderTrainerCallbackForDataRefresh
+
+Methods
+-------
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.EmbedderTrainerCallbackForDataRefresh.on_epoch_end
diff --git a/docs/source/API/abc/finetune/embedder/AbsModeling.rst b/docs/source/API/abc/finetune/embedder/AbsModeling.rst
@@ -0,0 +1,41 @@
+===========
+AbsModeling
+===========
+
+AbsEmbedderModel
+================
+
+.. autoclass:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel
+
+Methods
+-------
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel.encode
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel.compute_loss
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel.compute_score
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel.save
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel.get_local_score
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel.compute_local_score
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel.forward
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel.distill_loss
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel._compute_no_in_batch_neg_loss
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel._compute_in_batch_neg_loss
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel._compute_cross_device_neg_loss
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModel._dist_gather_tensor
+
+
+EmbedderOutput
+==============
+
+.. autoclass:: FlagEmbedding.abc.finetune.embedder.EmbedderOutput
diff --git a/docs/source/API/abc/finetune/embedder/AbsRunner.rst b/docs/source/API/abc/finetune/embedder/AbsRunner.rst
@@ -0,0 +1,21 @@
+=========
+AbsRunner
+=========
+
+AbsEmbedderTrainer
+==================
+
+.. autoclass:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderRunner
+
+Methods
+-------
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderRunner.load_tokenizer_and_model
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderRunner.load_trainer
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderRunner.load_train_dataset
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderRunner.load_data_collator
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderRunner.run
diff --git a/docs/source/API/abc/finetune/embedder/AbsTrainer.rst b/docs/source/API/abc/finetune/embedder/AbsTrainer.rst
@@ -0,0 +1,13 @@
+==========
+AbsTrainer
+==========
+
+AbsEmbedderTrainer
+==================
+
+.. autoclass:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderTrainer
+
+Methods
+-------
+
+.. automethod:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderTrainer.compute_loss
diff --git a/docs/source/API/abc/finetune/reranker.rst b/docs/source/API/abc/finetune/reranker.rst
@@ -0,0 +1,9 @@
+Reranker
+========
+
+.. toctree::
+    reranker/AbsArguments
+    reranker/AbsDataset
+    reranker/AbsModeling
+    reranker/AbsTrainer
+    reranker/AbsRunner
diff --git a/docs/source/API/abc/finetune/reranker/AbsArguments.rst b/docs/source/API/abc/finetune/reranker/AbsArguments.rst
@@ -0,0 +1,6 @@
+AbsArguments
+============
+
+.. autoclass:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderModelArguments
+
+.. autoclass:: FlagEmbedding.abc.finetune.embedder.AbsEmbedderDataArguments
diff --git a/docs/source/API/abc/finetune/reranker/AbsDataset.rst b/docs/source/API/abc/finetune/reranker/AbsDataset.rst
@@ -0,0 +1,32 @@
+==========
+AbsDataset
+==========
+
+AbsRerankerTrainDataset
+=======================
+
+.. autoclass:: FlagEmbedding.abc.finetune.reranker.AbsRerankerTrainDataset
+
+Methods
+-------
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerTrainDataset.create_one_example
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerTrainDataset._load_dataset
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerTrainDataset._shuffle_text
+
+AbsRerankerCollator
+===================
+
+.. autoclass:: FlagEmbedding.abc.finetune.reranker.AbsRerankerCollator
+
+AbsLLMRerankerTrainDataset
+==========================
+
+.. autoclass:: FlagEmbedding.abc.finetune.reranker.AbsLLMRerankerTrainDataset
+
+AbsLLMRerankerCollator
+======================
+
+.. autoclass:: FlagEmbedding.abc.finetune.reranker.AbsLLMRerankerCollator
diff --git a/docs/source/API/abc/finetune/reranker/AbsModeling.rst b/docs/source/API/abc/finetune/reranker/AbsModeling.rst
@@ -0,0 +1,31 @@
+===========
+AbsModeling
+===========
+
+AbsRerankerModel
+================
+
+.. autoclass:: FlagEmbedding.abc.finetune.reranker.AbsRerankerModel
+
+Methods
+-------
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerModel.encode
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerModel.gradient_checkpointing_enable
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerModel.enable_input_require_grads
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerModel.forward
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerModel.compute_loss
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerModel.save
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerModel.save_pretrained
+
+
+RerankerOutput
+==============
+
+.. autoclass:: FlagEmbedding.abc.finetune.reranker.RerankerOutput
diff --git a/docs/source/API/abc/finetune/reranker/AbsRunner.rst b/docs/source/API/abc/finetune/reranker/AbsRunner.rst
@@ -0,0 +1,21 @@
+=========
+AbsRunner
+=========
+
+AbsRerankerTrainer
+==================
+
+.. autoclass:: FlagEmbedding.abc.finetune.reranker.AbsRerankerRunner
+
+Methods
+-------
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerRunner.load_tokenizer_and_model
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerRunner.load_trainer
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerRunner.load_train_dataset
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerRunner.load_data_collator
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerRunner.run
diff --git a/docs/source/API/abc/finetune/reranker/AbsTrainer.rst b/docs/source/API/abc/finetune/reranker/AbsTrainer.rst
@@ -0,0 +1,13 @@
+==========
+AbsTrainer
+==========
+
+AbsRerankerTrainer
+==================
+
+.. autoclass:: FlagEmbedding.abc.finetune.reranker.AbsRerankerTrainer
+
+Methods
+-------
+
+.. automethod:: FlagEmbedding.abc.finetune.reranker.AbsRerankerTrainer.compute_loss
diff --git a/docs/source/API/abc/inference.rst b/docs/source/API/abc/inference.rst
@@ -0,0 +1,6 @@
+Inference
+=========
+
+.. toctree::
+    inference/AbsEmbedder
+    inference/AbsReranker
-Original file line number
+Diff line change
@@ -0,0 +1,3 @@
+    sphinx
+    myst-nb
+    furo