From 78f3a7c4428a2139f8d9ad8c12845965307a69a4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 6 Aug 2024 01:05:41 +0000 Subject: [PATCH] deploy: a66fd4984a1dc566c13465cea24a8ceec9cc5aa4 --- 404.html | 75 +- api/public/API/index.html | 75 +- api/public/cmf/index.html | 83 +- api/public/cmfquery/index.html | 81 +- api/public/dataslice/index.html | 81 +- architecture/advantages/index.html | 75 +- architecture/components/index.html | 75 +- architecture/overview/index.html | 75 +- assets/Tensorboard.png | Bin 0 -> 120432 bytes .../Getting Started with cmf/index.html | 75 +- cmf_client/cmf_client/index.html | 159 ++- cmf_client/minio-server/index.html | 75 +- cmf_client/ssh-setup/index.html | 75 +- cmf_client/step-by-step/index.html | 75 +- cmf_client/tensorflow_guide/index.html | 1074 +++++++++++++++++ cmf_server/cmf-server/index.html | 378 +++--- common-metadata-ontology/readme/index.html | 75 +- examples/getting_started/index.html | 75 +- index.html | 75 +- search/search_index.json | 2 +- sitemap.xml.gz | Bin 127 -> 127 bytes 21 files changed, 2470 insertions(+), 288 deletions(-) create mode 100644 assets/Tensorboard.png create mode 100644 cmf_client/tensorflow_guide/index.html diff --git a/404.html b/404.html index acbbb18f..ce23467d 100644 --- a/404.html +++ b/404.html @@ -16,7 +16,7 @@ - + @@ -604,7 +604,7 @@ - Public API + TensorBoard @@ -613,6 +613,69 @@ + + + + + + + + + + + + + + + +
  • + + + + + + + + +
    • + + + + + + + + + + + + + +
  • + + + + + + + + +
    • + + + + + + + + + + + + + +
  • + + + + + + + + +
    • + + + + + + + + + + + + + +
  • + + + + + + + + +
    • + + + + + + + + + + + + + +
  • + + + + + + + + +
    • + + + + + + + + + + + + + +
  • + + + + + + + + +
    • + +
  • + + + cmf metadata export + + +
    • @@ -790,7 +799,7 @@ - Public API + TensorBoard @@ -799,6 +808,69 @@ + + + + + + + + + + + + + + + +
  • + + + + + + + + +
    • + +
  • + + + cmf metadata export + + +
    • @@ -1115,9 +1196,9 @@

    Getting started with cmf-client commands

    cmf init

    -

    Usage: cmf init [-h] {minioS3,amazonS3,local,sshremote,show}
+
    Usage: cmf init [-h] {minioS3,amazonS3,local,sshremote,osdfremote,show}
 
    
-cmf init initializes an artifact repository for cmf. Local directory, Minio S3 bucket, Amazon S3 bucket and SSH Remote directory are the options available. Additionally, user can provide cmf-server url.
    
+cmf init initializes an artifact repository for cmf. Local directory, Minio S3 bucket, Amazon S3 bucket, SSH Remote and Remote OSDF directory are the options available. Additionally, user can provide cmf-server url.
    
 
    cmf init show
    
 
    Usage: cmf init show
 
    
@@ -1151,7 +1232,7 @@ 
    cmf init minioS3cmf init local
    
 
    Usage: cmf init local [-h] --path [path] -
@@ -1176,7 +1257,7 @@ 
    cmf init localcmf init amazonS3
    
 
    Before setting up, obtain AWS temporary security credentials using the AWS Security Token Service (STS). These credentials are short-term and can last from minutes to hours. They are dynamically generated and provided to trusted users upon request, and expire after use. Users with appropriate permissions can request new credentials before or upon expiration. For further information, refer to the  Temporary security credentials in IAM page.
    
@@ -1225,19 +1306,20 @@ 
    cmf init amazonS3
      --url [url]                           Specify bucket url.
   --access-key-id [access_key_id]       Specify Access Key Id.
   --secret-key [secret_key]             Specify Secret Key.
+  --session-token                       Specify session token. (default: )
   --git-remote-url [git_remote_url]     Specify git repo url.
 
    
 Optional Arguments
 
      -h, --help                          show this help message and exit
-  --session-token                     Specify session token. (default: )
   --cmf-server-url [cmf_server_url]   Specify cmf-server url. (default: http://127.0.0.1:80)
   --neo4j-user [neo4j_user]           Specify neo4j user. (default: None)
   --neo4j-password [neo4j_password]   Specify neo4j password. (default: None)
-  --neo4j-uri <neo4j_uri>             Specify neo4j uri. Eg bolt://localhost:7687 (default: None)
+  --neo4j-uri [neo4j_uri]             Specify neo4j uri. Eg bolt://localhost:7687 (default: None)
 
    
 
    cmf init sshremote
    
 
    Usage: cmf init sshremote [-h] --path [path] 
-                               --user [user] --port [port]
+                               --user [user]
+                               --port [port]
                                --password [password]  
                                --git-remote-url [git_remote_url] 
                                --cmf-server-url [cmf_server_url]
@@ -1256,17 +1338,17 @@ 
    cmf init sshremote
      -h, --help  show this help message and exit
+
      -h, --help                          show this help message and exit
   --cmf-server-url [cmf_server_url]   Specify cmf-server url. (default: http://127.0.0.1:80)
   --neo4j-user [neo4j_user]           Specify neo4j user. (default: None)
   --neo4j-password [neo4j_password]   Specify neo4j password. (default: None)
-  --neo4j-uri <neo4j_uri>             Specify neo4j uri. Eg bolt://localhost:7687 (default: None)
+  --neo4j-uri [neo4j_uri]             Specify neo4j uri. Eg bolt://localhost:7687 (default: None)
 
    
 
    cmf init osdfremote
    
 
    Usage: cmf init osdfremote [-h] --path [path] 
-                             --endpoint-url [endpoint_url]
-                             --access-key-id [access_key_id] 
-                             --secret-key [secret_key] 
+                             --key-id [key_id]
+                             --key-path [key_path] 
+                             --key-issuer [key_issuer] 
                              --git-remote-url[git_remote_url]  
                              --cmf-server-url [cmf_server_url]
                              --neo4j-user [neo4j_user]
@@ -1288,31 +1370,31 @@ 
    cmf init osdfremotecmf artifact
    
 
    Usage: cmf artifact [-h] {pull,push}
 
    
 cmf artifact pull or push artifacts from or to the user configured artifact repository, respectively.
    
 
    cmf artifact pull
    
-
    Usage: cmf artifact pull [-h] -p [pipeline_name] -f [file_name] [-a <artifact_name>]
+
    Usage: cmf artifact pull [-h] -p [pipeline_name] -f [file_name] -a [artifact_name]
 
    
 cmf artifact pull command pull artifacts from the user configured repository to the user's local machine.
-
    cmf artifact pull -p 'pipeline-name' 
+
    cmf artifact pull -p 'pipeline-name' -f '/path/to/mlmd-file-name' -a 'artifact-name'
 
    
 Required Arguments
 
      -p [pipeline_name], --pipeline-name [pipeline_name]   Specify Pipeline name.
 
    
 Optional Arguments
 
      -h, --help                                            show this help message and exit
-  -a <artifact_name>, --artifact_name <artifact_name>   Specify artifact name only; don't use folder name or absolute path.
+  -a [artifact_name], --artifact_name [artifact_name]   Specify artifact name only; don't use folder name or absolute path.
   -f [file_name],--file-name [file_name]                Specify mlmd file name.
 
    
 
    cmf artifact push
    
 
    Usage: cmf artifact push [-h] -p [pipeline_name] -f [file_name]
 
    
 cmf artifact push command push artifacts from the user's local machine to the user configured artifact repository.
-
    cmf artifact push -p 'pipeline_name'
+
    cmf artifact push -p 'pipeline_name' -f '/path/to/mlmd-file-name'
 
    
 Required Arguments
 
      -p [pipeline_name], --pipeline-name [pipeline_name]   Specify Pipeline name.
@@ -1322,14 +1404,14 @@ 
    cmf artifact pushcmf metadata
    
-
    Usage: cmf metadata [-h] {pull,push}
+
    Usage: cmf metadata [-h] {pull,push,export}
 
    
-cmf metadata push or pull the metadata file to and from the cmf-server, respectively.
    
+cmf metadata push, pull or export the metadata file to and from the cmf-server, respectively.
    
 
    cmf metadata pull
    
 
    Usage: cmf metadata pull [-h] -p [pipeline_name] -f [file_name]  -e [exec_id]
 
    
 cmf metadata pull command pulls the metadata file from the cmf-server to the user's local machine.
-
    cmf metadata pull -p 'pipeline-name' -f "/path/to/mlmd-file-name"
+
    cmf metadata pull -p 'pipeline-name' -f '/path/to/mlmd-file-name' -e 'execution_id'
 
    
 Required Arguments
 
      -p [pipeline_name], --pipeline_name [pipeline_name]     Specify Pipeline name.
@@ -1340,18 +1422,33 @@ 
    cmf metadata pullcmf metadata push
    
-
    Usage: cmf metadata push [-h] -p [pipeline_name] -f [file_name]  -e [exec_id]
+
    Usage: cmf metadata push [-h] -p [pipeline_name] -f [file_name] -e [exec_id] -t [tensorboard]
 
    
 cmf metadata push command pushes the metadata file from the local machine to the cmf-server.
-
    cmf metadata push -p 'pipeline-name' -f "/path/to/mlmd-file-name"
+
    cmf metadata push -p 'pipeline-name' -f '/path/to/mlmd-file-name' -e 'execution_id' -t '/path/to/tensorboard-log'
+
    
+Required Arguments
+
    -p [pipeline_name], --pipeline_name [pipeline_name]     Specify Pipeline name.
+
    
+
    Optional Arguments
+
      -h, --help                                         show this help message and exit
+  -f [file_name],   --file_name [file_name]          Specify mlmd file name.
+  -e [exec_id],     --execution [exec_id]            Specify execution id.
+  -t [tensorboard], --tensorboard [tensorboard]      Specify path to tensorboard logs for the pipeline.
+
    
+
    cmf metadata export
    
+
    Usage: cmf metadata export [-h] -p [pipeline_name] -j [json_file_name] -f [file_name]
+
    
+cmf metadata export export local mlmd's metadata in json format to a json file.
+
    cmf metadata export -p 'pipeline-name' -j '/path/to/json-file-name' -f '/path/to/mlmd-file-name'
 
    
 Required Arguments
 
    -p [pipeline_name], --pipeline_name [pipeline_name]     Specify Pipeline name.
 
    
 
    Optional Arguments
-
      -h, --help                                    show this help message and exit
-  -f [file_name], --file_name [file_name]       Specify mlmd file name.
-  -e [exec_id], --execution [exec_id]           Specify execution id.
+
      -h, --help                                               show this help message and exit
+  -f [file_name],      --file_name [file_name]             Specify mlmd file name.
+  -j [json_file_name], --json_file_name [json_file_name]   Specify json file name with full path.
 
    
 
 
diff --git a/cmf_client/minio-server/index.html b/cmf_client/minio-server/index.html
index 60a3fba4..7363bf2e 100644
--- a/cmf_client/minio-server/index.html
+++ b/cmf_client/minio-server/index.html
@@ -20,7 +20,7 @@
       
       
       
-      
+      
     
     
       
@@ -664,7 +664,7 @@
             
   
   
-    Public API
+    TensorBoard
   
   
 
@@ -673,6 +673,69 @@
         
         
+      
+    
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    
  • 
+      
+        
+        
+        
+        
+          
+          
+        
+        
    • 
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    
  • 
+      
+        
+        
+        
+        
+          
+          
+        
+        
    • 
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    
  • 
+      
+        
+        
+        
+        
+          
+          
+        
+        
    • 
       
@@ -730,7 +730,7 @@
             
   
   
-    Public API
+    TensorBoard
   
   
 
@@ -739,6 +739,69 @@
         
         
+      
+    
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    
  • 
+      
+        
+        
+        
+        
+          
+          
+        
+        
    • 
       
@@ -995,82 +1058,12 @@
 
 
    Getting started with cmf-server
    
 
    cmf-server is a key interface for the user to explore and track their ML training runs. It allows users to store the metadata file on the cmf-server. The user can retrieve the saved metadata file and can view the content of the saved metadata file using the UI provided by the cmf-server.
    
-
    API Reference
    
-
    cmf-server APIs are organized around FastAPI.
-They accept and return JSON-encoded request bodies and responses and return standard HTTP response codes.
    
-
    List of APIs
    
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
    MethodURLDescription
    Post/mlmd_pushUsed to push Json Encoded data to cmf-server
    Get/mlmd_pull/{pipeline_name}Retrieves a mlmd file from cmf-server
    Get/display_executionsRetrieves all executions from cmf-server
    Get/display_artifacts/{pipeline_name}/{data_type}Retrieves all artifacts from cmf-server for resp datat type
    Get/display_lineage/{lineage_type}/{pipeline_name}Creates lineage data from cmf-server
    Get/display_pipelinesRetrieves all pipelines present in mlmd file
    
-
    HTTP Response Status codes
    
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
    CodeTitleDescription
    200OKmlmd is successfully pushed (e.g. when using GET, POST).
    400Bad requestWhen the cmf-server is not available.
    500Internal server errorWhen an internal error has happened
    
 
    Setup a cmf-server
    
-
    There are two ways to start cmf server - 
-- Using docker compose file
-- Using docker run
    
+
    There are two ways to start cmf server - 
    
+
      
+
    • Using docker compose file
      • 
+
    • Using docker run
      • 
+
    
 
    Pre-requisites
    
 
      
 
    1. 
@@ -1134,13 +1127,14 @@ 
      Using docker compose file
 
      It is neccessary to rebuild images for cmf-server and ui-server after cmf version update or after pulling latest cmf code from git.
      
 
+
      OR
      
 
      Using docker run command
      
 
        
 
      1. 
 
        Install cmflib on your system.
        
 
        2. 
 
      2. 
-
        Go to server directory. 
+
        Go to cmf/server directory. 
    
        cd server
 
        
 
        3. 
@@ -1154,38 +1148,38 @@ 
        Using docker run command
        Usage:  docker build -t [image_name] -f ./Dockerfile ../
 
    
    Example:
-   
    docker build -t myimage -f ./Dockerfile ../
+   
    docker build -t server_image -f ./Dockerfile ../
 
    
    Note - '../'  represents the     Build context for the docker image.
    
 
 
  • 
-
    Launch a new docker container using the image with directory /home//cmf-server/data/static mounted.
-   pre-requisite - mkdir /home/<user>/cmf-server/data/static
-   
    Usage: docker run --name [container_name] -p 0.0.0.0:8080:80 -v /home/<user>/cmf-server/data/static:/cmf-server/data/static [image_name]
+
    Launch a new docker container using the image with directory /home/user/cmf-server/data mounted.
+   Pre-requisite: mkdir /home/<user>/cmf-server/data/static
+   
    Usage: docker run --name [container_name] -p 0.0.0.0:8080:80 -v /home/<user>/cmf-server/data:/cmf-server/data -e MYIP=XX.XX.XX.XX [image_name]
 
    
    Example:
-   
    docker run --name mycontainer -p 0.0.0.0:8080:80 -v /home/user/cmf-server/data/static:/cmf-server/data/static myimage
+   
    docker run --name cmf-server -p 0.0.0.0:8080:80 -v /home/user/cmf-server/data:/cmf-server/data -e MYIP=0.0.0.0 server_image
 
    
 
    • 
 
  • 
 
    After cmf-server container is up, start ui-server, Go to cmf/ui folder.
-   
    cd /cmf/ui
+   
    cd cmf/ui
 
    
 
    • 
 
  • 
 
    Execute the below-mentioned command to create a ui-server docker image.
-   
    Usage:  docker build -t [image_name] -f ./Dockerfile ../
+   
    Usage:  docker build -t [image_name] -f ./Dockerfile ./
 
    
    Example:
-   
    docker build -t uiimage -f ./Dockerfile ../
+   
    docker build -t ui_image -f ./Dockerfile ./
 
    
 
    • 
 
  • 
-
    Launch a new docker container using the image with directory
-   
    Usage: docker run --name [container_name] -p 0.0.0.0:3000:3000 [image_name]
+
    Launch a new docker container for UI.
+   
    Usage: docker run --name [container_name] -p 0.0.0.0:3000:3000 -e REACT_APP_MY_IP=XX.XX.XX.XX [image_name]
 
    
    Example:
-   
    docker run --name mycontainer -p 0.0.0.0:3000:3000 uiimage
+   
    docker run --name ui-server -p 0.0.0.0:3000:3000 -e REACT_APP_MY_IP=0.0.0.0 ui_image
 
    
       Note:
       If you face issue regarding Libzbar-dev similar to the snapshot, add proxies to '/.docker/config.json'
    
@@ -1217,6 +1211,78 @@ 
    Using docker run command
    
 
    • 
 
+
    API Reference
    
+
    cmf-server APIs are organized around FastAPI.
+They accept and return JSON-encoded request bodies and responses and return standard HTTP response codes.
    
+
    List of APIs
    
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
    MethodURLDescription
    Post/mlmd_pushUsed to push Json Encoded data to cmf-server
    Get/mlmd_pull/{pipeline_name}Retrieves a mlmd file from cmf-server
    Get/display_executionsRetrieves all executions from cmf-server
    Get/display_artifacts/{pipeline_name}/{data_type}Retrieves all artifacts from cmf-server for resp datat type
    Get/display_lineage/{lineage_type}/{pipeline_name}Creates lineage data from cmf-server
    Get/display_pipelinesRetrieves all pipelines present in mlmd file
    
+
    HTTP Response Status codes
    
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
    CodeTitleDescription
    200OKmlmd is successfully pushed (e.g. when using GET, POST).
    400Bad requestWhen the cmf-server is not available.
    500Internal server errorWhen an internal error has happened
    
 
 
 
diff --git a/common-metadata-ontology/readme/index.html b/common-metadata-ontology/readme/index.html
index 8a086624..8a7801bb 100644
--- a/common-metadata-ontology/readme/index.html
+++ b/common-metadata-ontology/readme/index.html
@@ -18,7 +18,7 @@
       
       
       
-      
+      
     
     
       
@@ -611,7 +611,7 @@
             
   
   
-    Public API
+    TensorBoard
   
   
 
@@ -620,6 +620,69 @@
         
         
+      
+    
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    
  • 
+      
+        
+        
+        
+        
+          
+          
+        
+        
    • 
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    
  • 
+      
+        
+        
+        
+        
+          
+          
+        
+        
    • 
+  
+
+    
+      
+      
+  
+  
+  
+  
+    
+    
+    
+    
+    
  • 
+      
+        
+        
+        
+        
+          
+          
+        
+