fix issue#115 Add expired_at for Domain System

src/config/tdb/migrations/global.gsql

@@ -1,6 +1,6 @@
 CREATE VERTEX Identities(PRIMARY_ID id STRING, id STRING, uuid STRING, platform STRING, identity STRING, display_name STRING, profile_url STRING, avatar_url STRING, created_at DATETIME, added_at DATETIME, updated_at DATETIME, uid STRING) WITH STATS="OUTDEGREE_BY_EDGETYPE"
 CREATE VERTEX Contracts(PRIMARY_ID id STRING, id STRING, uuid STRING, category STRING, address STRING, chain STRING, symbol STRING, updated_at DATETIME) WITH STATS="OUTDEGREE_BY_EDGETYPE"
-CREATE DIRECTED EDGE Proof_Forward(FROM Identities, TO Identities, DISCRIMINATOR(source STRING), created_at DATETIME, uuid STRING, level INT, record_id STRING, updated_at DATETIME, fetcher STRING) WITH REVERSE_EDGE="Proof_Backward"
+CREATE DIRECTED EDGE Proof_Forward(FROM Identities, TO Identities, DISCRIMINATOR(source STRING), created_at DATETIME, uuid STRING, level INT, record_id STRING, updated_at DATETIME, fetcher STRING, expired_at DATETIME) WITH REVERSE_EDGE="Proof_Backward"
 CREATE DIRECTED EDGE Hold_Identity(FROM Identities, TO Identities, DISCRIMINATOR(source STRING), uuid STRING, transaction STRING, id STRING, created_at DATETIME, updated_at DATETIME, fetcher STRING)
 CREATE DIRECTED EDGE Hold_Contract(FROM Identities, TO Contracts, DISCRIMINATOR(source STRING, transaction STRING, id STRING), uuid STRING, created_at DATETIME, updated_at DATETIME, fetcher STRING)
 CREATE DIRECTED EDGE Resolve(FROM Identities, TO Identities, DISCRIMINATOR(source STRING, system STRING, name STRING), uuid STRING, updated_at DATETIME, fetcher STRING)

src/tigergraph/edge/hold.rs

@@ -58,6 +58,9 @@ pub struct Hold {
     /// Who collects this data.
     /// It works as a "data cleansing" or "proxy" between `source`s and us.
     pub fetcher: DataFetcher,
+    #[serde(deserialize_with = "option_naive_datetime_from_string")]
+    #[serde(serialize_with = "option_naive_datetime_to_string")]
+    pub expired_at: Option<NaiveDateTime>,
 }
 
 impl Default for Hold {
@@ -70,6 +73,7 @@ impl Default for Hold {
             created_at: None,
             updated_at: naive_now(),
             fetcher: Default::default(),
+            expired_at: None,
         }
     }
 }
@@ -216,6 +220,24 @@ impl Transfer for HoldRecord {
                 op: None,
             },
         );
+
+        if let Some(expired_at) = self.expired_at {
+            attributes_map.insert(
+                "expired_at".to_string(),
+                Attribute {
+                    value: json!(expired_at),
+                    op: Some(OpCode::Max),
+                },
+            );
+        } else {
+            attributes_map.insert(
+                "expired_at".to_string(),
+                Attribute {
+                    value: json!("1970-01-01 00:00:00"), // default value
+                    op: None,
+                },
+            );
+        }
         attributes_map
     }
 
@@ -235,6 +257,11 @@ impl Transfer for HoldRecord {
         );
         map.insert("updated_at".to_string(), json!(self.updated_at));
         map.insert("fetcher".to_string(), json!(self.fetcher));
+        map.insert(
+            "expired_at".to_string(),
+            self.expired_at
+                .map_or(json!("1970-01-01 00:00:00"), |expired_at| json!(expired_at)),
+        );
         Value::Object(map)
     }
 }

src/tigergraph/tests.rs

@@ -12,36 +12,34 @@ mod tests {
             vertex::{Contract, Identity, NeighborsResponse},
         },
         upstream::{Chain, ContractCategory, DataSource, DomainNameSystem, Platform, ProofLevel},
-        util::make_http_client,
+        util::{
+            make_http_client, option_naive_datetime_from_string, option_naive_datetime_to_string,
+        },
     };
+    use chrono::{Duration, NaiveDateTime};
     use serde::{Deserialize, Serialize};
     use uuid::Uuid;
 
     #[tokio::test]
     async fn test_some() -> Result<(), Error> {
         #[derive(Debug, Clone, Serialize, Deserialize)]
         struct IdentityFilter {
-            platform: String,
-            identity: String,
-        }
-
-        #[derive(Clone, Debug, Serialize, Deserialize)]
-        struct IdentityGraphFilter {
-            by_graph_id: Option<Vec<String>>,
-            by_identity: Option<Vec<IdentityFilter>>,
-        }
-
-        let filter = IdentityGraphFilter {
-            by_graph_id: None,
-            by_identity: None,
-        };
-        if let Some(graph_ids) = filter.by_graph_id {
-            println!("some graph_ids : {:?}", graph_ids);
-        } else if let Some(identity_filters) = filter.by_identity {
-            println!("some identity_filters : {:?}", identity_filters);
-        } else {
-            println!("nothing");
+            #[serde(deserialize_with = "option_naive_datetime_from_string")]
+            #[serde(serialize_with = "option_naive_datetime_to_string")]
+            pub expired_at: Option<NaiveDateTime>,
         }
+        // 1970-01-01 00:00:00
+        let json_string = r###"
+            {
+                "expired_at": "1970-01-01 00:00:00"
+            }
+        "###;
+        let record: IdentityFilter = serde_json::from_str(json_string)?;
+        println!("{:?}", record);
+        // let filter = IdentityFilter { expired_at: None };
+        // let json_params =
+        //     serde_json::to_string(&filter).map_err(|err| Error::JSONParseError(err))?;
+        // println!("{}", json_params);
         Ok(())
     }
 

src/upstream/dotbit/mod.rs

@@ -11,7 +11,8 @@ use crate::upstream::{
     DataFetcher, DataSource, DomainNameSystem, Fetcher, Platform, Target, TargetProcessedList,
 };
 use crate::util::{
-    make_client, make_http_client, naive_now, parse_body, request_with_timeout, timestamp_to_naive,
+    make_client, make_http_client, naive_now, option_timestamp_to_naive, parse_body,
+    request_with_timeout, timestamp_to_naive,
 };
 use async_trait::async_trait;
 use hyper::{Body, Method, Request};
@@ -127,6 +128,8 @@ pub struct ReverseRecordRequest {
 pub struct AccountItem {
     pub account: String,
     pub account_alias: String,
+    pub registered_at: Option<i64>,
+    pub expired_at: Option<i64>,
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -286,6 +289,7 @@ async fn fetch_and_save_account_info(
 
     let cli = make_http_client(); // connect server
     let created_at_naive = timestamp_to_naive(account_info.create_at_unix, 0);
+    let expired_at_naive = timestamp_to_naive(account_info.expired_at_unix, 0);
 
     let addr_identity: Identity = Identity {
         uuid: Some(Uuid::new_v4()),
@@ -321,6 +325,7 @@ async fn fetch_and_save_account_info(
         created_at: created_at_naive,
         updated_at: naive_now(),
         fetcher: DataFetcher::RelationService,
+        expired_at: expired_at_naive,
     };
 
     let resolve: Resolve = Resolve {
@@ -485,12 +490,14 @@ async fn fetch_account_list_by_addrs(
     };
 
     for i in resp.result.data.unwrap().account_list.into_iter() {
+        let create_at_naive = option_timestamp_to_naive(i.registered_at, 0);
+        let expired_at_naive = option_timestamp_to_naive(i.expired_at, 0);
         let to: Identity = Identity {
             uuid: Some(Uuid::new_v4()),
             platform: Platform::Dotbit,
             identity: i.account.to_string(),
             uid: None,
-            created_at: None,
+            created_at: create_at_naive,
             display_name: Some(i.account.to_string()),
             added_at: naive_now(),
             avatar_url: None,
@@ -503,9 +510,10 @@ async fn fetch_account_list_by_addrs(
             source: DataSource::Dotbit,
             transaction: None,
             id: "".to_string(),
-            created_at: None,
+            created_at: create_at_naive,
             updated_at: naive_now(),
             fetcher: DataFetcher::RelationService,
+            expired_at: expired_at_naive,
         };
 
         let resolve: Resolve = Resolve {

src/upstream/farcaster/mod.rs

@@ -208,6 +208,7 @@ async fn save_profile_ethereum(
                     created_at: None,
                     updated_at: naive_now(),
                     fetcher: DataFetcher::DataMgrService,
+                    expired_at: None,
                 };
                 // hold record
                 create_identity_to_identity_hold_record(
@@ -268,6 +269,7 @@ async fn save_profile_signer(
         created_at: None,
         updated_at: naive_now(),
         fetcher: DataFetcher::DataMgrService,
+        expired_at: None,
     };
     // hold record
     create_identity_to_identity_hold_record(cli, &eth_identity, &farcaster_identity, &hold).await?;

src/upstream/farcaster/warpcast.rs

@@ -77,9 +77,10 @@ async fn save_verifications(
     user: &User,
     verification: Verification,
 ) -> Result<TargetProcessedList, Error> {
+    let protocol: Platform = verification.protocol.parse()?;
     let eth_identity: Identity = Identity {
         uuid: Some(Uuid::new_v4()),
-        platform: Platform::Ethereum,
+        platform: protocol,
         identity: verification.address.to_lowercase(),
         uid: None,
         created_at: None,
@@ -109,11 +110,12 @@ async fn save_verifications(
         created_at: Some(verification.timestamp),
         updated_at: naive_now(),
         fetcher: DataFetcher::RelationService,
+        expired_at: None,
     };
     create_identity_to_identity_hold_record(client, &eth_identity, &farcaster_identity, &hold)
         .await?;
     Ok(vec![Target::Identity(
-        Platform::Ethereum,
+        protocol,
         verification.address.to_lowercase().to_string(),
     )])
 }
@@ -196,6 +198,7 @@ pub struct Verification {
     #[serde(deserialize_with = "naive_datetime_from_milliseconds")]
     #[serde(serialize_with = "naive_datetime_to_milliseconds")]
     pub timestamp: NaiveDateTime,
+    pub protocol: String,
 }
 
 async fn user_by_username(username: &str) -> Result<User, Error> {

src/upstream/knn3/mod.rs

@@ -151,6 +151,7 @@ async fn fetch_ens_by_eth_wallet(identity: &str) -> Result<TargetProcessedList,
             created_at: None,
             updated_at: naive_now(),
             fetcher: DataFetcher::RelationService,
+            expired_at: None,
         };
         // hold record
         create_identity_to_contract_hold_record(&cli, &from, &to, &ownership).await?;
@@ -242,6 +243,7 @@ async fn fetch_eth_wallet_by_ens(id: &str) -> Result<TargetProcessedList, Error>
         created_at: None,
         updated_at: naive_now(),
         fetcher: DataFetcher::RelationService,
+        expired_at: None,
     };
     // hold record
     let cli = make_http_client();

src/upstream/lens/mod.rs

@@ -228,6 +228,7 @@ async fn save_profile(client: &Client<HttpConnector>, profile: &Profile) -> Resu
         created_at: None,
         updated_at: naive_now(),
         fetcher: DataFetcher::RelationService,
+        expired_at: None,
     };
     let resolve: Resolve = Resolve {
         uuid: Uuid::new_v4(),

src/upstream/rss3/mod.rs

@@ -249,6 +249,7 @@ async fn save_item(p: ResultItem) -> Result<TargetProcessedList, Error> {
         created_at: created_at_naive,
         updated_at: naive_now(),
         fetcher: DataFetcher::RelationService,
+        expired_at: None,
     };
     create_identity_to_contract_hold_record(&cli, &from, &to, &hold).await?;
 

src/upstream/space_id/mod.rs

@@ -113,6 +113,7 @@ async fn fetch_domain_by_address(
         created_at: None,
         updated_at: naive_now(),
         fetcher: DataFetcher::RelationService,
+        expired_at: None,
     };
 
     let resolve: Resolve = Resolve {
@@ -186,6 +187,7 @@ async fn fetch_address_by_domain(
         created_at: None,
         updated_at: naive_now(),
         fetcher: DataFetcher::RelationService,
+        expired_at: None,
     };
     let resolve: Resolve = Resolve {
         uuid: Uuid::new_v4(),

src/upstream/the_graph/mod.rs

@@ -13,7 +13,7 @@ use crate::upstream::{
     Chain, ContractCategory, DataFetcher, DataSource, DomainNameSystem, Fetcher, Platform, Target,
     TargetProcessedList,
 };
-use crate::util::{make_http_client, naive_now, parse_timestamp};
+use crate::util::{make_http_client, naive_now, parse_timestamp, timestamp_to_naive};
 use async_trait::async_trait;
 use gql_client::Client as GQLClient;
 use hyper::{client::HttpConnector, Client};
@@ -40,6 +40,7 @@ struct Domain {
     /// Creation timestamp (in secods)
     #[serde(rename = "createdAt")]
     created_at: String,
+    registration: Option<Registration>,
     /// ETH event logs for this ENS.
     events: Vec<DomainEvent>,
     /// Reverse resolve record set on this ENS.
@@ -49,6 +50,14 @@ struct Domain {
     owner: Account,
 }
 
+#[derive(Deserialize, Debug, Clone)]
+struct Registration {
+    #[serde(rename = "registrationDate")]
+    registration_date: i64,
+    #[serde(rename = "expiryDate")]
+    expiry_date: i64,
+}
+
 #[derive(Deserialize, Debug, Clone)]
 struct WrappedDomain {
     name: String,
@@ -73,6 +82,10 @@ const QUERY_BY_ENS: &str = r#"
             domains(where: { name: $target }) {
                 name
                 createdAt
+                registration {
+                    registrationDate
+                    expiryDate
+                }
                 events(first: 1) {
                     transactionID
                 }
@@ -88,6 +101,10 @@ const QUERY_BY_ENS: &str = r#"
               domain {
                 name
                 createdAt
+                registration {
+                    registrationDate
+                    expiryDate
+                }
                 events(first: 1) {
                     transactionID
                 }
@@ -110,6 +127,10 @@ const QUERY_BY_WALLET: &str = r#"
             domains(where: { owner: $target }) {
                 name
                 createdAt
+                registration {
+                    registrationDate
+                    expiryDate
+                }
                 events(first: 1) {
                     transactionID
                 }
@@ -125,6 +146,10 @@ const QUERY_BY_WALLET: &str = r#"
               domain {
                 name
                 createdAt
+                registration {
+                    registrationDate
+                    expiryDate
+                }
                 events(first: 1) {
                     transactionID
                 }
@@ -327,6 +352,10 @@ async fn create_or_update_own(
         .first() // TODO: really?
         .map(|event| event.transaction_id.clone());
     let ens_created_at = parse_timestamp(&domain.created_at).ok();
+    let ens_expired_at = match &domain.registration {
+        Some(registration) => timestamp_to_naive(registration.expiry_date, 0),
+        None => None,
+    };
     let owner = Identity {
         uuid: Some(Uuid::new_v4()),
         platform: Platform::Ethereum,
@@ -367,6 +396,7 @@ async fn create_or_update_own(
         created_at: ens_created_at,
         updated_at: naive_now(),
         fetcher: DataFetcher::RelationService,
+        expired_at: ens_expired_at,
     };
     // hold record
     create_identity_to_contract_hold_record(client, &owner, &contract, &ownership).await?;

src/upstream/types/platform.rs

@@ -32,6 +32,12 @@ pub enum Platform {
     #[graphql(name = "ethereum")]
     Ethereum,
 
+    /// Solana
+    #[strum(serialize = "solana")]
+    #[serde(rename = "solana")]
+    #[graphql(name = "solana")]
+    Solana,
+
     /// NextID
     #[strum(serialize = "nextid")]
     #[serde(rename = "nextid")]