From 8008c1d9c3efa47f80f3bb8944fca5a7e1509c81 Mon Sep 17 00:00:00 2001 From: Ox Cart Date: Tue, 10 Oct 2023 21:29:54 -0500 Subject: [PATCH] Stop using JSON across Go/Swift boundary and use pathdb's built-in support for protobuf --- go.mod | 2 +- go.sum | 4 +- internalsdk/model.go | 125 ++++++------------ internalsdk/session_model.go | 12 +- .../Lantern/Core/Db/SubscriberRequest.swift | 109 +-------------- 5 files changed, 49 insertions(+), 203 deletions(-) diff --git a/go.mod b/go.mod index 9180c2c54..d6fab97bc 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/getlantern/idletiming v0.0.0-20201229174729-33d04d220c4e github.com/getlantern/ipproxy v0.0.0-20230511223023-ee52513fd782 github.com/getlantern/mtime v0.0.0-20200417132445-23682092d1f7 - github.com/getlantern/pathdb v0.0.0-20231005134844-d5ef1ce95949 + github.com/getlantern/pathdb v0.0.0-20231011022612-e0fb1003dce8 github.com/getlantern/replica v0.14.1 github.com/gorilla/mux v1.8.0 github.com/stretchr/testify v1.8.3 diff --git a/go.sum b/go.sum index af19fd8f7..12e1939e7 100644 --- a/go.sum +++ b/go.sum @@ -484,8 +484,8 @@ github.com/getlantern/osversion v0.0.0-20230401075644-c2a30e73c451 h1:3Nn0AqIlIm github.com/getlantern/osversion v0.0.0-20230401075644-c2a30e73c451/go.mod h1:kaUdXyKE1Y8bwPnlN7ChFXWnkADpL0zZrk8F0XbpKcc= github.com/getlantern/packetforward v0.0.0-20201001150407-c68a447b0360 h1:pijUoofaQcAM/8zbDzZM2LQ90kGVbKfnSAkFnQwLZZU= github.com/getlantern/packetforward v0.0.0-20201001150407-c68a447b0360/go.mod h1:nsJPNYUSY96xB+p7uiDW8O4uiKea+KjeUdS5d6tf9IU= -github.com/getlantern/pathdb v0.0.0-20231005134844-d5ef1ce95949 h1:eeBSC1sYMtEcSq7gYnNtKfvAyFxQ8aWk26Sd6ZdC7qE= -github.com/getlantern/pathdb v0.0.0-20231005134844-d5ef1ce95949/go.mod h1:SFQy+f58IbLpnbq2nVqlq7ccwaUiO7ablKv631WVIuc= +github.com/getlantern/pathdb v0.0.0-20231011022612-e0fb1003dce8 h1:dw8BlhZotSe57aoDrtlluDBmO9fCptEzu4YZ8ItHq1o= +github.com/getlantern/pathdb v0.0.0-20231011022612-e0fb1003dce8/go.mod h1:SFQy+f58IbLpnbq2nVqlq7ccwaUiO7ablKv631WVIuc= github.com/getlantern/preconn v0.0.0-20180328114929-0b5766010efe/go.mod h1:FvIxQD61iYA42UjaJyzGl9DNne8jbowbgicdeNk/7kE= github.com/getlantern/preconn v1.0.0 h1:DsY3l/y/BJUj86WyaxXylbJnCC9QbKcc3D6js6rFL60= github.com/getlantern/preconn v1.0.0/go.mod h1:i/AnXvx715Fq7HgZLlmQlw3sGfEkku8BQT5hLHMK4+k= diff --git a/internalsdk/model.go b/internalsdk/model.go index 166fa35ac..6174e4c73 100644 --- a/internalsdk/model.go +++ b/internalsdk/model.go @@ -1,8 +1,6 @@ package internalsdk import ( - "encoding/base64" - "encoding/json" "fmt" "strings" @@ -38,54 +36,49 @@ type SubscriptionRequest struct { Updater UpdaterModel } -// ChangeSetInterface represents changes in a database. -type ChangeSetInterface struct { - UpdatesSerialized string - DeletesSerialized string +// ChangeSet represents changes in a database. +type ChangeSet struct { + cs *pathdb.ChangeSet[any] } -// ItemInterface represents an item in the database with its associated values. -type ItemInterface struct { - Path string - DetailPath string - Value *MyValue +type Update struct { + Path string + Value *minisql.Value +} + +func (cs *ChangeSet) HasUpdate() bool { + return len(cs.cs.Updates) > 0 +} + +func (cs *ChangeSet) PopUpdate() (*Update, error) { + for path, v := range cs.cs.Updates { + delete(cs.cs.Updates, path) + vb, err := v.Value.ValueOrProtoBytes() + if err != nil { + return nil, err + } + return &Update{Path: path, Value: minisql.NewValue(vb)}, nil + } + + return nil, nil +} + +func (cs *ChangeSet) HasDelete() bool { + return len(cs.cs.Deletes) > 0 +} + +func (cs *ChangeSet) PopDelete() string { + for path := range cs.cs.Deletes { + delete(cs.cs.Deletes, path) + return path + } + + return "" } // UpdaterModel defines an interface to handle database changes. type UpdaterModel interface { - OnChanges(cs *ChangeSetInterface) error -} -type MyValue struct { - minisql.Value -} - -// Custom JSON serialization method for Value type -func (v *MyValue) MarshalJSON() ([]byte, error) { - - switch v.Type { - case minisql.ValueTypeBytes: - return json.Marshal(map[string]interface{}{ - "Type": v.Type, - "Value": base64.StdEncoding.EncodeToString(v.Bytes()), - }) - case minisql.ValueTypeString: - return json.Marshal(map[string]interface{}{ - "Type": v.Type, - "Value": v.String(), - }) - case minisql.ValueTypeInt: - return json.Marshal(map[string]interface{}{ - "Type": v.Type, - "Value": v.Int(), - }) - case minisql.ValueTypeBool: - return json.Marshal(map[string]interface{}{ - "Type": v.Type, - "Value": v.Bool(), - }) - default: - return nil, fmt.Errorf("unsupported value type: %d", v.Type) - } + OnChanges(cs *ChangeSet) error } // NewModel initializes a new baseModel instance. @@ -118,49 +111,7 @@ func (m *baseModel) Subscribe(req *SubscriptionRequest) error { JoinDetails: req.JoinDetails, ReceiveInitial: req.ReceiveInitial, OnUpdate: func(cs *pathdb.ChangeSet[interface{}]) error { - updatesMap := make(map[string]*ItemInterface) - for k, itemWithRaw := range cs.Updates { - rawValue, err := itemWithRaw.Value.Value() - if err != nil { - log.Debugf("Error extracting raw value:", err) - return err - } - log.Debugf("Got update for value %v - value: %v rawValue: %v", k, rawValue, string(itemWithRaw.Value.Bytes)) - // When serlizeing might get other other type - //make sure to convered to only supported types - convertedValue := convertValueToSupportedTypes(rawValue) - val := minisql.NewValue(convertedValue) - // Need wrap to coz we need to send to json - myVal := &MyValue{Value: *val} - updatesMap[k] = &ItemInterface{ - Path: itemWithRaw.Path, - DetailPath: itemWithRaw.DetailPath, - Value: myVal, - } - } - - // Serialize updates and deletes to JSON - updatesSerialized, err := json.Marshal(updatesMap) - if err != nil { - log.Debugf("Error serializing updates:", err) - return err - } - - deletesSerialized, err := json.Marshal(cs.Deletes) - if err != nil { - log.Debugf("Error serializing deletes:", err) - - return err - } - // log.Debugf("Serialized updates:", string(updatesSerialized)) - log.Debugf("Serialized deletes:", string(deletesSerialized)) - - csInterface := &ChangeSetInterface{ - UpdatesSerialized: string(updatesSerialized), - DeletesSerialized: string(deletesSerialized), - } - - return req.Updater.OnChanges(csInterface) + return req.Updater.OnChanges(&ChangeSet{cs: cs}) }, } diff --git a/internalsdk/session_model.go b/internalsdk/session_model.go index c364b6d49..7e78b25a0 100644 --- a/internalsdk/session_model.go +++ b/internalsdk/session_model.go @@ -8,8 +8,6 @@ import ( "path/filepath" "strconv" - "google.golang.org/protobuf/proto" - "github.com/getlantern/flashlight/v7/common" "github.com/getlantern/flashlight/v7/logging" "github.com/getlantern/pathdb" @@ -81,6 +79,7 @@ func NewSessionModel(mdb minisql.DB, opts *SessionModelOpts) (*SessionModel, err if err != nil { return nil, err } + base.db.RegisterType(1000, &ServerInfo{}) m := &SessionModel{baseModel: base} m.initSessionModel(opts) return m, nil @@ -370,19 +369,12 @@ func (m *SessionModel) UpdateStats(p0 string, p1 string, p2 string, p3 int, p4 i CountryCode: p2, } - // Serialize the ServerInfo object to byte slice - serverInfoBytes, serverErr := proto.Marshal(serverInfo) - if serverErr != nil { - return serverErr - } - - log.Debugf("UpdateStats called with city %v and country %v and code %v with proxy %v server info bytes %v", p0, p1, p2, p5, serverInfoBytes) err := pathdb.Mutate(m.db, func(tx pathdb.TX) error { pathdb.Put[string](tx, SERVER_COUNTRY, p1, "") pathdb.Put[string](tx, SERVER_CITY, p0, "") pathdb.Put[string](tx, SERVER_COUNTRY_CODE, p2, "") pathdb.Put[bool](tx, HAS_SUCCEEDING_PROXY, p5, "") - pathdb.Put[[]byte](tx, PATH_SERVER_INFO, serverInfoBytes, "") + pathdb.Put[*ServerInfo](tx, PATH_SERVER_INFO, serverInfo, "") // Not using ads blocked any more return nil diff --git a/ios/Runner/Lantern/Core/Db/SubscriberRequest.swift b/ios/Runner/Lantern/Core/Db/SubscriberRequest.swift index 7adc63247..af01d325a 100644 --- a/ios/Runner/Lantern/Core/Db/SubscriberRequest.swift +++ b/ios/Runner/Lantern/Core/Db/SubscriberRequest.swift @@ -31,123 +31,26 @@ class DetailsSubscriber: InternalsdkSubscriptionRequest { class DetailsSubscriberUpdater: NSObject, InternalsdkUpdaterModelProtocol { var onChangesCallback: (([String: Any], [String]) -> Void)? - typealias DynamicKeysData = [String: ItemDetail] - func onChanges(_ cs: InternalsdkChangeSetInterface?) throws { + func onChanges(_ cs: InternalsdkChangeSet?) throws { guard let cs = cs else { throw NSError( domain: "onChangesError", code: 1, userInfo: [NSLocalizedDescriptionKey: "ChangeSet is nil"] ) } - let decoder = JSONDecoder() var updatesDictionary: [String: Any] = [:] var deletesList: [String] = [] // Deserialize updates - if let updatesData = cs.updatesSerialized.data(using: .utf8), !updatesData.isEmpty { - do { - let updates = try decoder.decode(DynamicKeysData.self, from: updatesData) - - updatesDictionary = Dictionary( - uniqueKeysWithValues: updates.map { (path, detail) -> (String, Any) in - return (path, detail.value.value) - }) - - } catch let jsonError { - logger.log("Error deserializing updates: \(jsonError.localizedDescription)") - throw jsonError - } - } else { - logger.log("No updatesSerialized data to parse or it's empty.") + while cs.hasUpdate() { + let update = try cs.popUpdate() + updatesDictionary[update.path] = ValueUtil.convertFromMinisqlValue(from:update.value!) } - // Deserialize deletes - if cs.deletesSerialized.lowercased() != "null" { - if let deletesData = cs.deletesSerialized.data(using: .utf8), !deletesData.isEmpty { - do { - - deletesList = try decoder.decode([String].self, from: deletesData) - } catch let jsonError { - logger.log("Error deserializing deletes: \(jsonError.localizedDescription)") - throw jsonError - } - } else { - logger.log("No deletesSerialized data to parse or it's empty.") - } + while cs.hasDelete() { + deletesList.append(cs.popDelete()) } onChangesCallback?(updatesDictionary, deletesList) - - } -} - -// Json Decode - -struct ItemDetail: Codable { - let path: String - let detailPath: String - let value: ValueStruct - - enum CodingKeys: String, CodingKey { - case path = "Path" - case detailPath = "DetailPath" - case value = "Value" - } -} - -struct ValueStruct: Codable { - let type: Int - var value: Any? - - init(from decoder: Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - type = try container.decode(Int.self, forKey: .type) - - switch type { - case ValueUtil.TYPE_BYTES: // Assuming this corresponds to bytes - // Handle bytes - let stringValue = try container.decode(String.self, forKey: .value) - value = Data(base64Encoded: stringValue) - case ValueUtil.TYPE_STRING: - value = try container.decode(String.self, forKey: .value) - case ValueUtil.TYPE_INT: - value = try container.decode(Int.self, forKey: .value) - case ValueUtil.TYPE_BOOL: - value = try container.decode(Bool.self, forKey: .value) - default: - throw DecodingError.dataCorruptedError( - forKey: .value, - in: container, - debugDescription: "Invalid type" - ) - } - } - - func encode(to encoder: Encoder) throws { - var container = encoder.container(keyedBy: CodingKeys.self) - try container.encode(type, forKey: .type) - - switch type { - case ValueUtil.TYPE_BYTES: - // Handle bytes - if let dataValue = value as? Data { - try container.encode(dataValue.base64EncodedString(), forKey: .value) - } - case ValueUtil.TYPE_STRING: - try container.encode(value as? String, forKey: .value) - case ValueUtil.TYPE_INT: - try container.encode(value as? Int, forKey: .value) - case ValueUtil.TYPE_BOOL: - try container.encode(value as? Bool, forKey: .value) - default: - throw EncodingError.invalidValue( - value as Any, - EncodingError.Context(codingPath: [CodingKeys.value], debugDescription: "Invalid type")) - } - } - - enum CodingKeys: String, CodingKey { - case type = "Type" - case value = "Value" } }