From b601548597ccee023b4e1943b5d25a6f87379668 Mon Sep 17 00:00:00 2001 From: oneiric Date: Sat, 17 Mar 2018 02:47:16 +0000 Subject: [PATCH] AddressBook: store only unique addresses New helper functions validate human readable hostnames, and ensure only unique entries are loaded from subscriptions. Refactor subscription validation + saving with new helpers. Refs #835 --- src/client/address_book/impl.cc | 147 +++++++++++++++++++++++--------- src/client/address_book/impl.h | 38 +++++++-- 2 files changed, 136 insertions(+), 49 deletions(-) diff --git a/src/client/address_book/impl.cc b/src/client/address_book/impl.cc index 39d8e36d..5889ce38 100644 --- a/src/client/address_book/impl.cc +++ b/src/client/address_book/impl.cc @@ -1,5 +1,5 @@ /** // - * Copyright (c) 2013-2017, The Kovri I2P Router Project // + * Copyright (c) 2013-2018, The Kovri I2P Router Project // * // * All rights reserved. // * // @@ -228,6 +228,7 @@ void AddressBook::DownloadSubscription() { << "AddressBook: picking random subscription from total publisher count: " << publisher_count; // Pick a random publisher to subscribe from + // TODO(oneiric): download all subscriptions not already stored auto publisher = kovri::core::RandInRange32(0, publisher_count - 1); m_SubscriberIsDownloading = true; try { @@ -295,6 +296,7 @@ bool AddressBook::SaveSubscription( LOG(debug) << "AddressBook: processing " << addresses.size() << " addresses"; // Stream may be a file or downloaded stream. // Regardless, we want to write/overwrite the subscription file. + // TODO(oneiric): save unique entries to userhosts.txt if (file_name.empty()) // Use default filename if none given. file_name = (core::GetPath(core::Path::AddressBook) / GetDefaultSubscriptionFilename()).string(); LOG(debug) << "AddressBook: opening subscription file " << file_name; @@ -307,11 +309,17 @@ bool AddressBook::SaveSubscription( for (auto const& address : addresses) { const std::string& host = address.first; const auto& ident = address.second; - // Write/overwrite Hostname=Base64Address pairing to subscription file - file << host << "=" << ident.ToBase64() << '\n'; // TODO(anonimal): this is not optimal, especially for large subscriptions - // Add to address book - m_Storage->AddAddress(ident); - m_Addresses[host] = ident.GetIdentHash(); // TODO(anonimal): setter? + try + { + // Write/overwrite Hostname=Base64Address pairing to subscription file + // TODO(anonimal): this is not optimal, especially for large subscriptions + file << host << "=" << ident.ToBase64() << '\n'; + } + catch (...) + { + m_Exception.Dispatch(__func__); + continue; + } } // Flush subscription file file << std::flush; @@ -319,10 +327,8 @@ bool AddressBook::SaveSubscription( m_Storage->Save(m_Addresses); m_SubscriptionIsLoaded = true; } - } catch (const std::exception& ex) { - LOG(error) << "AddressBook: exception in " << __func__ << ": " << ex.what(); } catch (...) { - LOG(error) << "AddressBook: unknown exception in " << __func__; + m_Exception.Dispatch(__func__); } return m_SubscriptionIsLoaded; } @@ -332,16 +338,8 @@ bool AddressBook::SaveSubscription( const std::map AddressBook::ValidateSubscription(std::istream& stream) { LOG(debug) << "AddressBook: validating subscription"; - // Map host to address identity std::map addresses; - // To ensure valid Hostname=Base64Address std::string line; - // To ensure valid hostname - // Note: uncomment if this regexp fails on some locales (to not rely on [a-z]) - //const std::string alpha = "abcdefghijklmnopqrstuvwxyz"; - // TODO(unassigned): expand when we want to venture beyond the .i2p TLD - // TODO(unassigned): IDN ccTLDs support? - std::regex regex("(?=^.{1,253}$)(^(((?!-)[a-zA-Z0-9-]{1,63})|((?!-)[a-zA-Z0-9-]{1,63}\\.)+[a-zA-Z]+[(i2p)]{2,63})$)"); try { // Read through stream, add to address book while (std::getline(stream, line)) { @@ -352,35 +350,56 @@ AddressBook::ValidateSubscription(std::istream& stream) { boost::trim(line); // Parse Hostname=Base64Address from line kovri::core::IdentityEx ident; - std::size_t pos = line.find('='); - if (pos != std::string::npos) { - const std::string host = line.substr(0, pos++); - const std::string addr = line.substr(pos); - // Ensure only valid lines - try - { - if (host.empty() || !std::regex_search(host, regex)) - throw std::runtime_error("AddressBook: invalid hostname"); - ident.FromBase64(addr); - } - catch (...) - { - m_Exception.Dispatch(__func__); - LOG(warning) << "AddressBook: malformed address, skipping"; - continue; - } - addresses[host] = ident; // Host is valid, save + std::vector host_ident; + boost::split(host_ident, line, boost::is_any_of("=")); + if (host_ident.size() == 2) + { + const std::string host = host_ident[0]; + const std::string addr = host_ident[1]; + try + { + ident.FromBase64(addr); + // Validate and insert address into address book + InsertAddress(host, ident); + // Add to return addresses + // TODO(oneiric): remove after resolving #337 + addresses[host] = ident; + } + catch (...) + { + m_Exception.Dispatch(__func__); + LOG(warning) << "AddressBook: malformed address, skipping"; + continue; + } + } } } - } catch (const std::exception& ex) { - LOG(error) << "AddressBook: exception during validation: ", ex.what(); - addresses.clear(); - } catch (...) { - throw std::runtime_error("AddressBook: unknown exception during validation"); - } + catch (...) + { + addresses.clear(); + m_Exception.Dispatch(__func__); + throw; + } return addresses; } +bool AddressBook::ValidateAddress(const std::string& address) const +{ + const std::regex valid_tld("(?!^[\\.-])[a-zA-Z0-9.-]{1,63}(\\.i2p$)"), + invalid_sequences("\\.\\.|\\.-|-\\.|(?!(xn))--|(\\.b32\\.i2p$)"), + reserved_names("(proxy|console|router)\\.i2p$"); // TODO(oneiric): more names? + + if (address.length() > MAX_HOSTNAME_LENGTH) + return false; + if (std::regex_search(address, invalid_sequences)) + return false; + if (std::regex_search(address, reserved_names)) + return false; + if (std::regex_match(address, valid_tld)) + return true; + return false; +} + // For in-net download only bool AddressBook::CheckAddressIdentHashFound( const std::string& address, @@ -441,6 +460,52 @@ std::unique_ptr AddressBook::GetLoadedAddressIdent return nullptr; } +// For in-net download only +bool AddressBook::AddressLoaded(const std::string& address) const +{ + const auto it = m_Addresses.find(address); + if (it != m_Addresses.end()) + { + LOG(debug) << "AddressBook: " << address << " already imported"; + return true; + } + else + { + LOG(debug) << "AddressBook: " << address << " not found in loaded addresses"; + return false; + } +} + +// TODO(oneiric): implement reverse-lookup map: identity hashes -> addresses +bool AddressBook::IdentityLoaded(const kovri::core::IdentHash& ident_hash) const +{ + if (!m_Addresses.empty()) + { + for (const auto& entry : m_Addresses) + if (entry.second == ident_hash) + return true; + } + return false; +} + +void AddressBook::InsertAddress( + const std::string& address, + const kovri::core::IdentityEx& identity) +{ + if (!ValidateAddress(address)) + throw std::runtime_error("AddressBook: invalid hostname"); + if (AddressLoaded(address)) + throw std::runtime_error("AddressBook: address already loaded"); + // Reject multiple hostnames for the same identity + // TODO(oneiric): Possibly support multiple hostnames after resolving #337 + const auto& ident_hash = identity.GetIdentHash(); + if (IdentityLoaded(ident_hash)) + throw std::runtime_error("AddressBook: identity already loaded"); + m_Addresses[address] = ident_hash; + LOG(info) << "AddressBook: " << address << "->" + << kovri::core::GetB32Address(ident_hash) << " added"; +} + // Used only by HTTP Proxy void AddressBook::InsertAddressIntoStorage( const std::string& address, diff --git a/src/client/address_book/impl.h b/src/client/address_book/impl.h index 5396825a..2d27e668 100644 --- a/src/client/address_book/impl.h +++ b/src/client/address_book/impl.h @@ -1,5 +1,5 @@ /** // - * Copyright (c) 2013-2017, The Kovri I2P Router Project // + * Copyright (c) 2013-2018, The Kovri I2P Router Project // * // * All rights reserved. // * // @@ -58,6 +58,8 @@ namespace kovri { namespace client { +const std::uint8_t MAX_HOSTNAME_LENGTH = 67; + class AddressBookSubscriber; /// @class AddressBook /// @brief Address book implementation @@ -101,6 +103,18 @@ class AddressBook : public AddressBookDefaults { std::unique_ptr GetLoadedAddressIdentHash( const std::string& address); + /// @brief Checks if address already loaded into memory + /// @param address Human-readable address + /// @returns True if address in memory + /// @notes Used for in-net downloads only + bool AddressLoaded(const std::string& address) const; + + /// @brief Checks if identity already loaded into memory + /// @param ident_hash Router identity hash + /// @returns True if identity loaded in memory + /// @notes Used for in-net downloads only + bool IdentityLoaded(const kovri::core::IdentHash& ident_hash) const; + /// @brief Used for destination to fetch subscription(s) from publisher(s) /// @return Shared pointer to client destination std::shared_ptr GetSharedLocalDestination() const { @@ -139,6 +153,11 @@ class AddressBook : public AddressBookDefaults { const std::map ValidateSubscription(std::istream& stream); + /// @brief Validates human-readable address + /// @param address Human-readable I2P address + /// @return True if address is valid + bool ValidateAddress(const std::string& address) const; + /// @brief Sets the download state as complete and resets timer as needed /// @details Resets update timer according to the state of completed download /// @param success True if successful download, false if not @@ -155,17 +174,20 @@ class AddressBook : public AddressBookDefaults { return GetB32Address(ident); } + /// @brief Insert address into in-memory storage + /// @param address Human-readable hostname to insert + /// @param identity Full router identity to insert + void InsertAddress( + const std::string& address, + const kovri::core::IdentityEx& identity); + /** // TODO(unassigned): currently unused - std::string ToAddress( - const kovri::core::IdentityEx& ident) { - return ToAddress(ident.GetAddressIdentHash()); + std::string ToAddress(const kovri::core::IdentityEx& ident) const + { + return ToAddress(ident.GetIdentHash()); } - // TODO(unassigned): currently unused - void InsertAddress( - const kovri::core::IdentityEx& address); - // TODO(unassigned): currently unused bool GetAddress( const std::string& address,