Skip to content

Commit

Permalink
AddressBook: store only unique addresses
Browse files Browse the repository at this point in the history 
New helper functions validate human readable hostnames,
and ensure only unique entries are loaded from subscriptions.

Refactor subscription validation + saving with new helpers.

Refs monero-project#835
  • Loading branch information
@coneiric
coneiric committed Mar 18, 2018
1 parent 8b43557 commit b601548
Show file tree
Hide file tree
Showing 2 changed files with 136 additions and 49 deletions.
147 changes: 106 additions & 41 deletions src/client/address_book/impl.cc
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/** //
* Copyright (c) 2013-2017, The Kovri I2P Router Project //
* Copyright (c) 2013-2018, The Kovri I2P Router Project //
* //
* All rights reserved. //
* //
Expand Down Expand Up @@ -228,6 +228,7 @@ void AddressBook::DownloadSubscription() {
<< "AddressBook: picking random subscription from total publisher count: "
<< publisher_count;
// Pick a random publisher to subscribe from
// TODO(oneiric): download all subscriptions not already stored
auto publisher = kovri::core::RandInRange32(0, publisher_count - 1);
m_SubscriberIsDownloading = true;
try {
Expand Down Expand Up @@ -295,6 +296,7 @@ bool AddressBook::SaveSubscription(
LOG(debug) << "AddressBook: processing " << addresses.size() << " addresses";
// Stream may be a file or downloaded stream.
// Regardless, we want to write/overwrite the subscription file.
// TODO(oneiric): save unique entries to userhosts.txt
if (file_name.empty()) // Use default filename if none given.
file_name = (core::GetPath(core::Path::AddressBook) / GetDefaultSubscriptionFilename()).string();
LOG(debug) << "AddressBook: opening subscription file " << file_name;
Expand All @@ -307,22 +309,26 @@ bool AddressBook::SaveSubscription(
for (auto const& address : addresses) {
const std::string& host = address.first;
const auto& ident = address.second;
// Write/overwrite Hostname=Base64Address pairing to subscription file
file << host << "=" << ident.ToBase64() << '\n'; // TODO(anonimal): this is not optimal, especially for large subscriptions
// Add to address book
m_Storage->AddAddress(ident);
m_Addresses[host] = ident.GetIdentHash(); // TODO(anonimal): setter?
try
{
// Write/overwrite Hostname=Base64Address pairing to subscription file
// TODO(anonimal): this is not optimal, especially for large subscriptions
file << host << "=" << ident.ToBase64() << '\n';
}
catch (...)
{
m_Exception.Dispatch(__func__);
continue;
}
}
// Flush subscription file
file << std::flush;
// Save a *list* of hosts within subscription to a catalog (CSV) file
m_Storage->Save(m_Addresses);
m_SubscriptionIsLoaded = true;
}
} catch (const std::exception& ex) {
LOG(error) << "AddressBook: exception in " << __func__ << ": " << ex.what();
} catch (...) {
LOG(error) << "AddressBook: unknown exception in " << __func__;
m_Exception.Dispatch(__func__);
}
return m_SubscriptionIsLoaded;
}
Expand All @@ -332,16 +338,8 @@ bool AddressBook::SaveSubscription(
const std::map<std::string, kovri::core::IdentityEx>
AddressBook::ValidateSubscription(std::istream& stream) {
LOG(debug) << "AddressBook: validating subscription";
// Map host to address identity
std::map<std::string, kovri::core::IdentityEx> addresses;
// To ensure valid Hostname=Base64Address
std::string line;
// To ensure valid hostname
// Note: uncomment if this regexp fails on some locales (to not rely on [a-z])
//const std::string alpha = "abcdefghijklmnopqrstuvwxyz";
// TODO(unassigned): expand when we want to venture beyond the .i2p TLD
// TODO(unassigned): IDN ccTLDs support?
std::regex regex("(?=^.{1,253}$)(^(((?!-)[a-zA-Z0-9-]{1,63})|((?!-)[a-zA-Z0-9-]{1,63}\\.)+[a-zA-Z]+[(i2p)]{2,63})$)");
try {
// Read through stream, add to address book
while (std::getline(stream, line)) {
Expand All @@ -352,35 +350,56 @@ AddressBook::ValidateSubscription(std::istream& stream) {
boost::trim(line);
// Parse Hostname=Base64Address from line
kovri::core::IdentityEx ident;
std::size_t pos = line.find('=');
if (pos != std::string::npos) {
const std::string host = line.substr(0, pos++);
const std::string addr = line.substr(pos);
// Ensure only valid lines
try
{
if (host.empty() || !std::regex_search(host, regex))
throw std::runtime_error("AddressBook: invalid hostname");
ident.FromBase64(addr);
}
catch (...)
{
m_Exception.Dispatch(__func__);
LOG(warning) << "AddressBook: malformed address, skipping";
continue;
}
addresses[host] = ident; // Host is valid, save
std::vector<std::string> host_ident;
boost::split(host_ident, line, boost::is_any_of("="));
if (host_ident.size() == 2)
{
const std::string host = host_ident[0];
const std::string addr = host_ident[1];
try
{
ident.FromBase64(addr);
// Validate and insert address into address book
InsertAddress(host, ident);
// Add to return addresses
// TODO(oneiric): remove after resolving #337
addresses[host] = ident;
}
catch (...)
{
m_Exception.Dispatch(__func__);
LOG(warning) << "AddressBook: malformed address, skipping";
continue;
}
}
}
}
} catch (const std::exception& ex) {
LOG(error) << "AddressBook: exception during validation: ", ex.what();
addresses.clear();
} catch (...) {
throw std::runtime_error("AddressBook: unknown exception during validation");
}
catch (...)
{
addresses.clear();
m_Exception.Dispatch(__func__);
throw;
}
return addresses;
}

bool AddressBook::ValidateAddress(const std::string& address) const
{
const std::regex valid_tld("(?!^[\\.-])[a-zA-Z0-9.-]{1,63}(\\.i2p$)"),
invalid_sequences("\\.\\.|\\.-|-\\.|(?!(xn))--|(\\.b32\\.i2p$)"),
reserved_names("(proxy|console|router)\\.i2p$"); // TODO(oneiric): more names?

if (address.length() > MAX_HOSTNAME_LENGTH)
return false;
if (std::regex_search(address, invalid_sequences))
return false;
if (std::regex_search(address, reserved_names))
return false;
if (std::regex_match(address, valid_tld))
return true;
return false;
}

// For in-net download only
bool AddressBook::CheckAddressIdentHashFound(
const std::string& address,
Expand Down Expand Up @@ -441,6 +460,52 @@ std::unique_ptr<const kovri::core::IdentHash> AddressBook::GetLoadedAddressIdent
return nullptr;
}

// For in-net download only
bool AddressBook::AddressLoaded(const std::string& address) const
{
const auto it = m_Addresses.find(address);
if (it != m_Addresses.end())
{
LOG(debug) << "AddressBook: " << address << " already imported";
return true;
}
else
{
LOG(debug) << "AddressBook: " << address << " not found in loaded addresses";
return false;
}
}

// TODO(oneiric): implement reverse-lookup map: identity hashes -> addresses
bool AddressBook::IdentityLoaded(const kovri::core::IdentHash& ident_hash) const
{
if (!m_Addresses.empty())
{
for (const auto& entry : m_Addresses)
if (entry.second == ident_hash)
return true;
}
return false;
}

void AddressBook::InsertAddress(
const std::string& address,
const kovri::core::IdentityEx& identity)
{
if (!ValidateAddress(address))
throw std::runtime_error("AddressBook: invalid hostname");
if (AddressLoaded(address))
throw std::runtime_error("AddressBook: address already loaded");
// Reject multiple hostnames for the same identity
// TODO(oneiric): Possibly support multiple hostnames after resolving #337
const auto& ident_hash = identity.GetIdentHash();
if (IdentityLoaded(ident_hash))
throw std::runtime_error("AddressBook: identity already loaded");
m_Addresses[address] = ident_hash;
LOG(info) << "AddressBook: " << address << "->"
<< kovri::core::GetB32Address(ident_hash) << " added";
}

// Used only by HTTP Proxy
void AddressBook::InsertAddressIntoStorage(
const std::string& address,
Expand Down
38 changes: 30 additions & 8 deletions src/client/address_book/impl.h
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/** //
* Copyright (c) 2013-2017, The Kovri I2P Router Project //
* Copyright (c) 2013-2018, The Kovri I2P Router Project //
* //
* All rights reserved. //
* //
Expand Down Expand Up @@ -58,6 +58,8 @@
namespace kovri {
namespace client {

const std::uint8_t MAX_HOSTNAME_LENGTH = 67;

class AddressBookSubscriber;
/// @class AddressBook
/// @brief Address book implementation
Expand Down Expand Up @@ -101,6 +103,18 @@ class AddressBook : public AddressBookDefaults {
std::unique_ptr<const kovri::core::IdentHash> GetLoadedAddressIdentHash(
const std::string& address);

/// @brief Checks if address already loaded into memory
/// @param address Human-readable address
/// @returns True if address in memory
/// @notes Used for in-net downloads only
bool AddressLoaded(const std::string& address) const;

/// @brief Checks if identity already loaded into memory
/// @param ident_hash Router identity hash
/// @returns True if identity loaded in memory
/// @notes Used for in-net downloads only
bool IdentityLoaded(const kovri::core::IdentHash& ident_hash) const;

/// @brief Used for destination to fetch subscription(s) from publisher(s)
/// @return Shared pointer to client destination
std::shared_ptr<ClientDestination> GetSharedLocalDestination() const {
Expand Down Expand Up @@ -139,6 +153,11 @@ class AddressBook : public AddressBookDefaults {
const std::map<std::string, kovri::core::IdentityEx>
ValidateSubscription(std::istream& stream);

/// @brief Validates human-readable address
/// @param address Human-readable I2P address
/// @return True if address is valid
bool ValidateAddress(const std::string& address) const;

/// @brief Sets the download state as complete and resets timer as needed
/// @details Resets update timer according to the state of completed download
/// @param success True if successful download, false if not
Expand All @@ -155,17 +174,20 @@ class AddressBook : public AddressBookDefaults {
return GetB32Address(ident);
}

/// @brief Insert address into in-memory storage
/// @param address Human-readable hostname to insert
/// @param identity Full router identity to insert
void InsertAddress(
const std::string& address,
const kovri::core::IdentityEx& identity);

/**
// TODO(unassigned): currently unused
std::string ToAddress(
const kovri::core::IdentityEx& ident) {
return ToAddress(ident.GetAddressIdentHash());
std::string ToAddress(const kovri::core::IdentityEx& ident) const
{
return ToAddress(ident.GetIdentHash());
}
// TODO(unassigned): currently unused
void InsertAddress(
const kovri::core::IdentityEx& address);
// TODO(unassigned): currently unused
bool GetAddress(
const std::string& address,
Expand Down

0 comments on commit b601548

Please sign in to comment.