Skip to content
This repository has been archived by the owner on Sep 1, 2022. It is now read-only.

Commit

Permalink
Merge pull request #564
Browse files Browse the repository at this point in the history 
1221a5e RouterProfile: style check-in for profile deletion (anonimal)
1108709 RouterProfile: fix profile saving/loading/deleting (anonimal)
47ea7d6 NetDb: style check-in for creation + loading (anonimal)
e5b6ca3 NetDb/Filesystem: resolve misc refactoring TODO's (anonimal)
ffb27e1 NetDb: fix RI loading/saving implementation (anonimal)
  • Loading branch information
@anonimal
anonimal committed Feb 18, 2017
2 parents 8faa026 + 1221a5e commit 2447d42
Show file tree
Hide file tree
Showing 4 changed files with 185 additions and 108 deletions.
148 changes: 91 additions & 57 deletions src/core/router/net_db/impl.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@

#include <string.h>

#include <cctype>
#include <fstream>
#include <memory>
#include <set>
Expand Down Expand Up @@ -285,72 +286,99 @@ void NetDb::SetUnreachable(

// TODO(unassigned): Move to reseed and/or scheduled tasks.
// (In java version, scheduler fixes this as well as sort RIs.)
bool NetDb::CreateNetDb(
boost::filesystem::path directory) {
LOG(debug) << "NetDb: creating " << directory.string();
if (!boost::filesystem::create_directory(directory)) {
LOG(error) << "NetDb: failed to create " << directory.string();
return false;
}
// list of chars might appear in base64 string
const char* chars = kovri::core::GetBase64SubstitutionTable(); // 64 bytes
boost::filesystem::path suffix;
for (int i = 0; i < 64; i++) {
#ifndef _WIN32
suffix = std::string("/r") + chars[i];
bool NetDb::CreateNetDb(boost::filesystem::path directory)
{
try
{
LOG(debug) << "NetDb: ensuring " << directory.string();
core::EnsurePath(directory);
// TODO(unassigned): this is a patch for #520 until we implement a database in #385
#if defined(_WIN32) || defined(__APPLE__)
core::EnsurePath(directory / "uppercase");
core::EnsurePath(directory / "lowercase");
#endif
// list of chars might appear in base64 string
const char* chars = kovri::core::GetBase64SubstitutionTable(); // 64 bytes
boost::filesystem::path suffix;
for (int i = 0; i < 64; i++)
{
#ifdef _WIN32
suffix = std::string("\\r") + chars[i];
#else
suffix = std::string("\\r") + chars[i];
suffix = std::string("/r") + chars[i];
#endif
if (!boost::filesystem::create_directory(
boost::filesystem::path(
directory / suffix) ))
// TODO(unassigned): this is a patch for #520 until we implement a database in #385
std::string sub_dir;
#if defined(_WIN32) || defined(__APPLE__)
sub_dir = std::isupper(chars[i]) ? "uppercase" : "lowercase";
#endif
const auto& path = directory / sub_dir / suffix;
LOG(debug) << "NetDb: ensuring " << path;
core::EnsurePath(path);
}
}
catch (...)
{
m_Exception.Dispatch(__func__);
return false;
}
}
return true;
}

bool NetDb::Load() {
boost::filesystem::path p(kovri::core::GetNetDbPath());
if (!boost::filesystem::exists(p)) {
// seems netDb doesn't exist yet
if (!CreateNetDb(p))
return false;
}
// make sure we cleanup netDb from previous attempts
bool NetDb::Load()
{
// Create NetDb if it does not exist
const auto& path = core::GetNetDbPath();
if (!CreateNetDb(path))
return false;
// Cleanup the database from previous attempts
m_RouterInfos.clear();
m_Floodfills.clear();
// load routers now
std::uint64_t ts = kovri::core::GetMillisecondsSinceEpoch();
boost::filesystem::directory_iterator end;
// Load RI's from given path
std::size_t num_routers = 0;
for (boost::filesystem::directory_iterator it(p); it != end; ++it) {
if (boost::filesystem::is_directory(it->status())) {
for (boost::filesystem::directory_iterator it1(it->path());
it1 != end;
++it1) {
#if BOOST_VERSION > 10500
const std::string& full_path = it1->path().string();
auto LoadRouterInfos = [&](const boost::filesystem::path& path) {
std::uint64_t timestamp = kovri::core::GetMillisecondsSinceEpoch();
boost::filesystem::directory_iterator end;
for (boost::filesystem::directory_iterator dir(path); dir != end; ++dir)
{
if (boost::filesystem::is_directory(dir->status()))
{
for (boost::filesystem::directory_iterator it(dir->path());
it != end;
++it)
{
const std::string& full_path = it->path().string();
auto router = std::make_shared<RouterInfo>(full_path);
if (!router->IsUnreachable()
&& (!router->UsesIntroducer()
|| timestamp < router->GetTimestamp()
+ GetType(NetDbTime::RouterExpiration)))
{
router->DeleteBuffer();
router->ClearProperties(); // properties are not used for regular routers
m_RouterInfos.insert(std::make_pair(router->GetIdentHash(), router));
if (router->IsFloodfill())
m_Floodfills.push_back(router);
num_routers++;
}
else
{
// Remove unreachable routers
if (boost::filesystem::remove(full_path))
LOG(debug) << "NetDb: " << full_path
<< " unreachable router removed";
}
}
}
}
};
// TODO(unassigned): this is a patch for #520 until we implement a database in #385
#if defined(_WIN32) || defined(__APPLE__)
LoadRouterInfos(path / "uppercase");
LoadRouterInfos(path / "lowercase");
#else
const std::string& full_path = it1->path();
LoadRouterInfos(path);
#endif
auto r = std::make_shared<RouterInfo>(full_path);
if (!r->IsUnreachable() &&
(!r->UsesIntroducer() || ts < r->GetTimestamp() +
static_cast<std::uint32_t>(NetDbTime::RouterExpiration))) {
r->DeleteBuffer();
r->ClearProperties(); // properties are not used for regular routers
m_RouterInfos[r->GetIdentHash()] = r;
if (r->IsFloodfill())
m_Floodfills.push_back(r);
num_routers++;
} else {
bool is_removed = boost::filesystem::remove(full_path);
if (is_removed)
LOG(debug) << "NetDb: " << full_path << " unreachable router removed";
}
}
}
}
LOG(debug) << "NetDb: " << num_routers << " routers loaded";
LOG(debug) << "NetDb: " << m_Floodfills.size() << " floodfills loaded";
return true;
Expand All @@ -360,8 +388,13 @@ void NetDb::SaveUpdated() {
auto GetFilePath = [](
const boost::filesystem::path& directory,
const RouterInfo* router_info) {
std::string s(router_info->GetIdentHashBase64());
return directory / (std::string("r") + s[0]) / ("router_info_" + s + ".dat");
const std::string base64(router_info->GetIdentHashBase64());
// TODO(unassigned): this is a patch for #520 until we implement a database in #385
std::string sub_dir;
#if defined(_WIN32) || defined(__APPLE__)
sub_dir = std::isupper(base64[0]) ? "uppercase" : "lowercase";
#endif
return directory / sub_dir / (std::string("r") + base64[0]) / ("router_info_" + base64 + ".dat");
};
boost::filesystem::path full_directory(kovri::core::GetNetDbPath());
int count = 0, deleted_count = 0;
Expand All @@ -370,6 +403,7 @@ void NetDb::SaveUpdated() {
for (auto it : m_RouterInfos) {
if (it.second->IsUpdated()) {
std::string f = GetFilePath(full_directory, it.second.get()).string();
LOG(debug) << "NetDb: " << __func__ << " saving " << f;
it.second->SaveToFile(f);
it.second->SetUpdated(false);
it.second->SetUnreachable(false);
Expand Down
137 changes: 88 additions & 49 deletions src/core/router/profiling.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#include <boost/property_tree/ini_parser.hpp>
#include <boost/property_tree/ptree.hpp>

#include <cctype>
#include <string>

#include "core/util/base64.h"
Expand All @@ -53,7 +54,8 @@ RouterProfile::RouterProfile(
m_NumTunnelsDeclined(0),
m_NumTunnelsNonReplied(0),
m_NumTimesTaken(0),
m_NumTimesRejected(0) {}
m_NumTimesRejected(0),
m_Exception(__func__) {}

boost::posix_time::ptime RouterProfile::GetTime() const {
return boost::posix_time::second_clock::local_time();
Expand Down Expand Up @@ -94,38 +96,61 @@ void RouterProfile::Save() {
pt.put_child(
PEER_PROFILE_SECTION_USAGE,
usage);
// save to file
auto path = kovri::core::GetProfilesPath();
if (!boost::filesystem::exists(path)) {
// Create directory is necessary
if (!boost::filesystem::create_directory(path)) {
LOG(error) << "RouterProfile: failed to create directory " << path;
return;
// Save to file
// TODO(unassigned): this entire block is a patch for #519 until we implement a database in #385
try
{
auto directory = kovri::core::GetProfilesPath();
std::string sub_dir;
if (!boost::filesystem::exists(directory))
{
LOG(debug) << "RouterProfile: ensuring " << directory;
core::EnsurePath(directory);
#if defined(_WIN32) || defined(__APPLE__)
core::EnsurePath(directory / "uppercase");
core::EnsurePath(directory / "lowercase");
#endif
// 64 bytes
const char* chars = kovri::core::GetBase64SubstitutionTable();
for (int i = 0; i < 64; i++)
{
#if defined(_WIN32) || defined(__APPLE__)
sub_dir = std::isupper(chars[i]) ? "uppercase" : "lowercase";
#endif
const auto& path =
directory / sub_dir / (std::string("p") + chars[i]);
LOG(debug) << "RouterProfile: ensuring " << path;
core::EnsurePath(path);
}
}
std::string base64 = m_IdentHash.ToBase64();
#if defined(_WIN32) || defined(__APPLE__)
sub_dir = std::isupper(base64[0]) ? "uppercase" : "lowercase";
#endif
directory = directory / sub_dir / (std::string("p") + base64[0]);
const auto& filename =
directory / (std::string(PEER_PROFILE_PREFIX) + base64 + ".txt");
LOG(debug) << "RouterProfile: saving " << filename;
boost::property_tree::write_ini(filename.string(), pt);
}
const char* chars = kovri::core::GetBase64SubstitutionTable(); // 64 bytes
for (int i = 0; i < 64; i++) {
auto path1 = path / (std::string("p") + chars[i]);
if (!boost::filesystem::create_directory(path1)) {
LOG(error) << "RouterProfile: failed to create directory " << path1;
return;
}
catch (...)
{
m_Exception.Dispatch(__func__);
// TODO(unassigned): surely we don't want to simply return
return;
}
}
std::string base64 = m_IdentHash.ToBase64();
path = path / (std::string("p") + base64[0]);
auto filename = path / (std::string(PEER_PROFILE_PREFIX) + base64 + ".txt");
try {
boost::property_tree::write_ini(filename.string(), pt);
} catch (std::exception& ex) {
LOG(error) << "RouterProfile: can't write " << filename << ": " << ex.what();
}
}

void RouterProfile::Load() {
std::string base64 = m_IdentHash.ToBase64();
auto path = kovri::core::GetProfilesPath();
path /= std::string("p") + base64[0];
auto filename = path / (std::string(PEER_PROFILE_PREFIX) + base64 + ".txt");
auto directory = kovri::core::GetProfilesPath();
// TODO(unassigned): this is a patch for #519 until we implement a database in #385
std::string sub_dir;
#if defined(_WIN32) || defined(__APPLE__)
sub_dir = std::isupper(base64[0]) ? "uppercase" : "lowercase";
#endif
directory = directory / sub_dir / (std::string("p") + base64[0]);
auto filename = directory / sub_dir / (std::string(PEER_PROFILE_PREFIX) + base64 + ".txt");
if (boost::filesystem::exists(filename)) {
boost::property_tree::ptree pt;
try {
Expand Down Expand Up @@ -224,30 +249,44 @@ std::shared_ptr<RouterProfile> GetRouterProfile(
return profile;
}

void DeleteObsoleteProfiles() {
int num = 0;
auto ts = boost::posix_time::second_clock::local_time();
boost::filesystem::path p(kovri::core::GetProfilesPath());
if (boost::filesystem::exists(p)) {
boost::filesystem::directory_iterator end;
for (boost::filesystem::directory_iterator it(p); it != end; ++it) {
if (boost::filesystem::is_directory(it->status())) {
for (boost::filesystem::directory_iterator it1(it->path());
it1 != end;
++it1) {
auto last_modified =
boost::posix_time::from_time_t(
boost::filesystem::last_write_time(
it1->path()));
if ((ts - last_modified).hours() >= PEER_PROFILE_EXPIRATION_TIMEOUT) {
boost::filesystem::remove(it1->path());
num++;
void DeleteObsoleteProfiles()
{
boost::filesystem::path path(kovri::core::GetProfilesPath());
std::size_t num_profiles = 0;
auto RemoveProfiles = [&](const boost::filesystem::path& path) {
auto timestamp = boost::posix_time::second_clock::local_time();
if (boost::filesystem::exists(path))
{
boost::filesystem::directory_iterator end;
for (boost::filesystem::directory_iterator dir(path); dir != end; ++dir)
{
if (boost::filesystem::is_directory(dir->status()))
{
for (boost::filesystem::directory_iterator it(dir->path());
it != end;
++it)
{
auto last_modified = boost::posix_time::from_time_t(
boost::filesystem::last_write_time(it->path()));
if ((timestamp - last_modified).hours()
>= PEER_PROFILE_EXPIRATION_TIMEOUT)
{
boost::filesystem::remove(it->path());
num_profiles++;
}
}
}
}
}
}
}
}
LOG(debug) << "Profiling: " << num << " obsolete profiles deleted";
};
// TODO(unassigned): this is a patch for #519 until we implement a database in #385
#if defined(_WIN32) || defined(__APPLE__)
RemoveProfiles(path / "uppercase");
RemoveProfiles(path / "lowercase");
#else
RemoveProfiles(path);
#endif
LOG(debug) << "Profiling: " << num_profiles << " obsolete profiles deleted";
}

} // namespace core
Expand Down
3 changes: 3 additions & 0 deletions src/core/router/profiling.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@

#include "core/router/identity.h"

#include "core/util/exception.h"

namespace kovri {
namespace core {

Expand Down Expand Up @@ -90,6 +92,7 @@ class RouterProfile {
// usage
std::uint32_t m_NumTimesTaken;
std::uint32_t m_NumTimesRejected;
core::Exception m_Exception;
};

std::shared_ptr<RouterProfile> GetRouterProfile(
Expand Down
5 changes: 3 additions & 2 deletions src/core/util/filesystem.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,10 @@ std::string g_AppName("kovri");
const boost::filesystem::path EnsurePath(
const boost::filesystem::path& path) {
if (!boost::filesystem::exists(path)) {
if (!boost::filesystem::create_directory(path)) {
boost::system::error_code ec;
if (!boost::filesystem::create_directory(path, ec)) {
throw std::runtime_error(
"Filesystem: failed to create " + std::string(path.string()));
"Filesystem: failed to create " + std::string(path.string() + ": " + ec.message()));
}
}
return path;
Expand Down

0 comments on commit 2447d42

Please sign in to comment.