Skip to content

Commit

Permalink
On POSIX systems, use *at APIs in recursive_directory_iterator.
Browse files Browse the repository at this point in the history 
This makes the iterator more resilient to concurrent filesystem
modifications.
  • Loading branch information
@Lastique
Lastique committed Feb 11, 2024
1 parent d3f4ad6 commit 7eece00
Show file tree
Hide file tree
Showing 7 changed files with 241 additions and 126 deletions.
3 changes: 2 additions & 1 deletion config/has_fdopendir_nofollow.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,6 @@ int main()
{
int fd = open(".", O_DIRECTORY | O_RDONLY | O_NOFOLLOW);
DIR* dir = fdopendir(fd);
return dir != 0;
int internal_fd = dirfd(dir);
return dir != 0 && internal_fd >= 0;
}
1 change: 1 addition & 0 deletions doc/release_history.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ <h2>1.85.0</h2>
<li><code>weakly_canonical</code> now produces an absolute path if the input path is relative and contains no elements that exist in the filesystem. (<a href="https://github.com/boostorg/filesystem/issues/300">#300</a>)</li>
<li>Added a new <code>copy_options::ignore_attribute_errors</code> option for <code>copy_file</code> and <code>copy</code> operations. The new option allows to ignore possible errors while copying file attributes. (<a href="https://github.com/boostorg/filesystem/issues/179">#179</a>)</li>
<li>On Linux, <code>copy_file</code> backends based on <code>sendfile</code> and <code>copy_file_range</code> system calls will attempt to preallocate storage for the target file. This may reduce filesystem fragmentation and provide early error indication if there is not enough free space. Not all filesystems support this feature; file copying proceeds if storage preallocation is not supported.</li>
<li>On POSIX systems that support <a href="https://pubs.opengroup.org/onlinepubs/9699919799/functions/fdopendir.html"><code>fdopendir</code></a>, <code>openat</code> and related APIs defined in POSIX.1-2008, <code>recursive_directory_iterator</code> now uses these APIs internally. This makes directory iteration more resilient to concurrent modifications of the filesystem.</li>
<li>Removed APIs that were previously declared deprecated. In particular, <code>path</code> and <code>recursive_directory_iterator</code> member functions, <code>is_regular</code>, <code>copy_directory</code>, <code>symbolic_link_exists</code>, <code>complete</code>, <code>copy_option</code>, <code>symlink_option</code>, as well as <code>boost/filesystem/convenience.hpp</code> and <code>boost/filesystem/path_traits.hpp</code> headers were removed. Possible replacements for the removed components are mentioned in <a href="deprecated.html#Deprecated-names">Deprecated names and features</a> section.</li>
<li>Support for <code>path</code> construction, assignment and appending from container types (e.g. <code>std::vector&lt;char&gt;</code>) is now disabled by default. Users can still enable this functionality by defining <code>BOOST_FILESYSTEM_DEPRECATED</code>. This functionality remains deprecated and will be completely removed in a future release.</li>
</ul>
Expand Down
17 changes: 11 additions & 6 deletions include/boost/filesystem/directory.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ enum class directory_options : unsigned int
BOOST_BITMASK(directory_options)

class directory_iterator;
class recursive_directory_iterator;

namespace detail {

Expand All @@ -63,6 +64,12 @@ struct directory_iterator_params;
BOOST_FILESYSTEM_DECL void directory_iterator_construct(directory_iterator& it, path const& p, directory_options opts, directory_iterator_params* params, system::error_code* ec);
BOOST_FILESYSTEM_DECL void directory_iterator_increment(directory_iterator& it, system::error_code* ec);

struct recur_dir_itr_imp;

BOOST_FILESYSTEM_DECL void recursive_directory_iterator_construct(recursive_directory_iterator& it, path const& dir_path, directory_options opts, system::error_code* ec);
BOOST_FILESYSTEM_DECL void recursive_directory_iterator_increment(recursive_directory_iterator& it, system::error_code* ec);
BOOST_FILESYSTEM_DECL void recursive_directory_iterator_pop(recursive_directory_iterator& it, system::error_code* ec);

} // namespace detail

//--------------------------------------------------------------------------------------//
Expand All @@ -80,6 +87,8 @@ class directory_entry
friend BOOST_FILESYSTEM_DECL void detail::directory_iterator_construct(directory_iterator& it, path const& p, directory_options opts, detail::directory_iterator_params* params, system::error_code* ec);
friend BOOST_FILESYSTEM_DECL void detail::directory_iterator_increment(directory_iterator& it, system::error_code* ec);

friend BOOST_FILESYSTEM_DECL void detail::recursive_directory_iterator_increment(recursive_directory_iterator& it, system::error_code* ec);

public:
typedef boost::filesystem::path::value_type value_type; // enables class path ctor taking directory_entry

Expand Down Expand Up @@ -660,6 +669,8 @@ class directory_iterator :
friend BOOST_FILESYSTEM_DECL void detail::directory_iterator_construct(directory_iterator& it, path const& p, directory_options opts, detail::directory_iterator_params* params, system::error_code* ec);
friend BOOST_FILESYSTEM_DECL void detail::directory_iterator_increment(directory_iterator& it, system::error_code* ec);

friend BOOST_FILESYSTEM_DECL void detail::recursive_directory_iterator_increment(recursive_directory_iterator& it, system::error_code* ec);

public:
directory_iterator() noexcept {} // creates the "end" iterator

Expand Down Expand Up @@ -808,8 +819,6 @@ namespace filesystem {
// //
//--------------------------------------------------------------------------------------//

class recursive_directory_iterator;

namespace detail {

struct recur_dir_itr_imp :
Expand All @@ -822,10 +831,6 @@ struct recur_dir_itr_imp :
explicit recur_dir_itr_imp(directory_options opts) noexcept : m_options(opts) {}
};

BOOST_FILESYSTEM_DECL void recursive_directory_iterator_construct(recursive_directory_iterator& it, path const& dir_path, directory_options opts, system::error_code* ec);
BOOST_FILESYSTEM_DECL void recursive_directory_iterator_increment(recursive_directory_iterator& it, system::error_code* ec);
BOOST_FILESYSTEM_DECL void recursive_directory_iterator_pop(recursive_directory_iterator& it, system::error_code* ec);

} // namespace detail

//--------------------------------------------------------------------------------------//
Expand Down
245 changes: 165 additions & 80 deletions src/directory.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,23 @@ inline system::error_code dir_itr_close(dir_itr_imp& imp) noexcept
return error_code();
}

#if defined(BOOST_FILESYSTEM_HAS_FDOPENDIR_NOFOLLOW)

// Obtains a file descriptor from the directory iterator
inline int dir_itr_fd(dir_itr_imp const& imp, system::error_code& ec)
{
int fd = ::dirfd(static_cast< DIR* >(imp.handle));
if (BOOST_UNLIKELY(fd < 0))
{
int err = errno;
ec = system::error_code(err, system::system_category());
}

return fd;
}

#endif // defined(BOOST_FILESYSTEM_HAS_FDOPENDIR_NOFOLLOW)

#if defined(BOOST_FILESYSTEM_USE_READDIR_R)

// Obtains maximum length of a path, not including the terminating zero
Expand Down Expand Up @@ -1020,6 +1037,21 @@ void directory_iterator_construct(directory_iterator& it, path const& p, directo

try
{
#if defined(BOOST_POSIX_API) && defined(BOOST_FILESYSTEM_HAS_FDOPENDIR_NOFOLLOW) && defined(BOOST_FILESYSTEM_HAS_POSIX_AT_APIS)
path dir_path;
if (params)
{
dir_path = params->basedir;
path_algorithms::append_v4(dir_path, p);
}
else
{
dir_path = p;
}
#else
path const& dir_path = p;
#endif

boost::intrusive_ptr< detail::dir_itr_imp > imp;
path filename;
file_status file_stat, symlink_file_stat;
Expand Down Expand Up @@ -1049,7 +1081,7 @@ void directory_iterator_construct(directory_iterator& it, path const& p, directo
&& (filename_str[1] == static_cast< path::string_type::value_type >('\0') ||
(filename_str[1] == path::dot && filename_str[2] == static_cast< path::string_type::value_type >('\0')))))
{
path full_path(p);
path full_path(dir_path);
path_algorithms::append_v4(full_path, filename);
imp->dir_entry.assign_with_status
(
Expand Down Expand Up @@ -1251,99 +1283,152 @@ void recursive_directory_iterator_pop(recursive_directory_iterator& it, system::
}
}

namespace {

enum push_directory_result
{
directory_not_pushed = 0u,
directory_pushed = 1u,
keep_depth = 1u << 1
};

// Returns: true if push occurs, otherwise false. Always returns false on error.
inline push_directory_result recursive_directory_iterator_push_directory(detail::recur_dir_itr_imp* imp, system::error_code& ec) noexcept
BOOST_FILESYSTEM_DECL
void recursive_directory_iterator_increment(recursive_directory_iterator& it, system::error_code* ec)
{
push_directory_result result = directory_not_pushed;
try
enum push_directory_result : unsigned int
{
// Discover if the iterator is for a directory that needs to be recursed into,
// taking symlinks and options into account.
directory_not_pushed = 0u,
directory_pushed = 1u,
keep_depth = 1u << 1u
};

if ((imp->m_options & directory_options::_detail_no_push) != directory_options::none)
struct local
{
//! Attempts to descend into a directory
static push_directory_result push_directory(detail::recur_dir_itr_imp* imp, system::error_code& ec)
{
imp->m_options &= ~directory_options::_detail_no_push;
return result;
}
push_directory_result result = directory_not_pushed;
try
{
// Discover if the iterator is for a directory that needs to be recursed into,
// taking symlinks and options into account.

file_type symlink_ft = status_error;
if ((imp->m_options & directory_options::_detail_no_push) != directory_options::none)
{
imp->m_options &= ~directory_options::_detail_no_push;
return result;
}

// If we are not recursing into symlinks, we are going to have to know if the
// stack top is a symlink, so get symlink_status and verify no error occurred.
if ((imp->m_options & directory_options::follow_directory_symlink) == directory_options::none ||
(imp->m_options & directory_options::skip_dangling_symlinks) != directory_options::none)
{
symlink_ft = imp->m_stack.back()->symlink_file_type(ec);
if (ec)
return result;
}
file_type symlink_ft = status_error;

// Logic for following predicate was contributed by Daniel Aarno to handle cyclic
// symlinks correctly and efficiently, fixing ticket #5652.
// if (((m_options & directory_options::follow_directory_symlink) == directory_options::follow_directory_symlink
// || !is_symlink(m_stack.back()->symlink_status()))
// && is_directory(m_stack.back()->status())) ...
// The predicate code has since been rewritten to pass error_code arguments,
// per ticket #5653.
#if defined(BOOST_POSIX_API) && defined(BOOST_FILESYSTEM_HAS_FDOPENDIR_NOFOLLOW) && defined(BOOST_FILESYSTEM_HAS_POSIX_AT_APIS)
int parentdir_fd = -1;
path dir_it_filename;
#endif

if ((imp->m_options & directory_options::follow_directory_symlink) != directory_options::none || symlink_ft != symlink_file)
{
file_type ft = imp->m_stack.back()->file_type(ec);
if (BOOST_UNLIKELY(!!ec))
{
if (ec == make_error_condition(system::errc::no_such_file_or_directory) && symlink_ft == symlink_file &&
(imp->m_options & (directory_options::follow_directory_symlink | directory_options::skip_dangling_symlinks)) == (directory_options::follow_directory_symlink | directory_options::skip_dangling_symlinks))
// If we are not recursing into symlinks, we are going to have to know if the
// stack top is a symlink, so get symlink_status and verify no error occurred.
if ((imp->m_options & directory_options::follow_directory_symlink) == directory_options::none ||
(imp->m_options & directory_options::skip_dangling_symlinks) != directory_options::none)
{
// Skip dangling symlink and continue iteration on the current depth level
ec = error_code();
}
#if defined(BOOST_POSIX_API) && defined(BOOST_FILESYSTEM_HAS_FDOPENDIR_NOFOLLOW) && defined(BOOST_FILESYSTEM_HAS_POSIX_AT_APIS)
directory_iterator const& dir_it = imp->m_stack.back();
if (filesystem::type_present(dir_it->m_symlink_status))
{
symlink_ft = dir_it->m_symlink_status.type();
}
else
{
parentdir_fd = dir_itr_fd(*dir_it.m_imp, ec);
if (ec)
return result;

dir_it_filename = detail::path_algorithms::filename_v4(dir_it->path());

symlink_ft = detail::symlink_status_impl(dir_it_filename, &ec, parentdir_fd).type();
if (ec)
return result;
}

return result;
}
#else
symlink_ft = imp->m_stack.back()->symlink_file_type(ec);
if (ec)
return result;
#endif
}

if (ft != directory_file)
return result;
// Logic for following predicate was contributed by Daniel Aarno to handle cyclic
// symlinks correctly and efficiently, fixing ticket #5652.
// if (((m_options & directory_options::follow_directory_symlink) == directory_options::follow_directory_symlink
// || !is_symlink(m_stack.back()->symlink_status()))
// && is_directory(m_stack.back()->status())) ...
// The predicate code has since been rewritten to pass error_code arguments,
// per ticket #5653.

if (BOOST_UNLIKELY((imp->m_stack.size() - 1u) >= static_cast< std::size_t >((std::numeric_limits< int >::max)())))
{
// We cannot let depth to overflow
ec = make_error_code(system::errc::value_too_large);
// When depth overflow happens, avoid popping the current directory iterator
// and attempt to continue iteration on the current depth.
result = keep_depth;
return result;
if ((imp->m_options & directory_options::follow_directory_symlink) != directory_options::none || symlink_ft != symlink_file)
{
directory_iterator const& dir_it = imp->m_stack.back();

#if defined(BOOST_POSIX_API) && defined(BOOST_FILESYSTEM_HAS_FDOPENDIR_NOFOLLOW) && defined(BOOST_FILESYSTEM_HAS_POSIX_AT_APIS)
if (parentdir_fd < 0)
{
parentdir_fd = dir_itr_fd(*dir_it.m_imp, ec);
if (ec)
return result;

dir_it_filename = detail::path_algorithms::filename_v4(dir_it->path());
}

file_type ft = status_error;
if (filesystem::type_present(dir_it->m_status))
ft = dir_it->m_status.type();
else
ft = detail::status_impl(dir_it_filename, &ec, parentdir_fd).type();
#else
file_type ft = dir_it->file_type(ec);
#endif
if (BOOST_UNLIKELY(!!ec))
{
if (ec == make_error_condition(system::errc::no_such_file_or_directory) && symlink_ft == symlink_file &&
(imp->m_options & (directory_options::follow_directory_symlink | directory_options::skip_dangling_symlinks)) == (directory_options::follow_directory_symlink | directory_options::skip_dangling_symlinks))
{
// Skip dangling symlink and continue iteration on the current depth level
ec = system::error_code();
}

return result;
}

if (ft != directory_file)
return result;

if (BOOST_UNLIKELY((imp->m_stack.size() - 1u) >= static_cast< std::size_t >((std::numeric_limits< int >::max)())))
{
// We cannot let depth to overflow
ec = make_error_code(system::errc::value_too_large);
// When depth overflow happens, avoid popping the current directory iterator
// and attempt to continue iteration on the current depth.
result = keep_depth;
return result;
}

#if defined(BOOST_POSIX_API) && defined(BOOST_FILESYSTEM_HAS_FDOPENDIR_NOFOLLOW) && defined(BOOST_FILESYSTEM_HAS_POSIX_AT_APIS)
detail::directory_iterator_params params;
params.basedir = dir_it->path().parent_path();
params.basedir_fd = parentdir_fd;
params.iterator_fd = -1;
directory_iterator next;
detail::directory_iterator_construct(next, dir_it_filename, imp->m_options, &params, &ec);
#else
directory_iterator next(dir_it->path(), imp->m_options, ec);
#endif
if (!ec && next != directory_iterator())
{
imp->m_stack.push_back(std::move(next)); // may throw
return directory_pushed;
}
}
}

directory_iterator next(imp->m_stack.back()->path(), imp->m_options, ec);
if (!ec && next != directory_iterator())
catch (std::bad_alloc&)
{
imp->m_stack.push_back(std::move(next)); // may throw
return directory_pushed;
ec = make_error_code(system::errc::not_enough_memory);
}
}
}
catch (std::bad_alloc&)
{
ec = make_error_code(system::errc::not_enough_memory);
}

return result;
}

} // namespace
return result;
}
};

BOOST_FILESYSTEM_DECL
void recursive_directory_iterator_increment(recursive_directory_iterator& it, system::error_code* ec)
{
BOOST_ASSERT_MSG(!it.is_end(), "increment() on end recursive_directory_iterator");
detail::recur_dir_itr_imp* const imp = it.m_imp.get();

Expand All @@ -1353,7 +1438,7 @@ void recursive_directory_iterator_increment(recursive_directory_iterator& it, sy
system::error_code local_ec;

// if various conditions are met, push a directory_iterator into the iterator stack
push_directory_result push_result = recursive_directory_iterator_push_directory(imp, local_ec);
push_directory_result push_result = local::push_directory(imp, local_ec);
if (push_result == directory_pushed)
return;

Expand All @@ -1373,7 +1458,7 @@ void recursive_directory_iterator_increment(recursive_directory_iterator& it, sy
system::error_code increment_ec;
directory_iterator& dir_it = imp->m_stack.back();
detail::directory_iterator_increment(dir_it, &increment_ec);
if (!increment_ec && dir_it != directory_iterator())
if (!increment_ec && !dir_it.is_end())
goto on_error_return;
}

Expand Down Expand Up @@ -1407,7 +1492,7 @@ void recursive_directory_iterator_increment(recursive_directory_iterator& it, sy
if (BOOST_UNLIKELY(!!local_ec))
goto on_error;

if (dir_it != directory_iterator())
if (!dir_it.is_end())
break;

imp->m_stack.pop_back();
Expand Down
Loading

0 comments on commit 7eece00

Please sign in to comment.