Alexandria  2.27.0
SDC-CH common library for the Euclid project
FileSystemProvider.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2012-2022 Euclid Science Ground Segment
3  *
4  * This library is free software; you can redistribute it and/or modify it under
5  * the terms of the GNU Lesser General Public License as published by the Free
6  * Software Foundation; either version 3.0 of the License, or (at your option)
7  * any later version.
8  *
9  * This library is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
11  * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
12  * details.
13  *
14  * You should have received a copy of the GNU Lesser General Public License
15  * along with this library; if not, write to the Free Software Foundation, Inc.,
16  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
28 #include "ElementsKernel/Logging.h"
29 #include "StringFunctions.h"
30 #include <boost/algorithm/string.hpp>
31 #include <boost/filesystem.hpp>
32 #include <fstream>
33 #include <set>
34 #include <string>
35 #include <unordered_set>
36 
37 namespace fs = boost::filesystem;
38 
39 namespace Euclid {
40 namespace XYDataset {
41 
42 static Elements::Logging logger = Elements::Logging::getLogger("FileSystemProvider");
43 
52 static std::vector<fs::path> getOrder(const fs::path& dir) {
53  std::vector<fs::path> result{};
54 
55  // First add the files in the order.txt
56  auto order_file = dir / "order.txt";
57  std::unordered_set<std::string> ordered_names{};
58  if (fs::exists(order_file)) {
59  std::ifstream in{order_file.c_str()};
60  while (in) {
61  std::string line;
62  getline(in, line);
63  size_t comment_pos = line.find('#');
64  if (comment_pos != std::string::npos) {
65  line = line.substr(0, comment_pos);
66  }
67  boost::trim(line);
68  if (!line.empty()) {
69  auto name = dir / line;
70  if (fs::exists(name)) {
71  result.emplace_back(name);
72  ordered_names.emplace(line);
73  } else {
74  logger.warn() << "Unknown name " << line << " in order.txt of " << dir << " directory";
75  }
76  }
77  }
78  }
79 
80  // Now we add any other files in the directory, which were not in the order.txt
81  // file. We use a set in order to avoid sorting problem between platforms.
82  std::set<fs::path> remaining_files{};
83  for (fs::directory_iterator iter{dir}; iter != fs::directory_iterator{}; ++iter) {
84  if (ordered_names.count(iter->path().filename().string()) == 0) {
85  remaining_files.emplace(*iter);
86  }
87  }
88 
89  // Put the remaining files into the result vector
90  std::copy(remaining_files.begin(), remaining_files.end(), std::back_inserter(result));
91  return result;
92 }
93 
95  std::vector<fs::path> result{};
96  auto ordered_contents = getOrder(dir);
97  for (auto& name : ordered_contents) {
98  if (fs::is_directory(name)) {
99  auto sub_dir_contents = getRecursiveDirectoryContents(name);
100  result.insert(result.end(), sub_dir_contents.begin(), sub_dir_contents.end());
101  } else {
102  result.emplace_back(name);
103  }
104  }
105  return result;
106 }
107 
108 //-----------------------------------------------------------------------------
109 // Constructor
110 //-----------------------------------------------------------------------------
111 
113  : XYDatasetProvider(), m_root_path(root_path), m_parser(std::move(parser)) {
114 
115  std::vector<std::string> string_vector{};
116 
117  // Make sure the root path finishes with a "/" and only one
119 
120  // Convert path to boost filesytem object
121  fs::path fspath(m_root_path);
122  if (!fs::exists(fspath)) {
123  throw Elements::Exception() << "From FileSystemProvider: root path not found : " << fspath;
124  }
125 
126  // Get all files below the root directory
127  if (fs::is_directory(fspath)) {
128  auto dir_contents = getRecursiveDirectoryContents(fspath);
129  for (const auto& file : dir_contents) {
130  if (fs::is_regular_file(file) && m_parser->isDatasetFile(file.string())) {
131  std::string dataset_name = m_parser->getName(file.string());
132  // Remove empty dataset name
133  if (dataset_name.empty()) {
134  continue;
135  }
136  // Remove the root part
137  std::string str = file.string();
138  str = str.substr(m_root_path.length(), str.length());
139  // Split by the character '/'
140  std::vector<std::string> groups{};
141  boost::split(groups, str, boost::is_any_of("/"));
142  // The last string is the file name, so we remove it
143  groups.pop_back();
144  QualifiedName qualified_name{groups, dataset_name};
145  // Fill up a map
146  auto ret = m_name_file_map.insert(make_pair(qualified_name, file.string()));
147  m_order_names.push_back(qualified_name);
148  // Check for unique record
149  if (!ret.second) {
150  throw Elements::Exception() << "Qualified name can not be inserted "
151  << "in the map. Qualify name : " << qualified_name.qualifiedName()
152  << " Path :" << file.string();
153  }
154  }
155  }
156  } else {
157  throw Elements::Exception() << " Root path : " << fspath.string() << " is not a directory!";
158  }
159 }
160 
161 //-----------------------------------------------------------------------------
162 // listContents function
163 //-----------------------------------------------------------------------------
164 
166 
167  std::string my_group = group;
168  // Make sure the group finishes with a "/" and only one
169  while (!my_group.empty() && my_group.back() == '/') {
170  my_group.pop_back();
171  }
172  // Make sure the group do not start with a "/"
173  size_t pos = my_group.find_first_not_of("/");
174  if (!my_group.empty() && pos != 0) {
175  my_group = my_group.substr(pos);
176  }
177  if (!my_group.empty()) {
178  my_group.push_back('/');
179  }
180 
181  std::vector<QualifiedName> qualified_name_vector{};
182 
183  // Fill up vector with qualified name from the map
184  // Insert all qualified name where path contains the group name at the
185  // first position
186  for (const auto& qualified_name : m_order_names) {
187  if (boost::starts_with(qualified_name.qualifiedName(), my_group)) {
188  qualified_name_vector.push_back(qualified_name);
189  }
190  } // Eof for
191 
192  return qualified_name_vector;
193 }
194 
195 //-----------------------------------------------------------------------------
196 // getDataset function
197 //-----------------------------------------------------------------------------
198 
200 
201  auto it = m_name_file_map.find(qualified_name);
202  return (it != m_name_file_map.end()) ? m_parser->getDataset(it->second) : nullptr;
203 }
204 
206  auto it = m_name_file_map.find(qualified_name);
207  return (it != m_name_file_map.end()) ? m_parser->getParameter(it->second, key_word) : "";
208 }
209 
210 } /* namespace XYDataset */
211 } // end of namespace Euclid
T back(T... args)
T back_inserter(T... args)
static Logging getLogger(const std::string &name="")
void warn(const std::string &logMessage)
std::vector< QualifiedName > m_order_names
std::vector< QualifiedName > listContents(const std::string &group) override
List all files which belong to a group.
std::string getParameter(const QualifiedName &qualified_name, const std::string &key_word) override
FileSystemProvider(const std::string &root_path, std::unique_ptr< FileParser > parser)
constructor The FileSystemProvider handles files in a directory tree.
std::unique_ptr< FileParser > m_parser
std::unique_ptr< XYDataset > getDataset(const QualifiedName &qualified_name) override
Get a dataset corresponding to an unique qualified name.
std::map< QualifiedName, std::string > m_name_file_map
Represents a name qualified with a set of groups.
Definition: QualifiedName.h:66
This interface class provides the dataset following a qualified name object.
T copy(T... args)
T emplace(T... args)
T empty(T... args)
T find_first_not_of(T... args)
T find(T... args)
ELEMENTS_API auto split(Args &&... args) -> decltype(splitPath(std::forward< Args >(args)...))
static Elements::Logging logger
static std::vector< fs::path > getRecursiveDirectoryContents(const fs::path &dir)
static std::vector< fs::path > getOrder(const fs::path &dir)
std::string checkEndSlashes(const std::string &input_str)
STL namespace.
T pop_back(T... args)
T push_back(T... args)
T length(T... args)
T substr(T... args)