Alexandria  2.19
Please provide a description of the project.
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
FileSystemProvider.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2012-2021 Euclid Science Ground Segment
3  *
4  * This library is free software; you can redistribute it and/or modify it under
5  * the terms of the GNU Lesser General Public License as published by the Free
6  * Software Foundation; either version 3.0 of the License, or (at your option)
7  * any later version.
8  *
9  * This library is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
11  * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
12  * details.
13  *
14  * You should have received a copy of the GNU Lesser General Public License
15  * along with this library; if not, write to the Free Software Foundation, Inc.,
16  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
28 #include "ElementsKernel/Logging.h"
29 #include "StringFunctions.h"
30 #include <boost/algorithm/string.hpp>
31 #include <boost/filesystem.hpp>
32 #include <fstream>
33 #include <set>
34 #include <string>
35 #include <unordered_set>
36 
37 namespace fs = boost::filesystem;
38 
39 namespace Euclid {
40 namespace XYDataset {
41 
42 static Elements::Logging logger = Elements::Logging::getLogger("FileSystemProvider");
43 
52 static std::vector<fs::path> getOrder(const fs::path& dir) {
53  std::vector<fs::path> result{};
54 
55  // First add the files in the order.txt
56  auto order_file = dir / "order.txt";
57  std::unordered_set<std::string> ordered_names{};
58  if (fs::exists(order_file)) {
59  std::ifstream in{order_file.c_str()};
60  while (in) {
61  std::string line;
62  getline(in, line);
63  size_t comment_pos = line.find('#');
64  if (comment_pos != std::string::npos) {
65  line = line.substr(0, comment_pos);
66  }
67  boost::trim(line);
68  if (!line.empty()) {
69  auto name = dir / line;
70  if (fs::exists(name)) {
71  result.emplace_back(name);
72  ordered_names.emplace(line);
73  } else {
74  logger.warn() << "Unknown name " << line << " in order.txt of " << dir << " directory";
75  }
76  }
77  }
78  }
79 
80  // Now we add any other files in the directory, which were not in the order.txt
81  // file. We use a set in order to avoid sorting problem between platforms.
82  std::set<fs::path> remaining_files{};
83  for (fs::directory_iterator iter{dir}; iter != fs::directory_iterator{}; ++iter) {
84  if (ordered_names.count(iter->path().filename().string()) == 0) {
85  remaining_files.emplace(*iter);
86  }
87  }
88 
89  // Put the remaining files into the result vector
90  for (auto& file : remaining_files) {
91  result.emplace_back(file);
92  }
93 
94  return result;
95 }
96 
98  std::vector<fs::path> result{};
99  auto ordered_contents = getOrder(dir);
100  for (auto& name : ordered_contents) {
101  if (fs::is_directory(name)) {
102  auto sub_dir_contents = getRecursiveDirectoryContents(name);
103  result.insert(result.end(), sub_dir_contents.begin(), sub_dir_contents.end());
104  } else {
105  result.emplace_back(name);
106  }
107  }
108  return result;
109 }
110 
111 //-----------------------------------------------------------------------------
112 // Constructor
113 //-----------------------------------------------------------------------------
114 
116  : XYDatasetProvider(), m_root_path(root_path), m_parser(std::move(parser)) {
117 
118  std::vector<std::string> string_vector{};
119 
120  // Make sure the root path finishes with a "/" and only one
122 
123  // Convert path to boost filesytem object
124  fs::path fspath(m_root_path);
125  if (!fs::exists(fspath)) {
126  throw Elements::Exception() << "From FileSystemProvider: root path not found : " << fspath;
127  }
128 
129  // Get all files below the root directory
130  if (fs::is_directory(fspath)) {
131  auto dir_contents = getRecursiveDirectoryContents(fspath);
132  for (auto& file : dir_contents) {
133  if (fs::is_regular_file(file) && m_parser->isDatasetFile(file.string())) {
134  std::string dataset_name = m_parser->getName(file.string());
135  // Remove empty dataset name
136  if (dataset_name.empty()) {
137  continue;
138  }
139  // Remove the root part
140  std::string str = file.string();
141  str = str.substr(m_root_path.length(), str.length());
142  // Split by the character '/'
143  std::vector<std::string> groups{};
144  boost::split(groups, str, boost::is_any_of("/"));
145  // The last string is the file name, so we remove it
146  groups.pop_back();
147  QualifiedName qualified_name{groups, dataset_name};
148  // Fill up a map
149  auto ret = m_name_file_map.insert(make_pair(qualified_name, file.string()));
150  m_order_names.push_back(qualified_name);
151  // Check for unique record
152  if (!ret.second) {
153  throw Elements::Exception() << "Qualified name can not be inserted "
154  << "in the map. Qualify name : " << qualified_name.qualifiedName()
155  << " Path :" << file.string();
156  }
157  }
158  }
159  } else {
160  throw Elements::Exception() << " Root path : " << fspath.string() << " is not a directory!";
161  }
162 }
163 
164 //-----------------------------------------------------------------------------
165 // listContents function
166 //-----------------------------------------------------------------------------
167 
169 
170  std::string my_group = group;
171  // Make sure the group finishes with a "/" and only one
172  while (!my_group.empty() && my_group.back() == '/') {
173  my_group.pop_back();
174  }
175  // Make sure the group do not start with a "/"
176  size_t pos = my_group.find_first_not_of("/");
177  if (!my_group.empty() && pos != 0) {
178  my_group = my_group.substr(pos);
179  }
180  if (!my_group.empty()) {
181  my_group.push_back('/');
182  }
183 
184  std::vector<QualifiedName> qualified_name_vector{};
185 
186  // Fill up vector with qualified name from the map
187  // Insert all qualified name where path contains the group name at the
188  // first position
189  for (auto qualified_name : m_order_names) {
190  if (boost::starts_with(qualified_name.qualifiedName(), my_group)) {
191  qualified_name_vector.push_back(qualified_name);
192  }
193  } // Eof for
194 
195  return (qualified_name_vector);
196 }
197 
198 //-----------------------------------------------------------------------------
199 // getDataset function
200 //-----------------------------------------------------------------------------
201 
203 
204  auto it = m_name_file_map.find(qualified_name);
205  return (it != m_name_file_map.end()) ? m_parser->getDataset(it->second) : nullptr;
206 }
207 
209  auto it = m_name_file_map.find(qualified_name);
210  return (it != m_name_file_map.end()) ? m_parser->getParameter(it->second, key_word) : "";
211 }
212 
213 } /* namespace XYDataset */
214 } // end of namespace Euclid
T empty(T...args)
ELEMENTS_API auto split(Args &&...args) -> decltype(splitPath(std::forward< Args >(args)...))
T find_first_not_of(T...args)
std::vector< QualifiedName > listContents(const std::string &group) override
List all files which belong to a group.
std::map< QualifiedName, std::string > m_name_file_map
std::string getParameter(const QualifiedName &qualified_name, const std::string &key_word) override
std::string checkEndSlashes(const std::string &input_str)
FileSystemProvider(const std::string &root_path, std::unique_ptr< FileParser > parser)
constructor The FileSystemProvider handles files in a directory tree.
static Elements::Logging logger
STL class.
T push_back(T...args)
void warn(const std::string &logMessage)
T pop_back(T...args)
std::unique_ptr< FileParser > m_parser
T find(T...args)
T length(T...args)
STL class.
STL class.
STL class.
std::vector< QualifiedName > m_order_names
T emplace(T...args)
T back(T...args)
T substr(T...args)
static std::vector< fs::path > getOrder(const fs::path &dir)
static std::vector< fs::path > getRecursiveDirectoryContents(const fs::path &dir)
This interface class provides the dataset following a qualified name object.
Represents a name qualified with a set of groups.
Definition: QualifiedName.h:66
static Logging getLogger(const std::string &name="")
STL class.
std::unique_ptr< XYDataset > getDataset(const QualifiedName &qualified_name) override
Get a dataset corresponding to an unique qualified name.