Caffe2 - C++ API
A deep learning, cross platform ML framework
file_store_handler.cc
1 
17 #include "file_store_handler_op.h"
18 
19 #include <errno.h>
20 #include <fcntl.h>
21 #include <limits.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <sys/stat.h>
25 
26 #include <array>
27 #include <chrono>
28 #include <iostream>
29 #include <thread>
30 
31 #if defined(_MSC_VER)
32 #include <direct.h> // for _mkdir
33 #endif
34 
35 #include "caffe2/utils/murmur_hash3.h"
36 
37 namespace caffe2 {
38 
39 static std::string encodeName(const std::string& name) {
40  std::array<uint64_t, 2> out;
41  MurmurHash3_x64_128(name.data(), name.size(), 0xcafef00d, out.data());
42 
43  // Size is 33 to have space for final NUL
44  std::array<char, 33> buf;
45  for (int i = 0; i < 16; i++) {
46  snprintf(&buf[i * 2], buf.size() - (i * 2), "%02x", ((char*)out.data())[i]);
47  }
48 
49  // Return everything but the final NUL
50  return std::string(buf.data(), buf.size() - 1);
51 }
52 
53 FileStoreHandler::FileStoreHandler(
54  const std::string& path,
55  const std::string& prefix) {
56  basePath_ = realPath(path);
57  if (!prefix.empty()) {
58  basePath_ = basePath_ + "/" + encodeName(prefix);
59  }
60 #if defined(_MSC_VER)
61  auto ret = _mkdir(basePath_.c_str());
62 #else
63  auto ret = mkdir(basePath_.c_str(), 0777);
64 #endif // defined(_MSC_VER)
65  if (ret == -1) {
66  CHECK_EQ(errno, EEXIST) << "mkdir: " << strerror(errno);
67  }
68 }
69 
70 FileStoreHandler::~FileStoreHandler() {}
71 
72 std::string FileStoreHandler::realPath(const std::string& path) {
73 #if defined(_MSC_VER)
74  std::array<char, _MAX_PATH> buf;
75  auto ret = _fullpath(buf.data(), path.c_str(), buf.size());
76 #else
77  std::array<char, PATH_MAX> buf;
78  auto ret = realpath(path.c_str(), buf.data());
79 #endif
80  CHECK_EQ(buf.data(), ret) << "realpath: " << strerror(errno);
81  return std::string(buf.data());
82 }
83 
84 std::string FileStoreHandler::tmpPath(const std::string& name) {
85  return basePath_ + "/." + encodeName(name);
86 }
87 
88 std::string FileStoreHandler::objectPath(const std::string& name) {
89  return basePath_ + "/" + encodeName(name);
90 }
91 
92 void FileStoreHandler::set(const std::string& name, const std::string& data) {
93  auto tmp = tmpPath(name);
94  auto path = objectPath(name);
95 
96  {
97  std::ofstream ofs(tmp.c_str(), std::ios::out | std::ios::trunc);
98  if (!ofs.is_open()) {
99  CAFFE_ENFORCE(
100  false, "File cannot be created: ", tmp, " (", ofs.rdstate(), ")");
101  }
102  ofs << data;
103  }
104 
105  // Atomically movve result to final location
106  auto rv = rename(tmp.c_str(), path.c_str());
107  CAFFE_ENFORCE_EQ(rv, 0, "rename: ", strerror(errno));
108 }
109 
110 std::string FileStoreHandler::get(const std::string& name) {
111  auto path = objectPath(name);
112  std::string result;
113 
114  // Block until key is set
115  wait({name});
116 
117  std::ifstream ifs(path.c_str(), std::ios::in);
118  if (!ifs) {
119  CAFFE_ENFORCE(
120  false, "File cannot be opened: ", path, " (", ifs.rdstate(), ")");
121  }
122  ifs.seekg(0, std::ios::end);
123  size_t n = ifs.tellg();
124  result.resize(n);
125  ifs.seekg(0);
126  ifs.read(&result[0], n);
127  return result;
128 }
129 
130 int64_t FileStoreHandler::add(
131  const std::string& /* unused */,
132  int64_t /* unused */) {
133  CHECK(false) << "add not implemented for FileStoreHandler";
134  return 0;
135 }
136 
137 bool FileStoreHandler::check(const std::vector<std::string>& names) {
138  std::vector<std::string> paths;
139  for (const auto& name : names) {
140  paths.push_back(objectPath(name));
141  }
142 
143  for (const auto& path : paths) {
144  int fd = open(path.c_str(), O_RDONLY);
145  if (fd == -1) {
146  // Only deal with files that don't exist.
147  // Anything else is a problem.
148  CHECK_EQ(errno, ENOENT);
149 
150  // One of the paths doesn't exist; return early
151  return false;
152  }
153 
154  close(fd);
155  }
156 
157  return true;
158 }
159 
160 void FileStoreHandler::wait(
161  const std::vector<std::string>& names,
162  const std::chrono::milliseconds& timeout) {
163  // Not using inotify because it doesn't work on many
164  // shared filesystems (such as NFS).
165  const auto start = std::chrono::steady_clock::now();
166  while (!check(names)) {
167  const auto elapsed = std::chrono::duration_cast<std::chrono::seconds>(
168  std::chrono::steady_clock::now() - start);
169  if (timeout != kNoTimeout && elapsed > timeout) {
170  STORE_HANDLER_TIMEOUT("Wait timeout for name(s): ", Join(" ", names));
171  }
172  /* sleep override */
173  std::this_thread::sleep_for(std::chrono::milliseconds(10));
174  }
175 }
176 }
Copyright (c) 2016-present, Facebook, Inc.