Program Listing for File data_helper.h

↰ Return to documentation for file (include/runtime/include/dataset/data_helper.h)

#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_

#include <sys/stat.h>

#if (defined(_WIN32) || defined(_WIN64)) && defined(_MSC_VER)
#define F_OK 0
#include <io.h>
#else
#include <unistd.h>
#endif

#include <fstream>
#include <iostream>
#include <map>
#include <memory>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>

#include "include/api/dual_abi_helper.h"
#include "include/api/status.h"

namespace mindspore {
namespace dataset {
class DATASET_API DataHelper {
 public:
  DataHelper() = default;

  ~DataHelper() = default;

  Status CreateAlbum(const std::string &in_dir, const std::string &out_dir) {
    return CreateAlbumIF(StringToChar(in_dir), StringToChar(out_dir));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<std::string> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), VectorStringToChar(value), StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<bool> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int8_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint8_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int16_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint16_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int32_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint32_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int64_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint64_t> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<float> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<double> &value,
                     const std::string &out_file = "") {
    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const std::string &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), StringToChar(value), StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const bool &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const int8_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const uint8_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const int16_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const uint16_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const int32_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const uint32_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const int64_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const uint64_t &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const float &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  Status UpdateValue(const std::string &in_file, const std::string &key, const double &value,
                     const std::string &out_file = "") {
    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
  }

  template <typename T>
  Status WriteBinFile(const std::string &in_file, const std::vector<T> &data) {
    std::ofstream ofs(in_file, std::ios::binary | std::ios::out);
    if (!ofs.is_open()) {
      return Status(kMDUnexpectedError, "Failed to open file: " + in_file);
    }
    size_t length = data.size();
    if (length == 0) {
      ofs.close();
      return Status(kMDUnexpectedError, "Input data is empty.");
    }
    (void)ofs.write(reinterpret_cast<const char *>(&data[0]), static_cast<std::streamsize>(length * sizeof(T)));
    if (!ofs.good()) {
      ofs.close();
      return Status(kMDUnexpectedError, "Failed to write file: " + in_file);
    }
    ofs.close();

    // Change the file mode
    if (access(in_file.c_str(), F_OK) == -1) {
      return Status(kMDUnexpectedError, "Couldn't access the file " + in_file);
    }
    try {
      if (chmod(in_file.c_str(), S_IRUSR | S_IWUSR) != 0) {
        return Status(kMDUnexpectedError, "Change file " + in_file + " mode fail.");
      }
    } catch (std::exception &e) {
      return Status(kMDUnexpectedError, "File " + in_file + " change mode failed! May be not exist.");
    }

    return Status::OK();
  }

  template <typename T>
  Status WriteBinFile(const std::string &in_file, T *data, size_t length) {
    if (data == nullptr) {
      return Status(kMDUnexpectedError, "Input data can not be nullptr.");
    }
    std::ofstream ofs(in_file, std::ios::binary | std::ios::out);
    if (!ofs.is_open()) {
      return Status(kMDUnexpectedError, "Failed to open file: " + in_file);
    }
    (void)ofs.write(reinterpret_cast<const char *>(data), static_cast<std::streamsize>(length * sizeof(T)));
    if (!ofs.good()) {
      ofs.close();
      return Status(kMDUnexpectedError, "Failed to write file: " + in_file);
    }
    ofs.close();

    // Change the file mode
    if (access(in_file.c_str(), F_OK) == -1) {
      return Status(kMDUnexpectedError, "Couldn't access the file " + in_file);
    }
    try {
      if (chmod(in_file.c_str(), S_IRUSR | S_IWUSR) != 0) {
        return Status(kMDUnexpectedError, "Change file " + in_file + " mode fail.");
      }
    } catch (std::exception &e) {
      return Status(kMDUnexpectedError, "File " + in_file + " change mode failed! May be not exist.");
    }

    return Status::OK();
  }

  size_t DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr, const size_t &buffer_size);

  Status RemoveKey(const std::string &in_file, const std::string &key, const std::string &out_file = "") {
    return RemoveKeyIF(StringToChar(in_file), StringToChar(key), StringToChar(out_file));
  }

  void Print(std::ostream &out) const;

  friend std::ostream &operator<<(std::ostream &out, const DataHelper &dh) {
    dh.Print(out);
    return out;
  }

 private:
  // Helper function for dual ABI support
  Status CreateAlbumIF(const std::vector<char> &in_dir, const std::vector<char> &out_dir);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<std::vector<char>> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<bool> &value,
                       const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<int8_t> &value,
                       const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<uint8_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<int16_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<uint16_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<int32_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<uint32_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<int64_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
                       const std::vector<uint64_t> &value, const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<float> &value,
                       const std::vector<char> &out_file);
  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<double> &value,
                       const std::vector<char> &out_file);

  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<char> &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const bool &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int8_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint8_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int16_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint16_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int32_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint32_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int64_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint64_t &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const float &value,
                       const std::vector<char> &out_file);
  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const double &value,
                       const std::vector<char> &out_file);
  Status RemoveKeyIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<char> &out_file);
};
}  // namespace dataset
}  // namespace mindspore
#endif  // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASET_DATA_HELPER_H_