Make FP16 code more reusable.

This allows writing FP16-independent code without
breaking the current code.

This is a non-invasive subset of nnstreamer#2403 to
address the concern of nnstreamer#2403 (comment)

I believe nntrainer can use this during the refactoring so that we
do not introduce additional code divergences.

Signed-off-by: MyungJoo Ham <[email protected]>

api/ccapi/include/tensor_dim.h

@@ -9,6 +9,7 @@
  * @see    https://github.com/nnstreamer/nntrainer
  * @author Jijoong Moon <[email protected]>
  * @bug    No known bugs except for NYI items
+ * @todo   Move FP16 macros to somewhere common enough.
  *
  */
 
@@ -20,6 +21,7 @@
 #include <iosfwd>
 
 #include <bitset>
+#include <cstdint>
 #include <vector>
 
 #ifdef ENABLE_FP16
@@ -28,8 +30,56 @@
 #else
 #define _FP16 _Float16
 #endif
+#else /* !ENABLE_FP16 */
+/* Keep the FP16 programming interface, but don't allow using it in run-time */
+#define _FP16 uint16_t
 #endif
 
+/**
+ * @brief Check if fp16 is enabled. Let's not use #if/#endif for FP16 elsewhere
+ * @todo Move to a proper header file!
+ */
+static inline bool is_fp16_enabled() {
+/** @todo if this becomes runtime conditionals, use likely/unlikely */
+#ifdef ENABLE_FP16
+  return true;
+#else
+  return false;
+#endif
+}
+
+#ifdef ENABLE_FP16
+#define THROW_UNLESS_FP16_ENABLED (void)0
+#define FP16_REQUIRED(...) \
+  do {                     \
+    __VA_ARGS;             \
+  } while (0)
+#else
+#define THROW_UNLESS_FP16_ENABLED                                       \
+  do {                                                                  \
+    throw std::runtime_error("The data type 'fp16' is not supported."); \
+  } while (0)
+#define FP16_REQUIRED(...)                                              \
+  do {                                                                  \
+    throw std::runtime_error("The data type 'fp16' is not supported."); \
+  } while (0)
+#endif
+/** If FP16-enable becomes dynamic, apply the following code.
+#define THROW_UNLESS_FP16_ENABLED                                         \
+  do {                                                                    \
+    if (unlikely(!is_fp16_enabled()))                                     \
+      throw std::runtime_error("The data type 'fp16' is not supported."); \
+  } while (0)
+#define FP16_REQUIRED(...) \
+  do { \
+    if (likely(is_fp16_enabled())) { \
+      __VA_ARGS; \
+    } else { \
+      throw std::runtime_error("The data type 'fp16' is not supported."); \
+    } \
+  } while (0)
+*/
+
 namespace ml {
 namespace train {
 

nntrainer/layers/acti_func.h

@@ -54,6 +54,9 @@ class ActiFunc {
   template <typename T = float> void setActiFunc(ActivationType acti_type) {
     activation_type = acti_type;
 
+    if (typeid(T) == typeid(_FP16))
+      THROW_UNLESS_FP16_ENABLED;
+
     switch (acti_type) {
     case ActivationType::ACT_TANH:
       this->setActivation<T>(tanhFloat<T>, tanhPrime<T>);

nntrainer/tensor/half_tensor.cpp

@@ -19,17 +19,21 @@
 namespace nntrainer {
 
 HalfTensor::HalfTensor(std::string name_, Tformat fm) :
-  TensorBase(name_, fm, Tdatatype::FP16) {}
+  TensorBase(name_, fm, Tdatatype::FP16) {
+  THROW_UNLESS_FP16_ENABLED;
+}
 
 HalfTensor::HalfTensor(const TensorDim &d, bool alloc_now, Initializer init,
                        std::string name) :
   TensorBase(d, alloc_now, init, name) {
+  THROW_UNLESS_FP16_ENABLED;
   if (alloc_now)
     allocate();
 }
 
 HalfTensor::HalfTensor(const TensorDim &d, const void *buf) :
   HalfTensor(d, true) {
+  THROW_UNLESS_FP16_ENABLED;
   if (d.getDataLen() != 0) {
     if (buf != nullptr)
       copy(buf);
@@ -40,6 +44,7 @@ HalfTensor::HalfTensor(
   std::vector<std::vector<std::vector<std::vector<_FP16>>>> const &d,
   Tformat fm) {
 
+  THROW_UNLESS_FP16_ENABLED;
   if (d.empty() || d[0].empty() || d[0][0].empty() || d[0][0][0].empty()) {
     throw std::out_of_range(
       "[Tensor] trying to initialize HalfTensor from empty vector");
@@ -89,6 +94,7 @@ HalfTensor::HalfTensor(
 }
 
 bool HalfTensor::operator==(const HalfTensor &rhs) const {
+  THROW_UNLESS_FP16_ENABLED;
   const _FP16 *_data = (_FP16 *)getData();
   const _FP16 *_rdata = (_FP16 *)rhs.getData();
   for (size_t i = 0; i < size(); ++i) {
@@ -102,6 +108,7 @@ bool HalfTensor::operator==(const HalfTensor &rhs) const {
 
 /// @todo support allocation by src_tensor
 void HalfTensor::allocate() {
+  THROW_UNLESS_FP16_ENABLED;
   if (empty() || data)
     /// already allocated
     return;

nntrainer/tensor/tensor_base.h

@@ -205,22 +205,21 @@ class TensorBase {
   virtual void print(std::ostream &out) const = 0;
 
   /**
-   * @copydoc TensorV2::apply(std::function<T(T)> f, TensorV2 &output)
+   * @copydoc TensorV2::apply(std::function<_FP16(_FP16)> f, TensorV2 &output)
    */
-  virtual TensorV2 &apply(std::function<float(float)> f,
+  virtual TensorV2 &apply(std::function<_FP16(_FP16)> f,
                           TensorV2 &output) const {
+    THROW_UNLESS_FP16_ENABLED;
     return output;
   }
 
-#ifdef ENABLE_FP16
   /**
    * @copydoc TensorV2::apply(std::function<T(T)> f, TensorV2 &output)
    */
-  virtual TensorV2 &apply(std::function<_FP16(_FP16)> f,
+  virtual TensorV2 &apply(std::function<float(float)> f,
                           TensorV2 &output) const {
     return output;
   }
-#endif
 
   /**
    * @brief     put data of Tensor

nntrainer/tensor/tensor_dim.cpp

@@ -150,11 +150,7 @@ TensorDim &TensorDim::operator=(TensorDim &&rhs) noexcept {
 uint TensorDim::getDataTypeSize() const {
   switch (t_type.data_type) {
   case TensorDim::DataType::FP16:
-#ifdef ENABLE_FP16
     return sizeof(_FP16);
-#else
-    return 2;
-#endif
   case TensorDim::DataType::FP32:
     return sizeof(float);
   case TensorDim::DataType::QINT8:

nntrainer/tensor/tensor_v2.h

@@ -405,6 +405,20 @@ class TensorV2 {
    */
   void initialize(Initializer init);
 
+  /**
+   * @brief Apply instantly to the element for _FP16
+   * @param[in] *function function pointer applied
+   * @return int ML_ERROR_NONE if successful
+   * @throws runtime_error if _FP16 is not supported.
+   */
+  int apply_i(std::function<_FP16(_FP16)> f) {
+    THROW_UNLESS_FP16_ENABLED;
+    TensorV2 result = *this;
+    apply<_FP16>(f, result);
+
+    return ML_ERROR_NONE;
+  };
+
   /**
    * @brief Apply instantly to the element
    * @param[in] *function function pointer applied