Remove dependencies on FP16 support status.

Most modules should NOT depend on the existnace of FP16 support.
Let a single (or minimal) entity (a single class?) know and handle it.

This is a suggestion commit that shows an example how to
refactor it. Most modules are not yet touched with this commit.

Signed-off-by: MyungJoo Ham <[email protected]>

Applications/YOLO/jni/yolo_v2_loss.cpp

@@ -178,11 +178,8 @@ calc_iou(nntrainer::Tensor &bbox1_x1, nntrainer::Tensor &bbox1_y1,
   if (type_intersection_width == ml::train::TensorDim::DataType::FP32) {
     intersection_width.apply_i<float>(nntrainer::ActiFunc::relu<float>);
   } else if (type_intersection_width == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     intersection_width.apply_i<_FP16>(nntrainer::ActiFunc::relu<_FP16>);
-#else
-    throw std::runtime_error("Not supported data type");
-#endif
   }
 
   intersection_y2.subtract(intersection_y1, intersection_height);
@@ -191,11 +188,8 @@ calc_iou(nntrainer::Tensor &bbox1_x1, nntrainer::Tensor &bbox1_y1,
   if (type_intersection_height == ml::train::TensorDim::DataType::FP32) {
     intersection_height.apply_i<float>(nntrainer::ActiFunc::relu<float>);
   } else if (type_intersection_height == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     intersection_height.apply_i<_FP16>(nntrainer::ActiFunc::relu<_FP16>);
-#else
-    throw std::runtime_error("Not supported data type");
-#endif
   }
 
   nntrainer::Tensor intersection =
@@ -238,12 +232,9 @@ std::vector<nntrainer::Tensor> calc_iou_grad(
     intersection_width_relu_prime =
       intersection_width.apply<float>(nntrainer::ActiFunc::reluPrime<float>);
   } else if (type_intersection_width == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     intersection_height_relu_prime =
       intersection_height.apply<_FP16>(nntrainer::ActiFunc::reluPrime<_FP16>);
-#else
-    throw std::runtime_error("Not supported data type");
-#endif
   }
 
   nntrainer::Tensor intersection_x2_local_grad =
@@ -539,22 +530,16 @@ void YoloV2LossLayer::forwarding(nntrainer::RunLayerContext &context,
   if (type_bbox_w_pred == ml::train::TensorDim::DataType::FP32) {
     bbox_w_pred.apply_i<float>(nntrainer::exp_util<float>);
   } else if (type_bbox_w_pred == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     bbox_w_pred.apply_i<_FP16>(nntrainer::exp_util<_FP16>);
-#else
-    throw std::runtime_error("Not supported data type");
-#endif
   }
 
   auto type_bbox_h_pred = bbox_h_pred.getDataType();
   if (type_bbox_h_pred == ml::train::TensorDim::DataType::FP32) {
     bbox_h_pred.apply_i<float>(nntrainer::exp_util<float>);
   } else if (type_bbox_h_pred == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     bbox_h_pred.apply_i<_FP16>(nntrainer::exp_util<_FP16>);
-#else
-    throw std::runtime_error("Not supported data type");
-#endif
   }
 
   sigmoid.run_fn(confidence_pred, confidence_pred);
@@ -569,23 +554,17 @@ void YoloV2LossLayer::forwarding(nntrainer::RunLayerContext &context,
   if (type_bbox_w_pred_anchor == ml::train::TensorDim::DataType::FP32) {
     bbox_w_pred_anchor.apply_i<float>(nntrainer::sqrtFloat<float>);
   } else if (type_bbox_w_pred_anchor == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     bbox_w_pred_anchor.apply_i<_FP16>(nntrainer::sqrtFloat<_FP16>);
-#else
-    throw std::runtime_error("Not supported data type");
-#endif
   }
 
   bbox_h_pred_anchor.multiply_i(anchors_h);
   auto type_bbox_h_pred_anchor = bbox_h_pred_anchor.getDataType();
   if (type_bbox_h_pred_anchor == ml::train::TensorDim::DataType::FP32) {
     bbox_h_pred_anchor.apply_i<float>(nntrainer::sqrtFloat<float>);
   } else if (type_bbox_h_pred_anchor == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     bbox_h_pred_anchor.apply_i<_FP16>(nntrainer::sqrtFloat<_FP16>);
-#else
-    throw std::runtime_error("Not supported data type");
-#endif
   }
 
   generate_ground_truth(context);
@@ -810,11 +789,8 @@ unsigned int YoloV2LossLayer::find_responsible_anchors(float bbox_ratio) {
   if (data_type == ml::train::TensorDim::DataType::FP32) {
     similarity.apply_i<float>(nntrainer::absFloat<float>);
   } else if (data_type == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     similarity.apply_i<_FP16>(nntrainer::absFloat<_FP16>);
-#else
-    throw std::runtime_error("Not supported data type");
-#endif
   }
   auto data = similarity.getData();
 

api/ccapi/include/layer.h

@@ -204,7 +204,6 @@ class Layer {
   virtual void getWeights(std::vector<float *> &weights,
                           std::vector<ml::train::TensorDim> &weights_dim) = 0;
 
-#ifdef ENABLE_FP16
   /**
    * @brief     Get weight data of the layer
    * @retval    weight data of the layer
@@ -225,7 +224,7 @@ class Layer {
   virtual void
   getFP16Weights(std::vector<_FP16 *> &weights,
                  std::vector<ml::train::TensorDim> &weights_dim) = 0;
-#endif
+
   /**
    * @brief     Set weight data of the layer
    * @note      Size of vector must be the same with number of weights.

api/ccapi/include/tensor_dim.h

@@ -21,15 +21,44 @@
 
 #include <bitset>
 #include <vector>
+#include <cstdint>
 
 #ifdef ENABLE_FP16
 #ifdef USE__FP16
 #define _FP16 __fp16
 #else
 #define _FP16 _Float16
 #endif
+#else /* !ENABLE_FP16 */
+#define _FP16 uint16_t /* Keep the FP16 programming interface, but don't allow using it in run-time */
 #endif
 
+/**
+ * @brief Check if fp16 is enabled. Let's not use #if/#endif for FP16 elsewhere
+ * @todo Move to a proper header file!
+ */
+static inline bool is_fp16_enabled ()
+{
+#ifdef ENABLE_FP16
+  return true;
+#else
+  return false;
+#endif
+}
+
+#ifdef ENABLE_FP16
+#define throw_unless_fp16_enabled do { } while(0)
+#else
+#define throw_unless_fp16_enabled do { throw std::runtime_error("The data type 'fp16' is not supported."); } while(0)
+#endif
+/** If FP16-enable becomes dynamic, apply the following code.
+#define throw_unless_fp16_enabled \
+  do { \
+    if (!is_fp16_enabled()) \
+      throw std::runtime_error("The data type 'fp16' is not supported."); \
+  } while (0)
+*/
+
 namespace ml {
 namespace train {
 

nntrainer/layers/layer_node.h

@@ -559,16 +559,13 @@ class LayerNode final : public ml::train::Layer, public GraphNode {
 
       if (getWeight(idx).getDataType() ==
           ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+        throw_unless_fp16_enabled;
         _FP16 *data = getWeight(idx).getData<_FP16>();
         float *d = new float[getWeight(idx).size()]();
         weights.emplace_back(d);
         for (unsigned int i = 0; i < getWeight(idx).size(); ++i) {
           weights[idx][i] = static_cast<float>(data[i]);
         }
-#else
-        throw std::runtime_error("enable-fp16 is not set");
-#endif
       } else {
         weights.emplace_back(getWeight(idx).getData());
       }
@@ -597,7 +594,6 @@ class LayerNode final : public ml::train::Layer, public GraphNode {
     }
     return;
   }
-#ifdef ENABLE_FP16
   /**
    * @brief     Get weight data of the layer
    * @retval    weight data of the layer
@@ -606,6 +602,8 @@ class LayerNode final : public ml::train::Layer, public GraphNode {
    * @note      layer needs to be finalized before called.
    */
   const std::vector<_FP16 *> getFP16Weights() override {
+    throw_unless_fp16_enabled;
+
     NNTR_THROW_IF(!run_context, std::runtime_error)
       << __func__ << " layer needs to be finalized first!";
 
@@ -626,6 +624,8 @@ class LayerNode final : public ml::train::Layer, public GraphNode {
    */
   void getFP16Weights(std::vector<_FP16 *> &weights,
                       std::vector<TensorDim> &weight_dim) override {
+    throw_unless_fp16_enabled;
+
     NNTR_THROW_IF(!run_context, std::runtime_error)
       << __func__ << " layer needs to be finalized first!";
 
@@ -637,7 +637,6 @@ class LayerNode final : public ml::train::Layer, public GraphNode {
     }
     return;
   }
-#endif
 
   /**
    * @brief     Set weight data of the layer

nntrainer/tensor/blas_interface.cpp

@@ -73,7 +73,6 @@
 
 namespace nntrainer {
 
-#ifdef ENABLE_FP16
 static void saxpy_FP16(const unsigned int N, const float alpha, const _FP16 *X,
                        const int incX, _FP16 *Y, const int incY) {
   if (incX < 0 or incY < 0)
@@ -415,7 +414,6 @@ void inv_sqrt_inplace(const unsigned int N, _FP16 *X) {
   }
 #endif
 }
-#endif
 
 #ifndef USE_BLAS
 static void saxpy_raw(const unsigned int N, const float alpha, const float *X,
@@ -538,11 +536,8 @@ void sscal(const unsigned int N, const float alpha, void *X, const int incX,
     sscal_raw(N, alpha, (float *)X, incX);
 #endif //  USE_BLAS
   } else if (d_type == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     sscal(N, alpha, (_FP16 *)X, incX);
-#else
-    throw std::invalid_argument("Error: enable-fp16 is not enabled");
-#endif
   }
 }
 
@@ -572,12 +567,9 @@ void saxpy(const unsigned int N, const float alpha, const void *X,
               static_cast<float *>(Y), incY);
 #endif
   } else if (d_type == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     saxpy_FP16(N, alpha, static_cast<const _FP16 *>(X), incX,
                static_cast<_FP16 *>(Y), incY);
-#else
-    throw std::invalid_argument("Error: enable-fp16 is not enabled");
-#endif
   }
 }
 
@@ -645,13 +637,10 @@ void sgemm(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA, CBLAS_TRANSPOSE TransB,
 #endif
 
   } else if (d_type == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     sgemm_FP16(
       order, TransA, TransB, M, N, K, alpha, static_cast<const _FP16 *>(A), lda,
       static_cast<const _FP16 *>(B), ldb, beta, static_cast<_FP16 *>(C), ldc);
-#else
-    throw std::invalid_argument("Error: enable-fp16 is not enabled");
-#endif
   }
 } // namespace nntrainer
 
@@ -714,11 +703,8 @@ void scopy(const unsigned int N, const void *X, const int incX, void *Y,
 #endif
 
   } else if (d_type == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     scopy_FP16(N, (_FP16 *)X, incX, (_FP16 *)Y, incY);
-#else
-    throw std::invalid_argument("Error: enable-fp16 is not enabled");
-#endif
   }
 }
 
@@ -811,13 +797,10 @@ void sgemv(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA, const unsigned int M,
                      static_cast<float *>(Y), incY);
 #endif
   } else if (d_type == ml::train::TensorDim::DataType::FP16) {
-#ifdef ENABLE_FP16
+    throw_unless_fp16_enabled;
     return sgemv_FP16(order, TransA, M, N, alpha, static_cast<const _FP16 *>(A),
                       lda, static_cast<const _FP16 *>(X), incX, beta,
                       static_cast<_FP16 *>(Y), incY);
-#else
-    throw std::invalid_argument("Error: enable-fp16 is not enabled");
-#endif
   }
 }
 

nntrainer/tensor/half_tensor.cpp

@@ -19,16 +19,18 @@
 namespace nntrainer {
 
 HalfTensor::HalfTensor(std::string name_, Tformat fm) :
-  TensorBase(name_, fm, Tdatatype::FP16) {}
+  TensorBase(name_, fm, Tdatatype::FP16) { throw_unless_fp16_enabled; }
 
 HalfTensor::HalfTensor(const TensorDim &d, bool alloc_now, Initializer init,
                        std::string name) :
   TensorBase(d, alloc_now, init, name) {
+  throw_unless_fp16_enabled;
   if (alloc_now)
     allocate();
 }
 
 HalfTensor::HalfTensor(const TensorDim &d, const void *buf) :
+  throw_unless_fp16_enabled;
   HalfTensor(d, true) {
   if (d.getDataLen() != 0) {
     if (buf != nullptr)
@@ -40,6 +42,7 @@ HalfTensor::HalfTensor(
   std::vector<std::vector<std::vector<std::vector<_FP16>>>> const &d,
   Tformat fm) {
 
+  throw_unless_fp16_enabled;
   if (d.empty() || d[0].empty() || d[0][0].empty() || d[0][0][0].empty()) {
     throw std::out_of_range(
       "[Tensor] trying to initialize HalfTensor from empty vector");
@@ -89,6 +92,7 @@ HalfTensor::HalfTensor(
 }
 
 bool HalfTensor::operator==(const HalfTensor &rhs) const {
+  throw_unless_fp16_enabled;
   const _FP16 *_data = (_FP16 *)getData();
   const _FP16 *_rdata = (_FP16 *)rhs.getData();
   for (size_t i = 0; i < size(); ++i) {
@@ -102,6 +106,7 @@ bool HalfTensor::operator==(const HalfTensor &rhs) const {
 
 /// @todo support allocation by src_tensor
 void HalfTensor::allocate() {
+  throw_unless_fp16_enabled;
   if (empty() || data)
     /// already allocated
     return;