Merge pull request #1351 from bradh/mini_heif_2024-10-29

mini: implement HEVC reading

libheif/box.cc

@@ -460,6 +460,10 @@ Error Box::read(BitstreamRange& range, std::shared_ptr<Box>* result, const heif_
       box = std::make_shared<Box_ftyp>();
       break;
 
+    case fourcc("free"):
+      box = std::make_shared<Box_free>();
+      break;
+
     case fourcc("meta"):
       box = std::make_shared<Box_meta>();
       break;
@@ -1147,6 +1151,29 @@ Error Box_ftyp::write(StreamWriter& writer) const
 }
 
 
+Error Box_free::parse(BitstreamRange& range, const heif_security_limits* limits)
+{
+  range.skip_to_end_of_box();
+  return range.get_error();
+}
+
+
+std::string Box_free::dump(Indent& indent) const
+{
+  std::ostringstream sstr;
+  sstr << BoxHeader::dump(indent);
+  return sstr.str();
+}
+
+
+Error Box_free::write(StreamWriter& writer) const
+{
+  size_t box_start = reserve_box_header_space(writer);
+  prepend_header(writer, box_start);
+  return Error::Ok;
+}
+
+
 Error Box_meta::parse(BitstreamRange& range, const heif_security_limits* limits)
 {
   parse_full_box_header(range);

libheif/box.h

@@ -412,6 +412,23 @@ class Box_ftyp : public Box
 };
 
 
+class Box_free : public Box
+{
+public:
+  Box_free()
+  {
+    set_short_type(fourcc("free"));
+  }
+
+  std::string dump(Indent&) const override;
+
+  Error write(StreamWriter& writer) const override;
+
+protected:
+  Error parse(BitstreamRange& range, const heif_security_limits*) override;
+};
+
+
 class Box_meta : public FullBox
 {
 public:

libheif/codecs/hevc_boxes.h

@@ -33,6 +33,10 @@
 
 class Box_hvcC : public Box
 {
+
+// allow access to protected parse() method
+friend class HeifFile;
+
 public:
   Box_hvcC()
   {

libheif/file.cc

@@ -27,6 +27,7 @@
 #include "image-items/jpeg.h"
 #include "image-items/vvc.h"
 #include "codecs/avif_boxes.h"
+#include "codecs/hevc_boxes.h"
 #include "codecs/uncompressed/unc_boxes.h"
 
 #include <cstdint>
@@ -299,7 +300,8 @@ static uint32_t get_item_type_for_brand(const heif_brand2 brand)
   switch(brand) {
   case heif_brand2_avif:
     return fourcc("av01");
-    // TODO: more
+  case heif_brand2_heic:
+    return fourcc("hvc1");
   default:
     return 0;
   }
@@ -437,16 +439,35 @@ Error HeifFile::parse_heif_file()
 
     m_ipco_box = std::make_shared<Box_ipco>();
 
-    // TODO: we should look this up based on the infe prop, not assume Box_av1C.
-    std::shared_ptr<Box_av1C> main_item_codec_prop = std::make_shared<Box_av1C>();
-    std::shared_ptr<StreamReader> istr = std::make_shared<StreamReader_memory>(
-        m_mini_box->get_main_item_codec_config().data(),
-        m_mini_box->get_main_item_codec_config().size(),
-        false
-    );
-    BitstreamRange codec_range(istr, m_mini_box->get_main_item_codec_config().size(), nullptr);
-    main_item_codec_prop->parse(codec_range, heif_get_global_security_limits());
-    m_ipco_box->append_child_box(main_item_codec_prop); // entry 1
+    if (m_mini_box->get_main_item_codec_config().size() != 0) {
+      std::shared_ptr<StreamReader> istr = std::make_shared<StreamReader_memory>(
+          m_mini_box->get_main_item_codec_config().data(),
+          m_mini_box->get_main_item_codec_config().size(),
+          false
+      );
+      BitstreamRange codec_range(istr, m_mini_box->get_main_item_codec_config().size(), nullptr);
+
+      std::shared_ptr<Box> main_item_codec_prop;
+      if (infe_type == fourcc("av01")) {
+        std::shared_ptr<Box_av1C> codec_prop = std::make_shared<Box_av1C>();
+        codec_prop->parse(codec_range, heif_get_global_security_limits());
+        main_item_codec_prop = std::move(codec_prop);
+      } else if (infe_type == fourcc("hvc1")) {
+        std::shared_ptr<Box_hvcC> codec_prop = std::make_shared<Box_hvcC>();
+        codec_prop->parse(codec_range, heif_get_global_security_limits());
+        main_item_codec_prop = std::move(codec_prop);
+      } else {
+        // not found
+        std::stringstream sstr;
+        sstr << "Minimised file requires infe support for " << fourcc_to_string(infe_type) << " but this is not yet supported.";
+        return Error(heif_error_Unsupported_filetype,
+                    heif_suberror_Unspecified,
+                    sstr.str());
+      }
+      m_ipco_box->append_child_box(main_item_codec_prop); // entry 1
+    } else {
+      m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 1
+    }
 
     std::shared_ptr<Box_ispe> ispe = std::make_shared<Box_ispe>();
     ispe->set_size(m_mini_box->get_width(), m_mini_box->get_height());
@@ -470,53 +491,101 @@ Error HeifFile::parse_heif_file()
     colr->set_color_profile(nclx);
     m_ipco_box->append_child_box(colr); // entry 4
 
-    std::shared_ptr<Box_colr> colr_icc = std::make_shared<Box_colr>();
-    std::shared_ptr<color_profile_raw> icc = std::make_shared<color_profile_raw>(fourcc("prof"), m_mini_box->get_icc_data());
-    colr_icc->set_color_profile(icc);
-    m_ipco_box->append_child_box(colr_icc); // entry 5
+    if (m_mini_box->get_icc_flag()) {
+      std::shared_ptr<Box_colr> colr_icc = std::make_shared<Box_colr>();
+      std::shared_ptr<color_profile_raw> icc = std::make_shared<color_profile_raw>(fourcc("prof"), m_mini_box->get_icc_data());
+      colr_icc->set_color_profile(icc);
+      m_ipco_box->append_child_box(colr_icc); // entry 5
+    } else {
+      m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 5
+    }
 
     if (m_mini_box->get_alpha_item_codec_config().size() != 0) {
-      std::shared_ptr<Box_av1C> alpha_item_codec_prop = std::make_shared<Box_av1C>();
       std::shared_ptr<StreamReader> istr = std::make_shared<StreamReader_memory>(
         m_mini_box->get_alpha_item_codec_config().data(),
         m_mini_box->get_alpha_item_codec_config().size(),
         false
       );
       BitstreamRange alpha_codec_range(istr, m_mini_box->get_alpha_item_codec_config().size(), nullptr);
-      alpha_item_codec_prop->parse(alpha_codec_range, heif_get_global_security_limits());
+      std::shared_ptr<Box> alpha_item_codec_prop;
+      if (infe_type == fourcc("av01")) {
+        std::shared_ptr<Box_av1C> codec_prop = std::make_shared<Box_av1C>();
+        codec_prop->parse(alpha_codec_range, heif_get_global_security_limits());
+        alpha_item_codec_prop = std::move(codec_prop);
+      } else if (infe_type == fourcc("hvc1")) {
+        std::shared_ptr<Box_hvcC> codec_prop = std::make_shared<Box_hvcC>();
+        codec_prop->parse(alpha_codec_range, heif_get_global_security_limits());
+        alpha_item_codec_prop = std::move(codec_prop);
+      } else {
+        // not found
+        std::stringstream sstr;
+        sstr << "Minimised file requires infe support for " << fourcc_to_string(infe_type) << " but this is not yet supported.";
+        return Error(heif_error_Unsupported_filetype,
+                    heif_suberror_Unspecified,
+                    sstr.str());
+      }
       m_ipco_box->append_child_box(alpha_item_codec_prop); // entry 6
+    } else {
+      m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 6
     }
 
     if (m_mini_box->get_alpha_item_data_size() != 0) {
       std::shared_ptr<Box_auxC> aux_type = std::make_shared<Box_auxC>();
       aux_type->set_aux_type("urn:mpeg:mpegB:cicp:systems:auxiliary:alpha");
       m_ipco_box->append_child_box(aux_type); // entry 7
+    } else {
+      m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 7
     }
 
-    // 8
+    // TODO: replace this placeholder with pixi box version 1 once that is supported
+    m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 8
+
+    if (m_mini_box->get_orientation() == 2) {
+      std::shared_ptr<Box_irot> irot = std::make_shared<Box_irot>();
+      irot->set_rotation_ccw(2 * 90);
+      m_ipco_box->append_child_box(irot); // entry 9
+    } else if ((m_mini_box->get_orientation() == 4) || (m_mini_box->get_orientation() == 6) || (m_mini_box->get_orientation() == 7)) {
+      std::shared_ptr<Box_irot> irot = std::make_shared<Box_irot>();
+      irot->set_rotation_ccw(1 * 90);
+      m_ipco_box->append_child_box(irot); // entry 9
+    } else if (m_mini_box->get_orientation() == 5) {
+      std::shared_ptr<Box_irot> irot = std::make_shared<Box_irot>();
+      irot->set_rotation_ccw(3 * 90);
+      m_ipco_box->append_child_box(irot); // entry 9
+    } else {
+      m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 9
+    }
 
-    // 9
+    if ((m_mini_box->get_orientation() == 1) || (m_mini_box->get_orientation() == 6)) {
+      std::shared_ptr<Box_imir> imir = std::make_shared<Box_imir>();
+      imir->set_mirror_direction(heif_transform_mirror_direction_horizontal);
+      m_ipco_box->append_child_box(imir); // entry 10
+    } else if ((m_mini_box->get_orientation() == 3) || (m_mini_box->get_orientation() == 4)) {
+      std::shared_ptr<Box_imir> imir = std::make_shared<Box_imir>();
+      imir->set_mirror_direction(heif_transform_mirror_direction_vertical);
+      m_ipco_box->append_child_box(imir); // entry 10
+    } else {
+      m_ipco_box->append_child_box(std::make_shared<Box_free>()); // placeholder for entry 10
+    }
 
-    // 10
     m_ipma_box = std::make_shared<Box_ipma>();
     m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(1)});
     m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{false, uint16_t(2)});
     m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{false, uint16_t(3)});
     m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(4)});
-    if (m_mini_box->get_icc_flag()) {
-      // m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(5)});
-    }
-    if (m_mini_box->get_alpha_item_data_size() > 0) {
+    m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(5)});
+    m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(9)});
+    m_ipma_box->add_property_for_item_ID(1, Box_ipma::PropertyAssociation{true, uint16_t(10)});
+
+    if (m_mini_box->get_alpha_item_data_size() != 0) {
       m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(6)});
       m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{false, uint16_t(2)});
       m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(7)});
-      // m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{false, uint16_t(8)});
-      // m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(9)});
-      // m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(10)});
+      m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{false, uint16_t(8)});
+      m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(9)});
+      m_ipma_box->add_property_for_item_ID(2, Box_ipma::PropertyAssociation{true, uint16_t(10)});
     }
-    // TODO: will need more
-
-
+    // TODO: will need more once we support HDR / gainmap representation
 
     m_iloc_box = std::make_shared<Box_iloc>();
     Box_iloc::Item main_item;

libheif/mini.h

@@ -45,6 +45,8 @@ class Box_mini : public Box
 
   uint8_t get_bit_depth() const { return m_bit_depth; }
 
+  uint8_t get_orientation() const { return m_orientation; }
+
   std::vector<uint8_t> get_main_item_codec_config() const { return m_main_item_codec_config; }
   std::vector<uint8_t> get_alpha_item_codec_config() const { return m_alpha_item_codec_config; }
   std::vector<uint8_t> get_icc_data() const { return m_icc_data; }

tests/mini_box.cc

@@ -224,3 +224,51 @@ TEST_CASE("check mini+exif+xmp version")
                         "exif_data offset: 770, size: 208\n"
                         "xmp_data offset: 978, size: 3426\n");
 }
+
+
+TEST_CASE("check heif mini")
+{
+  auto istr = std::unique_ptr<std::istream>(new std::ifstream(tests_data_directory + "/lightning_mini.heif", std::ios::binary));
+  auto reader = std::make_shared<StreamReader_istream>(std::move(istr));
+  FileLayout file;
+  Error err = file.read(reader, heif_get_global_security_limits());
+  REQUIRE(err.error_code == heif_error_Ok);
+
+  std::shared_ptr<Box_mini> mini = file.get_mini_box();
+  REQUIRE(mini->get_exif_flag() == false);
+  REQUIRE(mini->get_xmp_flag() == false);
+  REQUIRE(mini->get_bit_depth() == 8);
+  REQUIRE(mini->get_colour_primaries() == 1);
+  REQUIRE(mini->get_transfer_characteristics() == 13);
+  REQUIRE(mini->get_matrix_coefficients() == 6);
+  REQUIRE(mini->get_width() == 128);
+  REQUIRE(mini->get_height() == 128);
+  REQUIRE(mini->get_main_item_codec_config().size() == 112);
+  Indent indent;
+  std::string dumpResult = mini->dump(indent);
+  REQUIRE(dumpResult == "Box: mini -----\n"
+                        "size: 4710   (header size: 8)\n"
+                        "version: 0\n"
+                        "explicit_codec_types_flag: 0\n"
+                        "float_flag: 0\n"
+                        "full_range_flag: 1\n"
+                        "alpha_flag: 0\n"
+                        "explicit_cicp_flag: 0\n"
+                        "hdr_flag: 0\n"
+                        "icc_flag: 0\n"
+                        "exif_flag: 0\n"
+                        "xmp_flag: 0\n"
+                        "chroma_subsampling: 1\n"
+                        "orientation: 1\n"
+                        "width: 128\n"
+                        "height: 128\n"
+                        "chroma_is_horizontally_centered: 0\n"
+                        "chroma_is_vertically_centered: 0\n"
+                        "bit_depth: 8\n"
+                        "colour_primaries: 1\n"
+                        "transfer_characteristics: 13\n"
+                        "matrix_coefficients: 6\n"
+                        "main_item_code_config size: 112\n"
+                        "main_item_data offset: 144, size: 4582\n");
+}
+

tests/mini_decode.cc

@@ -57,3 +57,54 @@ TEST_CASE("check image handle size") {
   heif_context_free(context);
 }
 
+void check_image_size_heif_mini(struct heif_context *&context) {
+  heif_image_handle *handle = get_primary_image_handle(context);
+  heif_image *img = get_primary_image_ycbcr(handle, heif_chroma_444);
+
+  REQUIRE(heif_image_has_channel(img, heif_channel_Y) == 1);
+  REQUIRE(heif_image_has_channel(img, heif_channel_Cb) == 1);
+  REQUIRE(heif_image_has_channel(img, heif_channel_Cr) == 1);
+  REQUIRE(heif_image_has_channel(img, heif_channel_R) == 0);
+  REQUIRE(heif_image_has_channel(img, heif_channel_G) == 0);
+  REQUIRE(heif_image_has_channel(img, heif_channel_B) == 0);
+  REQUIRE(heif_image_has_channel(img, heif_channel_Alpha) == 0);
+  REQUIRE(heif_image_has_channel(img, heif_channel_interleaved) == 0);
+  int width = heif_image_get_primary_width(img);
+  REQUIRE(width == 128);
+  int height = heif_image_get_primary_height(img);
+  REQUIRE(height == 128);
+  width = heif_image_get_width(img, heif_channel_Y);
+  REQUIRE(width == 128);
+  height = heif_image_get_height(img, heif_channel_Y);
+  REQUIRE(height == 128);
+  width = heif_image_get_width(img, heif_channel_Cb);
+  REQUIRE(width == 128);
+  height = heif_image_get_height(img, heif_channel_Cr);
+  REQUIRE(height == 128);
+  width = heif_image_get_width(img, heif_channel_Cr);
+  REQUIRE(width == 128);
+  height = heif_image_get_height(img, heif_channel_Cr);
+  REQUIRE(height == 128);
+
+  int pixel_depth = heif_image_get_bits_per_pixel(img, heif_channel_Y);
+  REQUIRE(pixel_depth == 8);
+  pixel_depth = heif_image_get_bits_per_pixel(img, heif_channel_Cb);
+  REQUIRE(pixel_depth == 8);
+  pixel_depth = heif_image_get_bits_per_pixel(img, heif_channel_Cr);
+  REQUIRE(pixel_depth == 8);
+  int pixel_range = heif_image_get_bits_per_pixel_range(img, heif_channel_Y);
+  REQUIRE(pixel_range == 8);
+  pixel_range = heif_image_get_bits_per_pixel_range(img, heif_channel_Cb);
+  REQUIRE(pixel_range == 8);
+  pixel_range = heif_image_get_bits_per_pixel_range(img, heif_channel_Cr);
+  REQUIRE(pixel_range == 8);
+
+  heif_image_release(img);
+  heif_image_handle_release(handle);
+}
+
+TEST_CASE("check image size HEIF mini") {
+  auto context = get_context_for_test_file("lightning_mini.heif");
+  check_image_size_heif_mini(context);
+  heif_context_free(context);
+}