Add files via upload

README.md

@@ -0,0 +1,110 @@
+# Wheat Grains Detector
+**For Celiac Disease sufferers**
+
+<span style="display:block;text-align:center">![App in action](ezgif-1-138e5d970f8d.gif)</span>
+## Android + OpenCV + Tensorflow Object Detection API
+_**Please note:** this is a sidenotes for my own personal use rather than a detailed step-by-step instruction on how to build the project. However an experienced developer should get what's going on._
+
+## Preparing dataset
+
+`LabelImg`/`VoTT`/`LabelBox` for segmentation and annotation are all fine. `LabelImg` is finest.
+
+### Generate TFRecord
+
+```bash
+python create_pascal_tf_record_ex.py --annotations_dir dataset/train --label_map_path dataset/label_map.pbtxt --output_path train.record
+python create_pascal_tf_record_ex.py --annotations_dir dataset/val --label_map_path dataset/label_map.pbtxt --output_path val.record
+```
+
+## Training model
+
+Connect to [Colaboratory](https://colab.research.google.com) and upload _colab/Wheat_Grains_Detector.ipynb_
+
+To upload:
+* _colab/ssd_mobilenet_v2_coco.config_
+* _dataset/label_map.pbtxt_
+* _train.record_
+* _val.record_
+
+Will be downloaded:
+* _frozen.zip_
+
+## Building Android app
+
+### Install JDK
+
+```bash
+sudo apt-get install openjdk-8-jdk-headless
+```
+or
+
+Download [Java SE Development Kit 8](https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) and unpack to _`<JDK_DIR>`_. _jdk-8u181-linux-x64.tar.gz_ was used.
+```bash
+export PATH=$PATH:<JDK_DIR>/bin/
+```
+
+### Install Android SDK
+
+Download [SDK tools](https://developer.android.com/studio/) and unpack to _`<ANDROID_SDK_DIR>`_. _sdk-tools-linux-4333796.zip_ was used.
+```bash
+export ANDROID_SDK_HOME=<ANDROID_SDK_DIR>
+
+export PATH=$PATH:$ANDROID_SDK_HOME/tools/bin/
+
+sdkmanager "build-tools;28.0.2"
+export PATH=$PATH:$ANDROID_SDK_HOME/build-tools/28.0.2/
+
+sdkmanager "platforms;android-21"
+
+sudo apt-get install adb
+```
+Connect `adb` to a device.
+
+### Build OpenCV library
+
+Download [Android pack](https://opencv.org/releases.html) and unpack to _`<OPENCV_SDK_DIR>`_. _opencv-3.4.3-android-sdk.zip_ was used.
+```bash
+export OPENCV_SDK_JAVA=<OPENCV_SDK_DIR>/sdk/java/
+pushd $OPENCV_SDK_JAVA
+mkdir -p build/gen/ build/obj/
+aapt package -m -J build/gen/ -M AndroidManifest.xml -S res/ -I $ANDROID_SDK_HOME/platforms/android-21/android.jar
+aidl -obuild/gen/ src/org/opencv/engine/OpenCVEngineInterface.aidl
+```
+create _build/gen/BuildConfig.java_ with the following content:
+```java
+package org.opencv;
+
+public final class BuildConfig {
+    public static final boolean DEBUG = Boolean.parseBoolean("false");
+}
+```
+```bash
+javac -d build/obj/ -bootclasspath $ANDROID_SDK_HOME/platforms/android-21/android.jar build/gen/BuildConfig.java build/gen/org/opencv/R.java build/gen/org/opencv/*/*.java src/org/opencv/*/*.java
+aapt package -F build/opencv.jar -M AndroidManifest.xml -S res/ -I $ANDROID_SDK_HOME/platforms/android-21/android.jar build/obj/
+popd
+```
+
+### Build app
+
+```bash
+pushd android
+mkdir -p build/gen/ build/obj/ build/bin/lib/ build/bin/assets/
+aapt package -m -J build/gen/ -M AndroidManifest.xml -S $OPENCV_SDK_JAVA/res/ -S res/ -I $ANDROID_SDK_HOME/platforms/android-21/android.jar
+javac -d build/obj/ -bootclasspath $ANDROID_SDK_HOME/platforms/android-21/android.jar -classpath $OPENCV_SDK_JAVA/build/opencv.jar build/gen/com/github/failure_to_thrive/wheat_grains_detector/R.java src/com/github/failure_to_thrive/wheat_grains_detector/MainActivity.java
+dx --dex --output=build/bin/classes.dex $OPENCV_SDK_JAVA/build/opencv.jar build/obj/
+```
+copy _`<OPENCV_SDK_DIR>`/sdk/native/libs/armeabi-v7a/_ to _build/bin/lib/_  
+copy _`<OPENCV_SDK_DIR>`/sdk/native/libs/arm64-v8a/_ to _build/bin/lib/_  
+unpack _frozen.zip_ to _build/bin/assets/_
+```bash
+aapt package -F build/WheatGrainsDetector.unaligned.apk -M AndroidManifest.xml -S $OPENCV_SDK_JAVA/res/ -S res/ -I $ANDROID_SDK_HOME/platforms/android-21/android.jar build/bin/
+apksigner sign --key <key> --cert <cert> build/WheatGrainsDetector.unaligned.apk
+zipalign -p 4 build/WheatGrainsDetector.unaligned.apk build/WheatGrainsDetector.apk
+adb install build/WheatGrainsDetector.apk
+popd
+```
+```bash
+adb logcat WGD:* *:S
+```
+
+That's all!

android/AndroidManifest.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+          package="com.github.failure_to_thrive.wheat_grains_detector">
+
+    <application
+        android:label="@string/app_name"
+        android:icon="@drawable/gluten_free"
+        android:theme="@android:style/Theme.NoTitleBar.Fullscreen" >
+
+        <activity android:name="MainActivity"
+                  android:label="@string/app_name"
+                  android:screenOrientation="landscape"
+                  android:configChanges="keyboardHidden|orientation">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+    <supports-screens android:resizeable="true"
+                      android:smallScreens="true"
+                      android:normalScreens="true"
+                      android:largeScreens="true"
+                      android:anyDensity="true" />
+
+    <uses-sdk android:minSdkVersion="8" />
+
+    <uses-permission android:name="android.permission.CAMERA"/>
+
+    <uses-feature android:name="android.hardware.camera" android:required="false"/>
+    <uses-feature android:name="android.hardware.camera.autofocus" android:required="false"/>
+    <uses-feature android:name="android.hardware.camera.front" android:required="false"/>
+    <uses-feature android:name="android.hardware.camera.front.autofocus" android:required="false"/>
+<!--
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+-->
+</manifest>

android/res/layout/activity_main.xml

@@ -0,0 +1,15 @@
+<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    xmlns:opencv="http://schemas.android.com/apk/res-auto"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent" >
+
+    <org.opencv.android.JavaCameraView
+        android:layout_width="fill_parent"
+        android:layout_height="fill_parent"
+        android:visibility="visible"
+        android:id="@+id/CameraView"
+        opencv:show_fps="true"
+        opencv:camera_id="any" />
+
+</FrameLayout>

android/res/values/strings.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <string name="app_name">Wheat Grains Detector</string>
+</resources>

android/src/com/github/failure_to_thrive/wheat_grains_detector/MainActivity.java

@@ -0,0 +1,198 @@
+package com.github.failure_to_thrive.wheat_grains_detector;
+
+import org.opencv.android.BaseLoaderCallback;
+import org.opencv.android.CameraBridgeViewBase.CvCameraViewFrame;
+import org.opencv.android.LoaderCallbackInterface;
+import org.opencv.android.OpenCVLoader;
+import org.opencv.core.Mat;
+import org.opencv.android.CameraBridgeViewBase;
+import org.opencv.android.CameraBridgeViewBase.CvCameraViewListener2;
+import org.opencv.core.Core;
+import org.opencv.core.Point;
+import org.opencv.core.Scalar;
+import org.opencv.core.Size;
+import org.opencv.dnn.Net;
+import org.opencv.dnn.Dnn;
+import org.opencv.imgproc.Imgproc;
+
+import android.app.Activity;
+import android.os.Bundle;
+import android.util.Log;
+import android.view.SurfaceView;
+import android.view.WindowManager;
+import android.content.res.AssetManager;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+public class MainActivity extends Activity implements CvCameraViewListener2 {
+    private static final String TAG = "WGD";
+
+    private CameraBridgeViewBase mOpenCvCameraView;
+    private Net net;
+//    private String path;
+
+    private BaseLoaderCallback mLoaderCallback = new BaseLoaderCallback(this) {
+        @Override
+        public void onManagerConnected(int status) {
+            switch (status) {
+                case LoaderCallbackInterface.SUCCESS:
+                {
+                    Log.i(TAG, "OpenCV loaded successfully");
+                    mOpenCvCameraView.enableView();
+                } break;
+                default:
+                {
+                    super.onManagerConnected(status);
+                } break;
+            }
+        }
+    };
+
+    /** Called when the activity is first created. */
+    @Override
+    public void onCreate(Bundle savedInstanceState) {
+        super.onCreate(savedInstanceState);
+        getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
+
+        setContentView(R.layout.activity_main);
+
+        mOpenCvCameraView = (CameraBridgeViewBase) findViewById(R.id.CameraView);
+
+        mOpenCvCameraView.setCvCameraViewListener(this);
+    }
+
+    @Override
+    public void onPause()
+    {
+        super.onPause();
+        if (mOpenCvCameraView != null)
+            mOpenCvCameraView.disableView();
+    }
+
+    @Override
+    public void onResume()
+    {
+        super.onResume();
+        if (!OpenCVLoader.initDebug()) {
+            Log.d(TAG, "Internal OpenCV library not found. Using OpenCV Manager for initialization");
+            OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_3_0_0, this, mLoaderCallback);
+        } else {
+            Log.d(TAG, "OpenCV library found inside package. Using it!");
+            mLoaderCallback.onManagerConnected(LoaderCallbackInterface.SUCCESS);
+        }
+    }
+
+    public void onDestroy() {
+        super.onDestroy();
+        if (mOpenCvCameraView != null)
+            mOpenCvCameraView.disableView();
+    }
+
+    public void onCameraViewStarted(int width, int height) {
+        try {
+            net = Dnn.readNetFromTensorflow(Extract("frozen_inference_graph.pb"), Extract("config.pbtxt"));
+            Log.i(TAG, "Network loaded successfully");
+        } catch (IOException e) {
+            e.printStackTrace();
+            Log.d(TAG, e.getMessage());
+            finish();
+        }
+/*
+        File sd = new File(android.os.Environment.getExternalStorageDirectory(), "WGD_frames/");
+        if (!sd.exists())
+            sd.mkdir();
+        path = sd.getPath();
+*/
+    }
+
+    // Extract a file from assets to a storage and return a path.
+    private String Extract(String filename) throws IOException {
+        AssetManager assetManager = getAssets();
+        BufferedInputStream inputStream = new BufferedInputStream(assetManager.open(filename));
+        byte[] data = new byte[inputStream.available()];
+        inputStream.read(data);
+        inputStream.close();
+        File outFile = new File(getFilesDir(), filename);
+        FileOutputStream outputStream = new FileOutputStream(outFile);
+        outputStream.write(data);
+        outputStream.close();
+        return outFile.getAbsolutePath();
+    }
+
+    public void onCameraViewStopped() {
+    }
+
+    public Mat onCameraFrame(CvCameraViewFrame inputFrame) {
+        Mat frame = inputFrame.rgba();
+        Imgproc.cvtColor(frame, frame, Imgproc.COLOR_RGBA2RGB);
+
+        int cols = frame.cols();
+        int rows = frame.rows();
+
+        // Pad a frame to make it square. Shrinking with a loss of aspect ratio so commonly applying everywhere is a bad idea.
+        int largest = Math.max(cols, rows);
+        Mat square = new Mat();
+        Core.copyMakeBorder(frame, square, 0, largest - rows, 0, largest - cols, Core.BORDER_CONSTANT, new Scalar(0, 0, 0));
+
+        Mat blob = Dnn.blobFromImage(square, 1,
+                new Size(300, 300),
+                new Scalar(0, 0, 0), false, false);
+        net.setInput(blob);
+        Mat detections = net.forward();
+
+        detections = detections.reshape(0, (int)detections.total() / 7);
+        for (int i = 0; i < detections.rows(); ++i) {
+            double confidence = detections.get(i, 2)[0];
+            if (confidence > 0.3) {
+                int classId = (int)detections.get(i, 1)[0];
+                int left = (int)(detections.get(i, 3)[0] * cols);
+                int top = (int)(detections.get(i, 4)[0] * rows);
+                int right = (int)(detections.get(i, 5)[0] * cols);
+                int bottom = (int)(detections.get(i, 6)[0] * rows);
+                // Bring coordinates back to the original frame.
+                if (cols > rows) {
+                    top *= (double)cols/rows;
+                    bottom *= (double)cols/rows;
+                }
+                else {
+                    left *= (double)rows/cols;
+                    right *= (double)rows/cols;
+                }
+
+                Scalar color;
+                switch (classId) {
+                    case 2:
+                    case 3:
+                        color = confidence > 0.6 ? new Scalar(255, 0, 0) : new Scalar(255, 255, 0);
+                        break;
+                    default:
+                        color = new Scalar(127, 127, 127);
+                }
+                // Draw rectangle around detected object.
+                Imgproc.rectangle(frame, new Point(left, top),
+                        new Point(right, bottom),
+                        color, 2);
+/*
+                // Print class and confidence.
+                String label = String.format("%d %.3f", classId, confidence);
+                int[] baseline = new int[1];
+                Size labelSize = Imgproc.getTextSize(label, Core.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseline);
+                Imgproc.rectangle(frame, new Point(left, top),
+                        new Point(left + labelSize.width, top + labelSize.height + baseline[0]),
+                        color, Core.FILLED);
+                Imgproc.putText(frame, label, new Point(left, top + labelSize.height + baseline[0]/2),
+                        Core.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(0, 0, 0));
+*/
+             }
+        }
+/*
+        Mat save = new Mat();
+        Imgproc.cvtColor(frame, save, Imgproc.COLOR_RGB2BGR);
+        org.opencv.imgcodecs.Imgcodecs.imwrite(path + String.format("/%tj_%1$tH%1$tM%1$tS%1$tL.jpg", new java.util.Date()), save);
+*/
+        return frame;
+    }
+}