Issue/framemap viewer improvements (#92)

* framemap loads with scroll and displays elements ordered by number of detected frames

* fmt

* use threshold in ranking

examples/ranking.yaml

@@ -0,0 +1,6 @@
+folder: "temp/military_vehicles_ambaz"
+phase: "analyse"
+module: "ranking"
+config:
+  elements_in:
+    - "youtube/keras_pretrained"

src/lib/analysers/ranking/__init__.py

@@ -0,0 +1,5 @@
+"""The ranking module is an ANALYSER that ranks the output from a keras_pretrained analyser.
+"""
+from .main import RankingAnalyser as main
+
+__all__ = ["main"]

src/lib/analysers/ranking/args.yaml

@@ -0,0 +1,5 @@
+- name: "threshold"
+  required: false
+  input: "float"
+  desc: "The minimum score for which a prediction should be counted towards an element's rank"
+

src/lib/analysers/ranking/main.py

@@ -0,0 +1,73 @@
+from lib.common.analyser import Analyser
+
+# from lib.common.exceptions import ElementShouldSkipError
+from lib.common.etypes import Etype
+import os
+import json
+import operator
+import collections
+from shutil import copyfile
+
+
+class RankingAnalyser(Analyser):
+    def get_in_etype(self):
+        return Etype.Any
+
+    def get_out_etype(self):
+        return Etype.Any
+
+    def pre_analyse(self, config):
+        self.thresh = config["threshold"] if "threshold" in config else 0.5
+        self.rankingData = {}
+        self.wkDir = ""
+
+    def analyse_element(self, element, config):
+        id = element["id"]
+        src = element["base"]
+        dest = element["dest"]
+
+        # run once
+        if self.wkDir == "":
+            self.wkDir = dest.replace(id, "")
+
+        epath = src + "/" + element["id"] + ".json"
+        with open(epath, "r") as f:
+            edata = json.load(f)
+
+        labels = edata["labels"]
+        for label, preds in labels.items():
+            count = len(
+                [
+                    idx
+                    for idx, _ in enumerate(preds["frames"])
+                    if preds["scores"][idx] > self.thresh
+                ]
+            )
+            if count > 4:
+                self.logger(f"{label}: {count}")
+            if label not in self.rankingData:
+                self.rankingData[label] = {}
+            self.rankingData[label][id] = count
+
+        dpath = dest + "/" + element["id"] + ".json"
+        copyfile(epath, dpath)
+        self.logger(f"Rankings added for {element['id']}.")
+
+    def post_analyse(self, config, derived_dirs):
+        ranking = self.dataToRanking()
+        path = self.wkDir + "all"
+        if not os.path.exists(path):
+            os.makedirs(path)
+        file = path + "/rankings.json"
+        self.logger("All rankings aggregated, printed to all/rankings.json")
+        with open(file, "w") as f:
+            json.dump(ranking, f)
+
+    def dataToRanking(self):
+        sortedData = {}
+        for label, values in self.rankingData.items():
+            s_vals = sorted(values.items(), key=operator.itemgetter(1))
+            s_vals.reverse()
+            s_els = [t[0] for t in s_vals]
+            sortedData[label] = s_els
+        return sortedData

src/lib/viewers/framemap/src/App.vue

@@ -3,11 +3,11 @@
     <Header :version="version" />
     <div id="body-container">
       <div class="container">
-        <!-- <div class="info&#45;case"> -->
-        <!--   <Controls /> -->
-        <!-- </div> -->
+        <!-- <div class="info&#45;case">
+           <Controls />
+        </div> -->
         <div class="content-case">
-          <Container :label="this.label" />
+          <Container :label="label" :threshold="threshold" />
         </div>
       </div>
     </div>

src/lib/viewers/framemap/src/LABEL.js

@@ -1,3 +1,4 @@
 export default {
-  label: "candle",
+  label: 'rifle',
+  threshold: 0.5
 }

src/lib/viewers/framemap/src/components/Container.vue

@@ -1,7 +1,7 @@
 <template>
   <div class="table">
-    <h1>Showing matches for: {{ this.label }}</h1>
-    <Graph :elements="elements" :label="this.label" />
+    <h1>Showing matches for: {{ label }}</h1>
+    <Graph :elements="elements" :label="label" :threshold="threshold" />
     <Loading v-if="!!fetching" />
     <div v-if="!!error" class="flexc">
       <h1>A network connection occurred. Make sure you are correctly configured with a running backend.</h1>
@@ -11,31 +11,44 @@
 
 <script>
 import Loading from './Loading.vue'
+import Graph from './Graph.vue'
 import { mapState, mapActions } from 'vuex'
 
 export default {
   name: 'Container',
   components: {
     Loading,
-    'Graph': () => import('./Graph.vue')
+    Graph
   },
   props: {
     label: String,
+    threshold: Number
   },
   methods: {
     ...mapActions([
-      'fetchElements'
-    ])
+      'fetchElements',
+      'fetchRankedElements'
+    ]),
+    scroll() {
+      window.onscroll = () => {
+        let bottomOfWindow = Math.max(window.pageYOffset, document.documentElement.scrollTop, document.body.scrollTop) + window.innerHeight === document.documentElement.offsetHeight
+        if (bottomOfWindow) {
+         this.fetchRankedElements(this.label)
+        }
+      }
+    }
   },
   computed: {
     ...mapState({
       fetching: 'fetching',
       elements: 'elements',
       error: 'error',
+      ranking: 'ranking'
     })
   },
   mounted: function () {
-    this.fetchElements()
+    this.fetchRankedElements(this.label)
+    this.scroll()
   }
 }
 </script>

src/lib/viewers/framemap/src/components/FrameMap.vue

@@ -17,7 +17,7 @@
           :style="getFrameColor(index)"
         />
         <span>{{ timeFmt(index) }}</span><br>
-        <span>{{ scoreFmt(index )}}</span>
+        <span>{{ scoreFmt(index)}}</span>
       </v-tooltip>
       <div
         v-else
@@ -37,11 +37,14 @@ export default {
     video_id: String,
     length: Number,
     frames: Array,
-    scores: Array
+    scores: Array,
+    label: String,
+    threshold: Number
   },
   methods: {
     isOn: function(idx) {
-      return this.frames.indexOf(idx) > -1
+      const _idx = this.frames.indexOf(idx)
+      return _idx > -1 && this.scores[_idx] > this.threshold
     },
     openFrame: function(idx) {
       window.open(`https://youtu.be/${this.video_id}?t=${idx}`, '_blank')

src/lib/viewers/framemap/src/components/Graph.vue

@@ -11,6 +11,8 @@
       :duration="video.duration"
       :frames="getFrames(video)"
       :scores="getScores(video)"
+      :label="label"
+      :threshold="threshold"
     ></VideoCell>
   </div>
 </template>
@@ -27,6 +29,7 @@
     props: {
       elements: Array,
       label: String,
+      threshold: Number
     },
     methods: {
       getFrames(video) {

src/lib/viewers/framemap/src/components/VideoCell.vue

@@ -11,6 +11,8 @@
         :length="duration"
         :frames="frames"
         :scores="scores"
+        :label="label"
+        :threshold="threshold"
       />
     </div>
      <!-- TODO: break this into another component -->
@@ -69,7 +71,9 @@ export default {
     description: String,
     duration: Number,
     frames: Array,
-    scores: Array
+    scores: Array,
+    label: String,
+    threshold: Number
   },
   computed: {
     rankFmt: function() {

src/lib/viewers/framemap/src/lib/api.js

@@ -2,16 +2,31 @@ import axios from 'axios'
 
 const cfg = {
   url: "http://localhost:8080",
-  analysed: "keras_pretrained",
-  context: "triplechaser"
+  analyser: "ranking",
+  context: "military_vehicles_ambaz"
 }
 
+function fetchRankedElements (label, fromIndex) {
+  const rUrl = `${cfg.url}/element?q=youtube/${cfg.analyser}&context=${cfg.context}&id=all&media=rankings.json`
+  return axios.get(rUrl)
+    .then(response => {
+      const rankings = response.data
+      console.log(label)
+      const rankedElements = rankings[label]
+      const urls = rankedElements.slice(fromIndex, fromIndex + 15).map(getElementUrl)
+      const promises = urls.map(url => Promise.resolve(url).then(url => axios.get(url).catch(err => null)))
+      return Promise.all(promises)
+    })
+    .then(fullElements => {
+      return fullElements.map(el => el !== null ? el.data : null).filter(el => el !== null)
+    })
+}
 
-function fetchElements() {
+function fetchElements () {
   return axios.get(`${cfg.url}/elementmap`)
     .then(response => {
       const elementmap = response.data
-      const elementSets = elementmap.Analysed.filter(els => els.Component === cfg.analysed)
+      const elementSets = elementmap.Analysed.filter(els => els.Component === cfg.analyser)
       const elements = elementSets.filter(els => els.Context === cfg.context)
       if (elements.length !== 1) {
         alert("check your elements dir and context")
@@ -26,11 +41,35 @@ function fetchElements() {
     })
 }
 
-function getElementUrl(element) {
-  const id = element.Id
-  return `${cfg.url}/element?q=youtube/${cfg.analysed}&context=${cfg.context}&id=${id}&media=${id}.json`
+function fetchIndexedElements(fromIndex) {
+  return axios.get(`${cfg.url}/elementmap`)
+    .then(response => {
+
+      const elementmap = response.data
+
+      const elementSets = elementmap.Analysed.filter(els => els.Component === cfg.analyser)
+      const elements = elementSets.filter(els => els.Context === cfg.context)
+      if (elements.length !== 1) {
+        alert("check your elements dir and context")
+      }
+      const ctxObj = elements[0]
+
+      const urls = ctxObj.Elements.slice(fromIndex, fromIndex + 20).map(getElementUrl)
+      const promises = urls.map(url => Promise.resolve(url).then(url => axios.get(url).catch(err => null)))
+      return Promise.all(promises)
+    })
+    .then(fullElements => {
+      return fullElements.map(el => el !== null ? el.data : null).filter(el => el !== null)
+    })
+}
+
+function getElementUrl(id) {
+  // const id = element.Id
+  return `${cfg.url}/element?q=youtube/${cfg.analyser}&context=${cfg.context}&id=${id}&media=${id}.json`
 }
 
 export default {
   fetchElements,
+  fetchIndexedElements,
+  fetchRankedElements
 }

src/lib/viewers/framemap/src/mutation-types.js

@@ -1,5 +1,9 @@
 export default {
   FETCH_ELEMENTS_ATTEMPT: 'FETCH_ELEMENTS_ATTEMPT',
   FETCH_ELEMENTS: 'FETCH_ELEMENTS',
-  FETCH_ELEMENTS_ERROR: 'FETCH_ELEMENTS_ERROR'
+  FETCH_ELEMENTS_ERROR: 'FETCH_ELEMENTS_ERROR',
+  FETCH_NEXT_ELEMENTS_ATTEMPT: 'FETCH_NEXT_ELEMENTS_ATTEMPT',
+  FETCH_NEXT_ELEMENTS: 'FETCH_NEXT_ELEMENTS',
+  FETCH_NEXT_ELEMENTS_ERROR: 'FETCH_NEXT_ELEMENTS_ERROR',
+  FETCH_RANKING: 'FETCH_RANKING'
 }

src/lib/viewers/framemap/src/store.js

@@ -24,17 +24,33 @@ export default new Vuex.Store({
     },
     [types.FETCH_ELEMENTS_ERROR] (state, msg) {
       state.error = msg
+    },
+    [types.FETCH_NEXT_ELEMENTS_ATTEMPT] (state) {
+      state.fetching = true
+    },
+    [types.FETCH_NEXT_ELEMENTS] (state, elements) {
+      state.elements = state.elements.concat(elements)
+      state.fetching = false
+    },
+    [types.FETCH_NEXT_ELEMENTS_ERROR] (state, msg) {
+      state.error = msg
+    },
+    [types.FETCH_RANKING] (state, ranking) {
+      state.ranking = ranking
+      state.fetching = false
     }
   },
   actions: {
-    fetchElements ({ commit, state }, pages) {
-      commit(types.FETCH_ELEMENTS_ATTEMPT)
-      api.fetchElements()
+    fetchRankedElements ({ commit, state }, label) {
+      commit(types.FETCH_NEXT_ELEMENTS_ATTEMPT)
+      const fromIndex = this.state.elements.length
+      api.fetchRankedElements(label, fromIndex)
         .then(result => {
-          commit(types.FETCH_ELEMENTS, result)
+          commit(types.FETCH_NEXT_ELEMENTS, result)
         })
         .catch(err => {
-          commit(types.FETCH_ELEMENTS_ERROR, err.message)
+          console.log(err.message)
+          commit(types.FETCH_NEXT_ELEMENTS_ERROR, err.message)
         })
     }
   }