From c5753ed4dc2903c14ad42e0b06d3f1af8a8fd507 Mon Sep 17 00:00:00 2001 From: grzesiekb Date: Thu, 2 Nov 2017 13:23:25 +0100 Subject: [PATCH] #410 Correcting DataSourcePointsCache --- .gitignore | 12 +- WebContent/WEB-INF/classes/env.properties | 5 +- build.xml | 8 +- .../serotonin/mango/MangoContextListener.java | 57 ++++----- .../dataSource/modbus/ModbusIpDataSource.java | 17 ++- src/org/scada_lts/cache/DataPointsCache.java | 81 ------------ .../cache/DataSourcePointsCache.java | 120 ++++++++++++++++++ .../cache/UpdateDataSourcesPoints.java | 27 ++++ src/org/scada_lts/config/ScadaConfig.java | 21 ++- ...ointsCacheComposeDataSourcesDataTest.java} | 32 +++-- ... => DataSourcePointsCacheMockitoTest.java} | 16 +-- 11 files changed, 239 insertions(+), 157 deletions(-) delete mode 100644 src/org/scada_lts/cache/DataPointsCache.java create mode 100644 src/org/scada_lts/cache/DataSourcePointsCache.java create mode 100644 src/org/scada_lts/cache/UpdateDataSourcesPoints.java rename test/org/scada_lts/cached/{DataPointsCacheComposeDataTest.java => DataPointsCacheComposeDataSourcesDataTest.java} (76%) rename test/org/scada_lts/cached/{DataPointsCacheMockitoTest.java => DataSourcePointsCacheMockitoTest.java} (79%) diff --git a/.gitignore b/.gitignore index d0fa968abe..bd8acd11ba 100644 --- a/.gitignore +++ b/.gitignore @@ -77,4 +77,14 @@ seroUtils.jar ########################### # docker ########################### -docker/app/** \ No newline at end of file +docker/app/** + +########################### +# intelij +########################### +.idea + +########################### +# jrebel +########################### +catalina-jrebel.bat \ No newline at end of file diff --git a/WebContent/WEB-INF/classes/env.properties b/WebContent/WEB-INF/classes/env.properties index 844d1fbde9..0b3828295b 100644 --- a/WebContent/WEB-INF/classes/env.properties +++ b/WebContent/WEB-INF/classes/env.properties @@ -66,8 +66,9 @@ abilit.MILLIS_SECONDS_PERIOD_UPDATE_EVENT_DETECTORS=1000 abilit.MILLIS_SECONDS_PERIOD_UPDATE_PENDING_EVENTS=1000 abilit.CRONE_UPDATE_CACHE_POINT_HIERARCHY=0 0/10 * * * ? - - +# every minute +abilit.CRONE_UPDATE_DATA_SOURCES_POINTS=0 0/1 * * * ? +abilit.USE_CACHE_DATA_SOURCES_POINTS_WHEN_THE_SYSTEM_IS_READY=true #security.hashAlgorithm=NONE #grove.url=http://mango.serotoninsoftware.com/servlet diff --git a/build.xml b/build.xml index 6bca19b9dd..1ca4eea74c 100644 --- a/build.xml +++ b/build.xml @@ -23,7 +23,9 @@ update="true"/> - + + + @@ -278,11 +280,11 @@ haltonfailure="yes"> --> - - diff --git a/src/com/serotonin/mango/MangoContextListener.java b/src/com/serotonin/mango/MangoContextListener.java index 3782a583e1..70c7058b99 100644 --- a/src/com/serotonin/mango/MangoContextListener.java +++ b/src/com/serotonin/mango/MangoContextListener.java @@ -19,32 +19,7 @@ package com.serotonin.mango; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import javax.servlet.ServletContext; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.mozilla.javascript.ContextFactory; -import org.scada_lts.cache.DataPointsCache; -import org.scada_lts.cache.EventDetectorsCache; -import org.scada_lts.cache.PointHierarchyCache; -import org.scada_lts.cache.ViewHierarchyCache; -import org.scada_lts.dao.SystemSettingsDAO; -import org.scada_lts.mango.adapter.MangoScadaConfig; -import org.scada_lts.scripting.SandboxContextFactory; - +import br.org.scadabr.api.utils.APIUtils; import com.serotonin.ShouldNeverHappenException; import com.serotonin.mango.db.DatabaseAccess; import com.serotonin.mango.db.dao.DataPointDao; @@ -75,13 +50,35 @@ import com.serotonin.mango.web.dwr.BaseDwr; import com.serotonin.util.StringUtils; import com.serotonin.web.i18n.LocalizableMessage; - -import br.org.scadabr.api.utils.APIUtils; import freemarker.cache.FileTemplateLoader; import freemarker.cache.MultiTemplateLoader; import freemarker.cache.TemplateLoader; import freemarker.template.Configuration; import freemarker.template.DefaultObjectWrapper; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.mozilla.javascript.ContextFactory; +import org.scada_lts.cache.DataSourcePointsCache; +import org.scada_lts.cache.EventDetectorsCache; +import org.scada_lts.cache.PointHierarchyCache; +import org.scada_lts.cache.ViewHierarchyCache; +import org.scada_lts.dao.SystemSettingsDAO; +import org.scada_lts.mango.adapter.MangoScadaConfig; +import org.scada_lts.scripting.SandboxContextFactory; + +import javax.servlet.ServletContext; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; public class MangoContextListener implements ServletContextListener { private final Log log = LogFactory.getLog(MangoContextListener.class); @@ -132,7 +129,7 @@ public void contextInitialized(ServletContextEvent evt) { } try { - DataPointsCache.getInstance().cacheInitialize(); + DataSourcePointsCache.getInstance().cacheInitialize(); log.info("Cache data points initialized"); runtimeManagerInitialize(ctx); @@ -140,7 +137,7 @@ public void contextInitialized(ServletContextEvent evt) { } catch (Exception e) { log.error(e); } finally { - DataPointsCache.getInstance().cacheFinalized(); + DataSourcePointsCache.getInstance().cacheFinalized(); } diff --git a/src/com/serotonin/mango/rt/dataSource/modbus/ModbusIpDataSource.java b/src/com/serotonin/mango/rt/dataSource/modbus/ModbusIpDataSource.java index 92b3066fcb..5805bac2f6 100644 --- a/src/com/serotonin/mango/rt/dataSource/modbus/ModbusIpDataSource.java +++ b/src/com/serotonin/mango/rt/dataSource/modbus/ModbusIpDataSource.java @@ -18,13 +18,6 @@ */ package com.serotonin.mango.rt.dataSource.modbus; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.scada_lts.cache.DataPointsCache; - import com.serotonin.mango.Common; import com.serotonin.mango.db.dao.DataPointDao; import com.serotonin.mango.rt.dataImage.DataPointRT; @@ -38,6 +31,12 @@ import com.serotonin.modbus4j.ModbusFactory; import com.serotonin.modbus4j.ModbusMaster; import com.serotonin.modbus4j.ip.IpParameters; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.scada_lts.cache.DataSourcePointsCache; + +import java.util.ArrayList; +import java.util.List; public class ModbusIpDataSource extends ModbusDataSource { private final ModbusIpDataSourceVO configuration; @@ -69,8 +68,8 @@ public void addDataPoint(DataPointRT dataPoint) { boolean found = false; List points; - if (DataPointsCache.getInstance().isStart()) { - points = DataPointsCache.getInstance().getDataPoints((long) configuration.getId()); + if (DataSourcePointsCache.getInstance().isCacheEnabled()) { + points = DataSourcePointsCache.getInstance().getDataPoints((long) configuration.getId()); } else { points = dataPointDao.getDataPoints( configuration.getId(), null); diff --git a/src/org/scada_lts/cache/DataPointsCache.java b/src/org/scada_lts/cache/DataPointsCache.java deleted file mode 100644 index ec8b01389f..0000000000 --- a/src/org/scada_lts/cache/DataPointsCache.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.scada_lts.cache; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.scada_lts.dao.DataPointDAO; - -import com.serotonin.mango.vo.DataPointVO; - -public class DataPointsCache implements IDataPointsCacheWhenStart { - - private boolean start = false; - - private static DataPointsCache instance = null; - - private Map> dss = new HashMap>(); - - private DataPointsCache() { - - } - - public static DataPointsCache getInstance() { - if (instance==null) { - instance = new DataPointsCache(); - } - return instance; - } - - @Override - public List getDataPoints(Long dataSourceId) { - if (start) { - return dss.get(dataSourceId); - } else { - throw new RuntimeException("Cache may work only when scada start"); - } - } - - @Override - public void cacheFinalized() { - start = false; - instance = null; - } - - public boolean isStart() { - return start; - } - - @Override - public void cacheInitialize() { - - List dps = new DataPointDAO().getDataPoints(); - - dss = composeCashData(dps); - - start = true; - } - - public Map> composeCashData(List dps) { - - Map> dss = new HashMap>(); - if (dps != null && dps.size()>0) { - for (DataPointVO dp : dps) { - List cacheDs = dss.get((long)dp.getDataSourceId()); - if (cacheDs==null) { - cacheDs = new ArrayList(); - cacheDs.add(dp); - dss.put((long) dp.getDataSourceId(), cacheDs); - } else { - cacheDs.add(dp); - } - } - } - - return dss; - - - } - -} diff --git a/src/org/scada_lts/cache/DataSourcePointsCache.java b/src/org/scada_lts/cache/DataSourcePointsCache.java new file mode 100644 index 0000000000..60b5f47c7c --- /dev/null +++ b/src/org/scada_lts/cache/DataSourcePointsCache.java @@ -0,0 +1,120 @@ +package org.scada_lts.cache; + +import com.serotonin.mango.vo.DataPointVO; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.quartz.*; +import org.quartz.impl.StdSchedulerFactory; +import org.scada_lts.config.ScadaConfig; +import org.scada_lts.dao.DataPointDAO; + +import java.io.IOException; +import java.text.ParseException; +import java.util.*; + +public class DataSourcePointsCache implements IDataPointsCacheWhenStart { + + private static final Log LOG = LogFactory.getLog(DataSourcePointsCache.class); + + private boolean cacheEnabled = false; + + private static DataSourcePointsCache instance = null; + + private Map> dss = new TreeMap<>(); + + private DataSourcePointsCache() { + + } + + public static DataSourcePointsCache getInstance() { + if (instance==null) { + instance = new DataSourcePointsCache(); + } + return instance; + } + + @Override + public List getDataPoints(Long dataSourceId) { + + LOG.info("I'm using a cache with datasources points"); + + if (cacheEnabled) { + return dss.get(dataSourceId); + } else { + throw new RuntimeException("Cache may work only when scada cacheEnabled"); + } + } + + public void setData(Map> dss) { + this.dss = dss; + } + + @Override + public void cacheFinalized() { + try { + if (ScadaConfig.getInstance().getBoolean(ScadaConfig.USE_CACHE_DATA_SOURCES_POINTS_WHEN_THE_SYSTEM_IS_READY, false)) { + cronInitialize(); + cacheEnabled = true; + } else { + cacheEnabled = false; + instance = null; + } + } catch (IOException | SchedulerException | ParseException e) { + LOG.error(e); + } + } + + public boolean isCacheEnabled() { + return cacheEnabled; + } + + @Override + public void cacheInitialize() { + + List dps = new DataPointDAO().getDataPoints(); + + dss = composeCashData(dps); + + cacheEnabled = true; + } + + public Map> composeCashData(List dps) { + + Map> dss = new TreeMap<>(); + if (dps != null && dps.size()>0) { + for (DataPointVO dp : dps) { + List cacheDs = dss.get((long)dp.getDataSourceId()); + if (cacheDs==null) { + cacheDs = new ArrayList<>(); + cacheDs.add(dp); + dss.put((long) dp.getDataSourceId(), cacheDs); + } else { + cacheDs.add(dp); + } + } + } + return dss; + } + + private void cronInitialize() throws java.io.IOException, SchedulerException, ParseException { + + if (LOG.isTraceEnabled()) { + LOG.trace("cacheInitialize"); + } + + JobDetail job = new JobDetail(); + job.setName("UpdateDataSourcesPoints"); + job.setJobClass(UpdateDataSourcesPoints.class); + + + CronTrigger trigger = new CronTrigger(); + trigger.setName("Quartz - trigger-DataSourcePointsCache"); + String cronExpression = ScadaConfig.getInstance().getProperty(ScadaConfig.CRONE_UPDATE_CACHE_DATA_SOURCES_POINTS); + trigger.setCronExpression(cronExpression);//"0 15 1 ? * *" + Scheduler scheduler = new StdSchedulerFactory().getScheduler(); + scheduler.start(); + scheduler.scheduleJob(job, trigger); + + } + +} diff --git a/src/org/scada_lts/cache/UpdateDataSourcesPoints.java b/src/org/scada_lts/cache/UpdateDataSourcesPoints.java new file mode 100644 index 0000000000..27f23c2a86 --- /dev/null +++ b/src/org/scada_lts/cache/UpdateDataSourcesPoints.java @@ -0,0 +1,27 @@ +package org.scada_lts.cache; + +import com.serotonin.mango.vo.DataPointVO; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.quartz.StatefulJob; +import org.scada_lts.dao.DataPointDAO; + +import java.util.List; +import java.util.Map; + +public class UpdateDataSourcesPoints implements StatefulJob{ + + private static final Log LOG = LogFactory.getLog(UpdateDataSourcesPoints.class); + + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { + + LOG.trace("UpdateEventDetectors"); + List dps = new DataPointDAO().getDataPoints(); + Map> dss = DataSourcePointsCache.getInstance().composeCashData(dps); + DataSourcePointsCache.getInstance().setData(dss); + + } +} diff --git a/src/org/scada_lts/config/ScadaConfig.java b/src/org/scada_lts/config/ScadaConfig.java index 47648ffb98..a82c2da4ae 100644 --- a/src/org/scada_lts/config/ScadaConfig.java +++ b/src/org/scada_lts/config/ScadaConfig.java @@ -17,6 +17,10 @@ */ package org.scada_lts.config; +import com.serotonin.mango.Common; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -24,11 +28,6 @@ import java.nio.file.Paths; import java.util.Properties; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import com.serotonin.mango.Common; - /** * Class responsible for configuration scada LTS @@ -82,7 +81,17 @@ public class ScadaConfig { * Period update point hierarchy but can be delayed when system heavy loaded. For example cron 0 15 1 ? * *. */ public static final String CRONE_UPDATE_CACHE_POINT_HIERARCHY = "abilit.CRONE_UPDATE_CACHE_POINT_HIERARCHY"; - + + /** + * Period update data sources points. For example cron 0 15 1 ? * * the after start + */ + public static final String CRONE_UPDATE_CACHE_DATA_SOURCES_POINTS = "abilit.CRONE_UPDATE_DATA_SOURCES_POINTS"; + + /** + * Use Cache data sources points when the system is ready + */ + public static final String USE_CACHE_DATA_SOURCES_POINTS_WHEN_THE_SYSTEM_IS_READY = "abilit.USE_CACHE_DATA_SOURCES_POINTS_WHEN_THE_SYSTEM_IS_READY"; + private static final Log LOG = LogFactory.getLog(ScadaConfig.class); private static final String FILE_NAME_LOGO="logo.png"; private static final String FILE_NAME_PROPERTIES="env.properties"; diff --git a/test/org/scada_lts/cached/DataPointsCacheComposeDataTest.java b/test/org/scada_lts/cached/DataPointsCacheComposeDataSourcesDataTest.java similarity index 76% rename from test/org/scada_lts/cached/DataPointsCacheComposeDataTest.java rename to test/org/scada_lts/cached/DataPointsCacheComposeDataSourcesDataTest.java index 4d8e50294c..e4d500e1cf 100644 --- a/test/org/scada_lts/cached/DataPointsCacheComposeDataTest.java +++ b/test/org/scada_lts/cached/DataPointsCacheComposeDataSourcesDataTest.java @@ -1,31 +1,30 @@ package org.scada_lts.cached; -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - +import com.serotonin.mango.vo.DataPointVO; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import org.scada_lts.cache.DataPointsCache; +import org.scada_lts.cache.DataSourcePointsCache; -import com.serotonin.mango.vo.DataPointVO; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertTrue; @RunWith(JUnit4.class) -public class DataPointsCacheComposeDataTest { +public class DataPointsCacheComposeDataSourcesDataTest { @Before public void init() { - DataPointsCache.getInstance(); + DataSourcePointsCache.getInstance(); } @After public void finalized() { - DataPointsCache.getInstance().cacheFinalized(); + //DataSourcePointsCache.getInstance().cacheFinalized(); } @Test @@ -46,7 +45,7 @@ public void composeDsWithDp() { lst.add(dpvo1); lst.add(dpvo2); - Map> map = DataPointsCache.getInstance().composeCashData(lst); + Map> map = DataSourcePointsCache.getInstance().composeCashData(lst); assertTrue(map.size()==2); @@ -62,7 +61,7 @@ public void composeDsWithDpOne() { lst.add(dpvo); - Map> map = DataPointsCache.getInstance().composeCashData(lst); + Map> map = DataSourcePointsCache.getInstance().composeCashData(lst); assertTrue(map.size()==1); @@ -80,7 +79,7 @@ public void composeDsWithDpTwo() { lst.add(dpvo); } - Map> map = DataPointsCache.getInstance().composeCashData(lst); + Map> map = DataSourcePointsCache.getInstance().composeCashData(lst); assertTrue(map.size()==1); @@ -110,7 +109,7 @@ public void composeDsWithDpThree() { lst.add(dpvo); } - Map> map = DataPointsCache.getInstance().composeCashData(lst); + Map> map = DataSourcePointsCache.getInstance().composeCashData(lst); assertTrue(map.size()==100); assertTrue(map.get(1L).size()==countOne); @@ -118,6 +117,5 @@ public void composeDsWithDpThree() { assertTrue(map.get(100L).size()==1); } - - + } diff --git a/test/org/scada_lts/cached/DataPointsCacheMockitoTest.java b/test/org/scada_lts/cached/DataSourcePointsCacheMockitoTest.java similarity index 79% rename from test/org/scada_lts/cached/DataPointsCacheMockitoTest.java rename to test/org/scada_lts/cached/DataSourcePointsCacheMockitoTest.java index ce7e464977..06e5ce3dd6 100644 --- a/test/org/scada_lts/cached/DataPointsCacheMockitoTest.java +++ b/test/org/scada_lts/cached/DataSourcePointsCacheMockitoTest.java @@ -1,21 +1,21 @@ package org.scada_lts.cached; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; - -import java.util.ArrayList; -import java.util.List; - +import com.serotonin.mango.vo.DataPointVO; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import org.scada_lts.cache.IDataPointsCacheWhenStart; -import com.serotonin.mango.vo.DataPointVO; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @RunWith(JUnit4.class) -public class DataPointsCacheMockitoTest { +public class DataSourcePointsCacheMockitoTest { IDataPointsCacheWhenStart cache;