Skip to content
Snippets Groups Projects
Commit 9da84f54 authored by Piotr Gawron's avatar Piotr Gawron
Browse files

invalid urls retrieved from miriam connector are cached only for a day (in...

invalid urls retrieved from miriam connector are cached only for a day (in case of changes in miriam registry)
parent 476f815f
No related branches found
No related tags found
1 merge request!138Resolve "miRNA target interface: some Pubmeds not hyperlinked"
Showing
with 938 additions and 939 deletions
......@@ -21,123 +21,129 @@ import lcsb.mapviewer.model.cache.CacheType;
*/
public final class ApplicationLevelCache extends XmlParser implements QueryCacheInterface {
/**
* How many values should be stored in cache before we even try to consider
* releasing object due to huge memory usage.
*/
static final int MIN_CACHED_VALUES_BEFORE_CLEAN = 100;
/**
* Default class logger.
*/
private static Logger logger = Logger.getLogger(ApplicationLevelCache.class);
/**
* Cached nodes stored locally and identified by string.
*/
private static Map<String, Node> cachedQueryNodes = new HashMap<String, Node>();
/**
* Cached strings stored locally and identified by string.
*/
private static Map<String, String> cachedQueryString = new HashMap<String, String>();
/**
* Global instance of cache.
*/
private static ApplicationLevelCache cache = null;
/**
* Returns single instance of global application cache (singleton pattern).
*
* @return single instance of global application cache
*/
public static ApplicationLevelCache getInstance() {
if (Configuration.isApplicationCacheOn()) {
if (cache == null) {
cache = new ApplicationLevelCache();
}
return cache;
} else {
return null;
}
}
/**
* Default constructor. Prevents instatiation.
*/
private ApplicationLevelCache() {
}
@Override
public synchronized Node getXmlNodeByQuery(String query, CacheType type) {
Node result = null;
result = cachedQueryNodes.get(type.getId() + "\n" + query);
return result;
}
@Override
public synchronized String getStringByQuery(String query, CacheType type) {
String result = null;
result = cachedQueryString.get(type.getId() + "\n" + query);
return result;
}
@Override
public synchronized void setCachedQuery(String query, CacheType type, Object object) {
performMemoryBalance();
if (object instanceof String) {
cachedQueryString.put(type.getId() + "\n" + query, (String) object);
} else if (object instanceof Node) {
cachedQueryNodes.put(type.getId() + "\n" + query, (Node) object);
} else if (object == null) {
cachedQueryString.put(type.getId() + "\n" + query, (String) object);
} else {
throw new CacheException("Unknown object type: " + object.getClass());
}
}
/**
* Method that clean cache if memory usage is too high.
*/
synchronized void performMemoryBalance() {
long cacheCount = 0;
cacheCount += cachedQueryString.size();
cacheCount += cachedQueryNodes.size();
// check memory usage only if we have some data to clear
if (cacheCount > MIN_CACHED_VALUES_BEFORE_CLEAN) {
Runtime runtime = Runtime.getRuntime();
long maxMem = runtime.maxMemory();
long useMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
// if free memory is less than 10% then we clear cache
if (useMem > maxMem * Configuration.getMemorySaturationRatioTriggerClean()) {
logger.info("Cache will be cleared");
logger.info("Memory usage: " + useMem + " out of " + maxMem + " is used. Exceeded: " + maxMem * Configuration.getMemorySaturationRatioTriggerClean());
logger.info("Elements in cache: " + cacheCount);
clearCache();
System.gc();
}
}
}
@Override
public synchronized void removeByQuery(String query, CacheType type) {
cachedQueryString.remove(type.getId() + "\n" + query);
cachedQueryNodes.remove(type.getId() + "\n" + query);
}
@Override
public synchronized void clearCache() {
logger.info("Clearing application cache");
cachedQueryNodes.clear();
cachedQueryString.clear();
}
@Override
public void invalidateByQuery(String query, CacheType type) {
removeByQuery(query, type);
}
/**
* How many values should be stored in cache before we even try to consider
* releasing object due to huge memory usage.
*/
static final int MIN_CACHED_VALUES_BEFORE_CLEAN = 100;
/**
* Default class logger.
*/
private static Logger logger = Logger.getLogger(ApplicationLevelCache.class);
/**
* Cached nodes stored locally and identified by string.
*/
private static Map<String, Node> cachedQueryNodes = new HashMap<String, Node>();
/**
* Cached strings stored locally and identified by string.
*/
private static Map<String, String> cachedQueryString = new HashMap<String, String>();
/**
* Global instance of cache.
*/
private static ApplicationLevelCache cache = null;
/**
* Returns single instance of global application cache (singleton pattern).
*
* @return single instance of global application cache
*/
public static ApplicationLevelCache getInstance() {
if (Configuration.isApplicationCacheOn()) {
if (cache == null) {
cache = new ApplicationLevelCache();
}
return cache;
} else {
return null;
}
}
/**
* Default constructor. Prevents instantiation.
*/
private ApplicationLevelCache() {
}
@Override
public synchronized Node getXmlNodeByQuery(String query, CacheType type) {
Node result = null;
result = cachedQueryNodes.get(type.getId() + "\n" + query);
return result;
}
@Override
public synchronized String getStringByQuery(String query, CacheType type) {
String result = null;
result = cachedQueryString.get(type.getId() + "\n" + query);
return result;
}
@Override
public synchronized void setCachedQuery(String query, CacheType type, Object object) {
setCachedQuery(query, type, object, type.getValidity());
}
@Override
public synchronized void setCachedQuery(String query, CacheType type, Object object, int validDays) {
performMemoryBalance();
if (object instanceof String) {
cachedQueryString.put(type.getId() + "\n" + query, (String) object);
} else if (object instanceof Node) {
cachedQueryNodes.put(type.getId() + "\n" + query, (Node) object);
} else if (object == null) {
cachedQueryString.put(type.getId() + "\n" + query, (String) object);
} else {
throw new CacheException("Unknown object type: " + object.getClass());
}
}
/**
* Method that clean cache if memory usage is too high.
*/
synchronized void performMemoryBalance() {
long cacheCount = 0;
cacheCount += cachedQueryString.size();
cacheCount += cachedQueryNodes.size();
// check memory usage only if we have some data to clear
if (cacheCount > MIN_CACHED_VALUES_BEFORE_CLEAN) {
Runtime runtime = Runtime.getRuntime();
long maxMem = runtime.maxMemory();
long useMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
// if free memory is less than 10% then we clear cache
if (useMem > maxMem * Configuration.getMemorySaturationRatioTriggerClean()) {
logger.info("Cache will be cleared");
logger.info("Memory usage: " + useMem + " out of " + maxMem + " is used. Exceeded: "
+ maxMem * Configuration.getMemorySaturationRatioTriggerClean());
logger.info("Elements in cache: " + cacheCount);
clearCache();
System.gc();
}
}
}
@Override
public synchronized void removeByQuery(String query, CacheType type) {
cachedQueryString.remove(type.getId() + "\n" + query);
cachedQueryNodes.remove(type.getId() + "\n" + query);
}
@Override
public synchronized void clearCache() {
logger.info("Clearing application cache");
cachedQueryNodes.clear();
cachedQueryString.clear();
}
@Override
public void invalidateByQuery(String query, CacheType type) {
removeByQuery(query, type);
}
}
......@@ -108,6 +108,12 @@ public abstract class CachableInterface extends XmlParser {
}
}
protected final void setCacheValue(String key, String result, int validDays) {
if (cache != null) {
cache.setCachedQuery(key, getCacheType(), result, validDays);
}
}
/**
* Retrieves string from cache.
*
......@@ -181,11 +187,11 @@ public abstract class CachableInterface extends XmlParser {
}
/**
* Returns a content of the webpage for a given url using GET request.
* Returns a content of the web page for a given url using GET request.
*
* @param accessUrl
* webpage url address
* @return content of the webpage
* web page url address
* @return content of the web page
* @throws IOException
* thrown when there are problems with connection to ChEMBL database
*/
......@@ -194,16 +200,16 @@ public abstract class CachableInterface extends XmlParser {
}
/**
* Returns a content of the webpage for a given url. If postData is not null,
* Returns a content of the web page for a given url. If postData is not null,
* the page will be accessed using POST request. Otherwise GET will be used.
*
* @param accessUrl
* webpage url address
* web page url address
* @param httpRequestMethod
* type of HTTP request (GET, POST, PUT, PATCH, DELETE, ...)
* @param postData
* string to be sent in the body of the request
* @return content of the webpage
* @return content of the web page
* @throws IOException
* thrown when there are problems with connection to ChEMBL database
*/
......@@ -235,7 +241,7 @@ public abstract class CachableInterface extends XmlParser {
public String cleanHtml(String text) {
int startIndex;
int endIndex;
// and now clean the descripton from html tags (should be somehow
// and now clean the description from html tags (should be somehow
// improved...)
StringBuilder result = new StringBuilder();
......
package lcsb.mapviewer.annotation.cache;
import lcsb.mapviewer.common.exception.InvalidArgumentException;
import lcsb.mapviewer.model.cache.CacheType;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.w3c.dom.Node;
import lcsb.mapviewer.common.exception.InvalidArgumentException;
import lcsb.mapviewer.model.cache.CacheType;
/**
* Cache used by the application. It contains two sub-classes responsible for
* application level cache (for single run of the application) and database
* level cache (for information that were gathere since the beginning).
* level cache (for information that were gathered since the beginning).
*
* @author Piotr Gawron
*
......@@ -19,127 +19,135 @@ import org.w3c.dom.Node;
@Transactional(value = "txManager")
public class GeneralCache implements GeneralCacheInterface {
/**
* Default class logger.
*/
@SuppressWarnings("unused")
private static Logger logger = Logger.getLogger(GeneralCache.class);
/**
* Application level cache. More information can be found
* {@link ApplicationLevelCache here}.
*/
private QueryCacheInterface cache1 = ApplicationLevelCache.getInstance();
/**
* Database level cache. More information can be found
* {@link PermanentDatabaseLevelCache here}.
*/
@Autowired
private QueryCacheInterface cache2;
@Override
public void clearCache() {
if (cache1 != null) {
cache1.clearCache();
}
if (cache2 != null) {
cache2.clearCache();
}
}
@Override
public Node getXmlNodeByQuery(String query, CacheType type) {
if (type == null) {
throw new InvalidArgumentException("Cache type cannot be null");
}
Node result = null;
if (cache1 != null) {
result = cache1.getXmlNodeByQuery(query, type);
}
if (result == null && cache2 != null) {
result = cache2.getXmlNodeByQuery(query, type);
if (result != null && cache1 != null) {
cache1.setCachedQuery(query, type, result);
}
}
return result;
}
@Override
public String getStringByQuery(String query, CacheType type) {
if (type == null) {
throw new InvalidArgumentException("Cache type cannot be null");
}
String result = null;
if (cache1 != null) {
result = cache1.getStringByQuery(query, type);
}
if (result == null && cache2 != null) {
result = cache2.getStringByQuery(query, type);
if (result != null && cache1 != null) {
cache1.setCachedQuery(query, type, result);
}
}
return result;
}
@Override
public void setCachedQuery(String query, CacheType type, Object object) {
if (type == null) {
throw new InvalidArgumentException("Cache type cannot be null");
}
if (cache1 != null) {
cache1.setCachedQuery(query, type, object);
}
if (cache2 != null) {
cache2.setCachedQuery(query, type, object);
}
}
@Override
public void removeByQuery(String query, CacheType type) {
if (type == null) {
throw new InvalidArgumentException("Cache type cannot be null");
}
if (cache1 != null) {
cache1.removeByQuery(query, type);
}
if (cache2 != null) {
cache2.removeByQuery(query, type);
}
}
@Override
public void invalidateByQuery(String query, CacheType type) {
if (type == null) {
throw new InvalidArgumentException("Cache type cannot be null");
}
if (cache1 != null) {
cache1.invalidateByQuery(query, type);
}
if (cache2 != null) {
cache2.invalidateByQuery(query, type);
}
}
/**
* @return the cache2
* @see #cache2
*/
public QueryCacheInterface getCache2() {
return cache2;
}
/**
* @param cache2
* the cache2 to set
* @see #cache2
*/
public void setCache2(QueryCacheInterface cache2) {
this.cache2 = cache2;
}
/**
* Default class logger.
*/
@SuppressWarnings("unused")
private static Logger logger = Logger.getLogger(GeneralCache.class);
/**
* Application level cache. More information can be found
* {@link ApplicationLevelCache here}.
*/
private QueryCacheInterface cache1 = ApplicationLevelCache.getInstance();
/**
* Database level cache. More information can be found
* {@link PermanentDatabaseLevelCache here}.
*/
@Autowired
private QueryCacheInterface cache2;
@Override
public void clearCache() {
if (cache1 != null) {
cache1.clearCache();
}
if (cache2 != null) {
cache2.clearCache();
}
}
@Override
public Node getXmlNodeByQuery(String query, CacheType type) {
if (type == null) {
throw new InvalidArgumentException("Cache type cannot be null");
}
Node result = null;
if (cache1 != null) {
result = cache1.getXmlNodeByQuery(query, type);
}
if (result == null && cache2 != null) {
result = cache2.getXmlNodeByQuery(query, type);
if (result != null && cache1 != null) {
cache1.setCachedQuery(query, type, result);
}
}
return result;
}
@Override
public String getStringByQuery(String query, CacheType type) {
if (type == null) {
throw new InvalidArgumentException("Cache type cannot be null");
}
String result = null;
if (cache1 != null) {
result = cache1.getStringByQuery(query, type);
}
if (result == null && cache2 != null) {
result = cache2.getStringByQuery(query, type);
if (result != null && cache1 != null) {
cache1.setCachedQuery(query, type, result);
}
}
return result;
}
@Override
public void setCachedQuery(String query, CacheType type, Object object) {
if (type == null) {
throw new InvalidArgumentException("Cache type cannot be null");
}
setCachedQuery(query, type, object, type.getValidity());
}
@Override
public void setCachedQuery(String query, CacheType type, Object object, int validDays) {
if (type == null) {
throw new InvalidArgumentException("Cache type cannot be null");
}
if (cache1 != null) {
cache1.setCachedQuery(query, type, object, validDays);
}
if (cache2 != null) {
cache2.setCachedQuery(query, type, object, validDays);
}
}
@Override
public void removeByQuery(String query, CacheType type) {
if (type == null) {
throw new InvalidArgumentException("Cache type cannot be null");
}
if (cache1 != null) {
cache1.removeByQuery(query, type);
}
if (cache2 != null) {
cache2.removeByQuery(query, type);
}
}
@Override
public void invalidateByQuery(String query, CacheType type) {
if (type == null) {
throw new InvalidArgumentException("Cache type cannot be null");
}
if (cache1 != null) {
cache1.invalidateByQuery(query, type);
}
if (cache2 != null) {
cache2.invalidateByQuery(query, type);
}
}
/**
* @return the cache2
* @see #cache2
*/
public QueryCacheInterface getCache2() {
return cache2;
}
/**
* @param cache2
* the cache2 to set
* @see #cache2
*/
public void setCache2(QueryCacheInterface cache2) {
this.cache2 = cache2;
}
}
......@@ -7,7 +7,7 @@ import lcsb.mapviewer.common.exception.InvalidArgumentException;
import lcsb.mapviewer.model.cache.CacheType;
/**
* This implementation of cache works as a nomral cache except of the fact that
* This implementation of cache works as a normal cache except of the fact that
* first few query requests are ignored (return null). It's used when refreshing
* data from cache. Object of this class is injected into a
* {@link CachableInterface} and after that normal call is performed. First
......@@ -18,76 +18,81 @@ import lcsb.mapviewer.model.cache.CacheType;
*
*/
public class GeneralCacheWithExclusion implements GeneralCacheInterface {
/**
* Default class logger.
*/
private Logger logger = Logger.getLogger(GeneralCacheWithExclusion.class);
/**
* How many queries should be ignored.
*/
private int counter = 0;
/**
* Default class logger.
*/
private Logger logger = Logger.getLogger(GeneralCacheWithExclusion.class);
/**
* Original cache object.
*/
private GeneralCacheInterface cache;
/**
* How many queries should be ignored.
*/
private int counter = 0;
/**
* Constructor that creates cache.
*
* @param originalCache
* {@link #cache}
* @param exclusionCount
* {@link #counter}
*/
public GeneralCacheWithExclusion(GeneralCacheInterface originalCache, int exclusionCount) {
if (originalCache == null) {
throw new InvalidArgumentException("Cache passed as argument cannot be null");
}
this.counter = exclusionCount;
this.cache = originalCache;
}
/**
* Original cache object.
*/
private GeneralCacheInterface cache;
@Override
public Node getXmlNodeByQuery(String identifier, CacheType type) {
if (counter > 0) {
counter--;
logger.debug("Ignoring cache query due to cache refresh: " + identifier + ", " + type);
return null;
}
return cache.getXmlNodeByQuery(identifier, type);
}
/**
* Constructor that creates cache.
*
* @param originalCache
* {@link #cache}
* @param exclusionCount
* {@link #counter}
*/
public GeneralCacheWithExclusion(GeneralCacheInterface originalCache, int exclusionCount) {
if (originalCache == null) {
throw new InvalidArgumentException("Cache passed as argument cannot be null");
}
this.counter = exclusionCount;
this.cache = originalCache;
}
@Override
public String getStringByQuery(String identifier, CacheType type) {
if (counter > 0) {
counter--;
logger.debug("Ignoring cache query due to cache refresh: " + identifier + ", " + type);
return null;
}
return cache.getStringByQuery(identifier, type);
}
@Override
public Node getXmlNodeByQuery(String identifier, CacheType type) {
if (counter > 0) {
counter--;
logger.debug("Ignoring cache query due to cache refresh: " + identifier + ", " + type);
return null;
}
return cache.getXmlNodeByQuery(identifier, type);
}
@Override
public void setCachedQuery(String identifier, CacheType type, Object value) {
cache.setCachedQuery(identifier, type, value);
}
@Override
public String getStringByQuery(String identifier, CacheType type) {
if (counter > 0) {
counter--;
logger.debug("Ignoring cache query due to cache refresh: " + identifier + ", " + type);
return null;
}
return cache.getStringByQuery(identifier, type);
}
@Override
public void clearCache() {
cache.clearCache();
@Override
public void setCachedQuery(String identifier, CacheType type, Object value) {
cache.setCachedQuery(identifier, type, value);
}
@Override
public void setCachedQuery(String identifier, CacheType type, Object value, int validDays) {
cache.setCachedQuery(identifier, type, value, validDays);
}
}
@Override
public void clearCache() {
cache.clearCache();
@Override
public void removeByQuery(String identifier, CacheType type) {
cache.removeByQuery(identifier, type);
}
}
@Override
public void invalidateByQuery(String identifier, CacheType type) {
cache.invalidateByQuery(identifier, type);
}
@Override
public void removeByQuery(String identifier, CacheType type) {
cache.removeByQuery(identifier, type);
}
@Override
public void invalidateByQuery(String identifier, CacheType type) {
cache.invalidateByQuery(identifier, type);
}
}
......@@ -27,9 +27,9 @@ public interface PermanentDatabaseLevelCacheInterface extends QueryCacheInterfac
boolean refreshIsBusy();
/**
* Returns aproximate number of refresh tasks that are currently executed.
* Returns approximate number of refresh tasks that are currently executed.
*
* @return aproximate number of refresh tasks that are currently executed.
* @return approximate number of refresh tasks that are currently executed.
*/
int getRefreshExecutingTasksSize();
......
......@@ -12,67 +12,69 @@ import org.w3c.dom.Node;
*/
public interface QueryCacheInterface {
/**
* Returns xml node from the cache. The entry (xml node in this case) is
* identified by string and type associated with the entry.
*
* @param identifier
* string identifing node in the database
* @param type
* type of the entry
* @return node from cache for given identifier and type
*/
Node getXmlNodeByQuery(String identifier, CacheType type);
/**
* Returns xml node from the cache. The entry (xml node in this case) is
* identified by string and type associated with the entry.
*
* @param identifier
* string identifying node in the database
* @param type
* type of the entry
* @return node from cache for given identifier and type
*/
Node getXmlNodeByQuery(String identifier, CacheType type);
/**
* Returns string from the cache. The entry (string in this case) is
* identified by string and type associated with the entry.
*
* @param identifier
* string identifing node in the database
* @param type
* type of the entry
* @return node from cache for given string identifier and type
*/
String getStringByQuery(String identifier, CacheType type);
/**
* Returns string from the cache. The entry (string in this case) is identified
* by string and type associated with the entry.
*
* @param identifier
* string identifying node in the database
* @param type
* type of the entry
* @return node from cache for given string identifier and type
*/
String getStringByQuery(String identifier, CacheType type);
/**
* Puts new value into cache. The entry is identified with type and identfier.
*
* @param identifier
* string identifing entry in the database
* @param type
* type of the entry
* @param value
* value to be stored in the cache
*/
void setCachedQuery(String identifier, CacheType type, Object value);
/**
* Puts new value into cache. The entry is identified with type and identifier.
*
* @param identifier
* string identifying entry in the database
* @param type
* type of the entry
* @param value
* value to be stored in the cache
*/
void setCachedQuery(String identifier, CacheType type, Object value);
void setCachedQuery(String identifier, CacheType type, Object value, int validDays);
/**
* Clears permanently whole cache.
*/
void clearCache();
/**
* Clears permanently whole cache.
*/
void clearCache();
/**
* Removes element from the database.
*
* @param identifier
* string identifing entry in the database
* @param type
* type of the entry
*/
void removeByQuery(String identifier, CacheType type);
/**
* Removes element from the database.
*
* @param identifier
* string identifying entry in the database
* @param type
* type of the entry
*/
void removeByQuery(String identifier, CacheType type);
/**
* This method invalidate result, but doesn't remove it (data should be
* available until resource will be updated from original source). Moreover,
* value reload method is called.
*
* @param identifier
* string identifing entry in the database
* @param type
* type of the entry
*/
void invalidateByQuery(String identifier, CacheType type);
/**
* This method invalidate result, but doesn't remove it (data should be
* available until resource will be updated from original source). Moreover,
* value reload method is called.
*
* @param identifier
* string identifying entry in the database
* @param type
* type of the entry
*/
void invalidateByQuery(String identifier, CacheType type);
}
......@@ -101,7 +101,8 @@ public final class MiriamConnector extends CachableInterface implements IExterna
return result;
} else {
logger.warn("Cannot find url for miriam: " + miriamData);
setCacheValue(query, INVALID_LINK);
// if url cannot be found then mark miriam data as invalid for one day
setCacheValue(query, INVALID_LINK, 1);
return null;
}
}
......
package lcsb.mapviewer.annotation.cache;
import lcsb.mapviewer.model.cache.CacheType;
import org.w3c.dom.Node;
public class CacheMock implements GeneralCacheInterface {
@Override
public Node getXmlNodeByQuery(String query, CacheType type) {
// TODO Auto-generated method stub
return null;
}
@Override
public String getStringByQuery(String query, CacheType source) {
// TODO Auto-generated method stub
return null;
}
@Override
public void setCachedQuery(String query, CacheType type, Object node) {
// TODO Auto-generated method stub
}
@Override
public void clearCache() {
// TODO Auto-generated method stub
}
@Override
public void removeByQuery(String query, CacheType type) {
// TODO Auto-generated method stub
}
@Override
public void invalidateByQuery(String query, CacheType type) {
// TODO Auto-generated method stub
}
}
-- clear invalid urls from miriam connector
delete from cachequery where type = (select iddb from cache_type where classname='lcsb.mapviewer.annotation.services.MiriamConnector') and value = 'INVALID';
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment