From 8039415742840ee8cacf5c230bffdae784b2dae7 Mon Sep 17 00:00:00 2001
From: Piotr Gawron <piotr.gawron@uni.lu>
Date: Fri, 15 Jun 2018 10:09:44 +0200
Subject: [PATCH] resolving of available genome organisms fixed

---
 .../genome/UcscReferenceGenomeConnector.java  |  851 ++++----
 .../UcscReferenceGenomeConnectorTest.java     | 1839 +++++++++--------
 2 files changed, 1349 insertions(+), 1341 deletions(-)

diff --git a/annotation/src/main/java/lcsb/mapviewer/annotation/services/genome/UcscReferenceGenomeConnector.java b/annotation/src/main/java/lcsb/mapviewer/annotation/services/genome/UcscReferenceGenomeConnector.java
index d32370f063..9af8f7fdc3 100644
--- a/annotation/src/main/java/lcsb/mapviewer/annotation/services/genome/UcscReferenceGenomeConnector.java
+++ b/annotation/src/main/java/lcsb/mapviewer/annotation/services/genome/UcscReferenceGenomeConnector.java
@@ -42,429 +42,432 @@ import lcsb.mapviewer.model.map.layout.ReferenceGenomeType;
  */
 public class UcscReferenceGenomeConnector extends AbstractReferenceGenomeConnector implements ReferenceGenomeConnector {
 
-	/**
-	 * Server domain name.
-	 */
-	private static final String	SERVER															= "hgdownload.cse.ucsc.edu";
-
-	/**
-	 * Prefix string used for marking queries in cache database that identifies
-	 * list of reference genome versions by organism id.
-	 */
-	static final String					FILENAME_BY_ORGANISM_VERSION_PREFIX	= "ORGANISM_VERSION_FILE:";
-
-	/**
-	 * Default class logger.
-	 */
-	private Logger							logger															= Logger.getLogger(UcscReferenceGenomeConnector.class);
-
-	/**
-	 * Regex pattern that helps to find out organism names in source file.
-	 */
-	private Pattern							organismNamePattern									= Pattern.compile("<!--([A-Za-z\\-\\.\\ ]+)Downloads [=]+ -->");
-
-	/**
-	 * Regex pattern that helps to find out reference genome versions.
-	 */
-	private Pattern							organismDataUrlPattern							= Pattern.compile("\\/goldenPath\\/([A-Za-z0-9\\-\\.]+)\\/bigZips\\/");
-
-	/**
-	 * Access point to taxonomy information.
-	 */
-	@Autowired
-	private TaxonomyBackend			taxonomyBackend;
-
-	/**
-	 * Default constructor.
-	 */
-	public UcscReferenceGenomeConnector() {
-		super(UcscReferenceGenomeConnector.class);
-	}
-
-	@Override
-	public List<String> getDownloadedGenomeVersions(MiriamData organism) {
-		List<String> results = new ArrayList<>();
-
-		List<ReferenceGenome> genomes = getReferenceGenomeDao().getByType(ReferenceGenomeType.UCSC);
-		for (ReferenceGenome referenceGenome : genomes) {
-			if (referenceGenome.getOrganism().equals(organism)) {
-				results.add(referenceGenome.getVersion());
-			}
-		}
-		return results;
-	}
-
-	@Override
-	public List<String> getAvailableGenomeVersion(MiriamData organism) throws ReferenceGenomeConnectorException {
-		Set<String> ids = new HashSet<>();
-		try {
-			String content = getWebPageContent("http://hgdownload.cse.ucsc.edu/downloads.html");
-			Integer start = null;
-			Integer end = content.length();
-			Matcher matcher = organismNamePattern.matcher(content);
-			while (matcher.find()) {
-				String name = matcher.group(1).trim();
-				if (start != null) {
-					end = matcher.start();
-					break;
-				}
-
-				if (name.equalsIgnoreCase("Shared Data")) {
-					continue;
-				}
-				if (name.equalsIgnoreCase("liftOver File")) {
-					continue;
-				}
-				MiriamData taxonomy = taxonomyBackend.getByName(name);
-				if (organism.equals(taxonomy)) {
-					start = matcher.end();
-				}
-			}
-			// we haven't found a start point for our organism (organism couldn't be
-			// found in the list of available organisms)
-			if (start != null) {
-				String organismContent = content.substring(start, end);
-
-				matcher = organismDataUrlPattern.matcher(organismContent);
-				while (matcher.find()) {
-					String name = matcher.group(1).trim();
-					ids.add(name);
-				}
-			}
-
-		} catch (IOException | TaxonomySearchException e) {
-			throw new ReferenceGenomeConnectorException("Problem with accessing UCSC database", e);
-		}
-		List<String> result = new ArrayList<>();
-		result.addAll(ids);
-		Collections.sort(result, new Comparator<String>() {
-			public int compare(String o1, String o2) {
-				return extractInt(o2) - extractInt(o1);
-			}
-
-		});
-		return result;
-	}
-
-	@Override
-	public List<MiriamData> getAvailableOrganisms() throws ReferenceGenomeConnectorException {
-		try {
-			List<MiriamData> result = new ArrayList<>();
-			String content = getWebPageContent("http://hgdownload.cse.ucsc.edu/downloads.html");
-
-			Matcher matcher = organismNamePattern.matcher(content);
-			while (matcher.find()) {
-				String name = matcher.group(1).trim();
-				if (name.equalsIgnoreCase("Shared Data")) {
-					continue;
-				}
-				if (name.equalsIgnoreCase("liftOver File")) {
-					continue;
-				}
-				MiriamData taxonomy = taxonomyBackend.getByName(name);
-				if (taxonomy != null) {
-					result.add(taxonomy);
-				}
-			}
-
-			return result;
-		} catch (IOException | TaxonomySearchException e) {
-			throw new ReferenceGenomeConnectorException("Problem with accessing UCSC database", e);
-		}
-	}
-
-	@Override
-	public void downloadGenomeVersion(MiriamData organism, String version, IProgressUpdater updater, boolean async)
-			throws FileNotAvailableException, IOException, ReferenceGenomeConnectorException {
-		try {
-			downloadGenomeVersion(organism, version, updater, async, getGenomeVersionFile(organism, version));
-		} catch (URISyntaxException e) {
-			throw new InvalidStateException(e);
-		}
-	}
-
-	@Override
-	public Object refreshCacheQuery(Object query) throws SourceNotAvailable {
-		String result = null;
-		try {
-			if (query instanceof String) {
-				String name = (String) query;
-				if (name.startsWith("http")) {
-					result = getWebPageContent(name);
-				} else if (name.startsWith(FILENAME_BY_ORGANISM_VERSION_PREFIX)) {
-					String[] tmp = name.substring(FILENAME_BY_ORGANISM_VERSION_PREFIX.length()).split("\n");
-					result = getGenomeVersionFile(new MiriamData(MiriamType.TAXONOMY, tmp[0]), tmp[1]);
-				} else {
-					throw new InvalidArgumentException("Don't know what to do with string \"" + query + "\"");
-				}
-			} else {
-				throw new InvalidArgumentException("Don't know what to do with class: " + query.getClass());
-			}
-		} catch (FileNotAvailableException e) {
-			throw new SourceNotAvailable("Cannot find file for the query: " + query, e);
-		} catch (IOException e) {
-			throw new SourceNotAvailable(e);
-		}
-		return result;
-	}
-
-	@Override
-	public void removeGenomeVersion(MiriamData organism, String version) throws IOException {
-		List<ReferenceGenome> genomes = getReferenceGenomeDao().getByType(ReferenceGenomeType.UCSC);
-		for (ReferenceGenome referenceGenome : genomes) {
-			if (referenceGenome.getOrganism().equals(organism) && referenceGenome.getVersion().equals(version)) {
-				// removing file from big file cache might not be the best idea here
-				if (getBigFileCache().isCached(referenceGenome.getSourceUrl())) {
-					getBigFileCache().removeFile(referenceGenome.getSourceUrl());
-				}
-				for (ReferenceGenomeGeneMapping mapping : referenceGenome.getGeneMapping()) {
-					if (getBigFileCache().isCached(mapping.getSourceUrl())) {
-						getBigFileCache().removeFile(mapping.getSourceUrl());
-					}
-				}
-				getReferenceGenomeDao().delete(referenceGenome);
-			}
-		}
-	}
-
-	/**
-	 * @return the taxonomyBackend
-	 * @see #taxonomyBackend
-	 */
-	public TaxonomyBackend getTaxonomyBackend() {
-		return taxonomyBackend;
-	}
-
-	/**
-	 * @param taxonomyBackend
-	 *          the taxonomyBackend to set
-	 * @see #taxonomyBackend
-	 */
-	public void setTaxonomyBackend(TaxonomyBackend taxonomyBackend) {
-		this.taxonomyBackend = taxonomyBackend;
-	}
-
-	/**
-	 * Task that will be able to fetch genome file from ftp server.
-	 * 
-	 * @author Piotr Gawron
-	 *
-	 */
-	private final class DownloadGenomeVersionTask implements Callable<Void> {
-
-		/**
-		 * Url to the file that we want to download.
-		 * 
-		 */
-		private String					 url;
-
-		/**
-		 * Callback listener that will receive information about upload progress.
-		 * 
-		 */
-		private IProgressUpdater updater;
-
-		/**
-		 * Organism for which we want to fetch genome.
-		 */
-		private MiriamData			 organism;
-
-		/**
-		 * Version of the genome.
-		 */
-		private String					 version;
-
-		/**
-		 * Default constructor.
-		 * 
-		 * @param url
-		 *          {@link #url}
-		 * @param updater
-		 *          {@link #updater}
-		 * @param organism
-		 *          {@link #organism}
-		 * @param version
-		 *          {@link #version}
-		 */
-		private DownloadGenomeVersionTask(MiriamData organism, String version, String url, IProgressUpdater updater) {
-			this.url = url;
-			this.organism = organism;
-			this.version = version;
-			if (updater != null) {
-				this.updater = updater;
-			} else {
-				this.updater = new IProgressUpdater() {
-					@Override
-					public void setProgress(double progress) {
-					}
-				};
-			}
-		}
-
-		@Override
-		public Void call() throws Exception {
-			getDbUtils().createSessionForCurrentThread();
-			try {
-				ReferenceGenome referenceGenome = new ReferenceGenome();
-				referenceGenome.setOrganism(organism);
-				referenceGenome.setType(ReferenceGenomeType.UCSC);
-				referenceGenome.setVersion(version);
-				referenceGenome.setSourceUrl(url);
-				getReferenceGenomeDao().add(referenceGenome);
-				getReferenceGenomeDao().flush();
-				getReferenceGenomeDao().commit();
-				getDbUtils().closeSessionForCurrentThread();
-
-				getBigFileCache().downloadFile(url, false, new IProgressUpdater() {
-					@Override
-					public void setProgress(double progress) {
-						if (updater != null) {
-							updater.setProgress(progress);
-						}
-						// we have to get the object because it's in separate thred
-						ReferenceGenome temp = getReferenceGenomeDao().getById(referenceGenome.getId());
-						temp.setDownloadProgress(progress);
-						getReferenceGenomeDao().update(temp);
-						getReferenceGenomeDao().commit();
-					}
-				});
-				return null;
-			} finally {
-				getDbUtils().closeSessionForCurrentThread();
-			}
-		}
-
-	}
-
-	@Override
-	public void downloadGenomeVersion(MiriamData organism, String version, IProgressUpdater updater, boolean async, String customUrl)
-			throws IOException, URISyntaxException, ReferenceGenomeConnectorException {
-		Callable<Void> computations = new DownloadGenomeVersionTask(organism, version, customUrl, updater);
-		if (async) {
-			getAsyncExecutorService().submit(computations);
-		} else {
-			Future<Void> task = getSyncExecutorService().submit(computations);
-			executeTask(task);
-		}
-	}
-
-	/**
-	 * Returns local path on ftp server to folder with data about given organism
-	 * and version.
-	 * 
-	 * @param organism
-	 *          organism of reference genome
-	 * @param version
-	 *          of reference genome
-	 * @return local path on ftp server to folder with data about reference genome
-	 */
-	private String getGenomePath(MiriamData organism, String version) {
-		return "/goldenPath/" + version + "/bigZips/";
-	}
-
-	@Override
-	public String getGenomeVersionFile(MiriamData organism, String version) throws FileNotAvailableException {
-		String filename = super.getCacheValue(FILENAME_BY_ORGANISM_VERSION_PREFIX + organism.getResource() + "\n" + version);
-		if (filename != null) {
-			return filename;
-		}
-		FTPClient ftp = createFtpClient();
-		try {
-			ftp.connect(SERVER);
-			// After connection attempt, you should check the reply code to verify
-			// success.
-			int reply = ftp.getReplyCode();
-
-			if (!FTPReply.isPositiveCompletion(reply)) {
-				throw new FileNotAvailableException("Cannot find file with genome for: " + organism + "; " + version + ". FTP server refused connection.");
-			} else {
-				ftp.enterLocalPassiveMode();
-				ftp.login("anonymous", "");
-				ftp.setFileType(FTP.BINARY_FILE_TYPE);
-
-				String remotePath = getGenomePath(organism, version);
-				FTPFile[] files = ftp.listFiles(remotePath);
-				for (FTPFile ftpFile : files) {
-					if (ftpFile.getName().endsWith(".2bit")) {
-						if (filename != null) {
-							logger.warn("More than one 2bit file found in a folder: " + remotePath + ". Using first: " + filename);
-						} else {
-							filename = ftpFile.getName();
-						}
-					}
-				}
-				ftp.logout();
-			}
-		} catch (IOException e) {
-			throw new FileNotAvailableException(e);
-		} finally {
-			if (ftp.isConnected()) {
-				try {
-					ftp.disconnect();
-				} catch (IOException ioe) {
-					throw new FileNotAvailableException("Cannot find file with genome for: " + organism + "; " + version + ". Problem with ftp connection.", ioe);
-				}
-			}
-		}
-		if (filename == null) {
-			throw new FileNotAvailableException("Cannot find file with genome for: " + organism + "; " + version);
-		}
-		String result = "ftp://" + SERVER + getGenomePath(organism, version) + filename;
-		super.setCacheValue(FILENAME_BY_ORGANISM_VERSION_PREFIX + organism.getResource() + "\n" + version, result);
-		return result;
-	}
-
-	/**
-	 * Creates new instance of {@link FTPClient}.
-	 * 
-	 * @return new instance of {@link FTPClient}
-	 */
-	FTPClient createFtpClient() {
-		FTPClient ftp = new FTPClient();
-		return ftp;
-	}
-
-	/**
-	 * Extracts int from the version of the genome. The genome version look like
-	 * follow: xxxx011.
-	 * 
-	 * @param s
-	 *          genome version where suffix part is integer number that informs
-	 *          about version
-	 * @return {@link Integer} representing version of the genome from string that
-	 *         describes genome version (it contains also some letters characters}
-	 */
-	int extractInt(String s) {
-		int startIndex = 0;
-		int endIndex = s.length() - 2;
-		for (int i = 0; i < s.length(); i++) {
-			startIndex = i;
-			if (s.charAt(i) >= '0' && s.charAt(i) <= '9') {
-				break;
-			}
-		}
-		for (int i = startIndex; i < s.length(); i++) {
-			if (s.charAt(i) < '0' || s.charAt(i) > '9') {
-				break;
-			}
-			endIndex = i;
-		}
-		endIndex++;
-		if (startIndex >= endIndex) {
-			return 0;
-		} else {
-			return Integer.parseInt(s.substring(startIndex, endIndex));
-		}
-	}
-
-	@Override
-	protected WebPageDownloader getWebPageDownloader() {
-		return super.getWebPageDownloader();
-	}
-
-	@Override
-	protected void setWebPageDownloader(WebPageDownloader webPageDownloader) {
-		super.setWebPageDownloader(webPageDownloader);
-	}
+  /**
+   * Server domain name.
+   */
+  private static final String SERVER = "hgdownload.cse.ucsc.edu";
+
+  /**
+   * Prefix string used for marking queries in cache database that identifies list
+   * of reference genome versions by organism id.
+   */
+  static final String FILENAME_BY_ORGANISM_VERSION_PREFIX = "ORGANISM_VERSION_FILE:";
+
+  /**
+   * Default class logger.
+   */
+  private Logger logger = Logger.getLogger(UcscReferenceGenomeConnector.class);
+
+  /**
+   * Regex pattern that helps to find out organism names in source file.
+   */
+  private Pattern organismNamePattern = Pattern.compile("<!--([A-Za-z\\-\\.\\ ]+)Downloads [=]*[\\ ]*-->");
+
+  /**
+   * Regex pattern that helps to find out reference genome versions.
+   */
+  private Pattern organismDataUrlPattern = Pattern.compile("\\/goldenPath\\/([A-Za-z0-9\\-\\.]+)\\/bigZips\\/");
+
+  /**
+   * Access point to taxonomy information.
+   */
+  @Autowired
+  private TaxonomyBackend taxonomyBackend;
+
+  /**
+   * Default constructor.
+   */
+  public UcscReferenceGenomeConnector() {
+    super(UcscReferenceGenomeConnector.class);
+  }
+
+  @Override
+  public List<String> getDownloadedGenomeVersions(MiriamData organism) {
+    List<String> results = new ArrayList<>();
+
+    List<ReferenceGenome> genomes = getReferenceGenomeDao().getByType(ReferenceGenomeType.UCSC);
+    for (ReferenceGenome referenceGenome : genomes) {
+      if (referenceGenome.getOrganism().equals(organism)) {
+        results.add(referenceGenome.getVersion());
+      }
+    }
+    return results;
+  }
+
+  @Override
+  public List<String> getAvailableGenomeVersion(MiriamData organism) throws ReferenceGenomeConnectorException {
+    Set<String> ids = new HashSet<>();
+    try {
+      String content = getWebPageContent("http://hgdownload.cse.ucsc.edu/downloads.html");
+      Integer start = null;
+      Integer end = content.length();
+      Matcher matcher = organismNamePattern.matcher(content);
+      while (matcher.find()) {
+        String name = matcher.group(1).trim();
+        if (start != null) {
+          end = matcher.start();
+          break;
+        }
+
+        if (name.equalsIgnoreCase("Shared Data")) {
+          continue;
+        }
+        if (name.equalsIgnoreCase("liftOver File")) {
+          continue;
+        }
+        MiriamData taxonomy = taxonomyBackend.getByName(name);
+        if (organism.equals(taxonomy)) {
+          start = matcher.end();
+        }
+      }
+      // we haven't found a start point for our organism (organism couldn't be
+      // found in the list of available organisms)
+      if (start != null) {
+        String organismContent = content.substring(start, end);
+
+        matcher = organismDataUrlPattern.matcher(organismContent);
+        while (matcher.find()) {
+          String name = matcher.group(1).trim();
+          ids.add(name);
+        }
+      }
+
+    } catch (IOException | TaxonomySearchException e) {
+      throw new ReferenceGenomeConnectorException("Problem with accessing UCSC database", e);
+    }
+    List<String> result = new ArrayList<>();
+    result.addAll(ids);
+    Collections.sort(result, new Comparator<String>() {
+      public int compare(String o1, String o2) {
+        return extractInt(o2) - extractInt(o1);
+      }
+
+    });
+    return result;
+  }
+
+  @Override
+  public List<MiriamData> getAvailableOrganisms() throws ReferenceGenomeConnectorException {
+    try {
+      List<MiriamData> result = new ArrayList<>();
+      String content = getWebPageContent("http://hgdownload.cse.ucsc.edu/downloads.html");
+
+      Matcher matcher = organismNamePattern.matcher(content);
+      while (matcher.find()) {
+        String name = matcher.group(1).trim();
+        if (name.equalsIgnoreCase("Shared Data")) {
+          continue;
+        }
+        if (name.equalsIgnoreCase("liftOver File")) {
+          continue;
+        }
+        MiriamData taxonomy = taxonomyBackend.getByName(name);
+        if (taxonomy != null) {
+          result.add(taxonomy);
+        }
+      }
+
+      return result;
+    } catch (IOException | TaxonomySearchException e) {
+      throw new ReferenceGenomeConnectorException("Problem with accessing UCSC database", e);
+    }
+  }
+
+  @Override
+  public void downloadGenomeVersion(MiriamData organism, String version, IProgressUpdater updater, boolean async)
+      throws FileNotAvailableException, IOException, ReferenceGenomeConnectorException {
+    try {
+      downloadGenomeVersion(organism, version, updater, async, getGenomeVersionFile(organism, version));
+    } catch (URISyntaxException e) {
+      throw new InvalidStateException(e);
+    }
+  }
+
+  @Override
+  public Object refreshCacheQuery(Object query) throws SourceNotAvailable {
+    String result = null;
+    try {
+      if (query instanceof String) {
+        String name = (String) query;
+        if (name.startsWith("http")) {
+          result = getWebPageContent(name);
+        } else if (name.startsWith(FILENAME_BY_ORGANISM_VERSION_PREFIX)) {
+          String[] tmp = name.substring(FILENAME_BY_ORGANISM_VERSION_PREFIX.length()).split("\n");
+          result = getGenomeVersionFile(new MiriamData(MiriamType.TAXONOMY, tmp[0]), tmp[1]);
+        } else {
+          throw new InvalidArgumentException("Don't know what to do with string \"" + query + "\"");
+        }
+      } else {
+        throw new InvalidArgumentException("Don't know what to do with class: " + query.getClass());
+      }
+    } catch (FileNotAvailableException e) {
+      throw new SourceNotAvailable("Cannot find file for the query: " + query, e);
+    } catch (IOException e) {
+      throw new SourceNotAvailable(e);
+    }
+    return result;
+  }
+
+  @Override
+  public void removeGenomeVersion(MiriamData organism, String version) throws IOException {
+    List<ReferenceGenome> genomes = getReferenceGenomeDao().getByType(ReferenceGenomeType.UCSC);
+    for (ReferenceGenome referenceGenome : genomes) {
+      if (referenceGenome.getOrganism().equals(organism) && referenceGenome.getVersion().equals(version)) {
+        // removing file from big file cache might not be the best idea here
+        if (getBigFileCache().isCached(referenceGenome.getSourceUrl())) {
+          getBigFileCache().removeFile(referenceGenome.getSourceUrl());
+        }
+        for (ReferenceGenomeGeneMapping mapping : referenceGenome.getGeneMapping()) {
+          if (getBigFileCache().isCached(mapping.getSourceUrl())) {
+            getBigFileCache().removeFile(mapping.getSourceUrl());
+          }
+        }
+        getReferenceGenomeDao().delete(referenceGenome);
+      }
+    }
+  }
+
+  /**
+   * @return the taxonomyBackend
+   * @see #taxonomyBackend
+   */
+  public TaxonomyBackend getTaxonomyBackend() {
+    return taxonomyBackend;
+  }
+
+  /**
+   * @param taxonomyBackend
+   *          the taxonomyBackend to set
+   * @see #taxonomyBackend
+   */
+  public void setTaxonomyBackend(TaxonomyBackend taxonomyBackend) {
+    this.taxonomyBackend = taxonomyBackend;
+  }
+
+  /**
+   * Task that will be able to fetch genome file from ftp server.
+   * 
+   * @author Piotr Gawron
+   *
+   */
+  private final class DownloadGenomeVersionTask implements Callable<Void> {
+
+    /**
+     * Url to the file that we want to download.
+     * 
+     */
+    private String url;
+
+    /**
+     * Callback listener that will receive information about upload progress.
+     * 
+     */
+    private IProgressUpdater updater;
+
+    /**
+     * Organism for which we want to fetch genome.
+     */
+    private MiriamData organism;
+
+    /**
+     * Version of the genome.
+     */
+    private String version;
+
+    /**
+     * Default constructor.
+     * 
+     * @param url
+     *          {@link #url}
+     * @param updater
+     *          {@link #updater}
+     * @param organism
+     *          {@link #organism}
+     * @param version
+     *          {@link #version}
+     */
+    private DownloadGenomeVersionTask(MiriamData organism, String version, String url, IProgressUpdater updater) {
+      this.url = url;
+      this.organism = organism;
+      this.version = version;
+      if (updater != null) {
+        this.updater = updater;
+      } else {
+        this.updater = new IProgressUpdater() {
+          @Override
+          public void setProgress(double progress) {
+          }
+        };
+      }
+    }
+
+    @Override
+    public Void call() throws Exception {
+      getDbUtils().createSessionForCurrentThread();
+      try {
+        ReferenceGenome referenceGenome = new ReferenceGenome();
+        referenceGenome.setOrganism(organism);
+        referenceGenome.setType(ReferenceGenomeType.UCSC);
+        referenceGenome.setVersion(version);
+        referenceGenome.setSourceUrl(url);
+        getReferenceGenomeDao().add(referenceGenome);
+        getReferenceGenomeDao().flush();
+        getReferenceGenomeDao().commit();
+        getDbUtils().closeSessionForCurrentThread();
+
+        getBigFileCache().downloadFile(url, false, new IProgressUpdater() {
+          @Override
+          public void setProgress(double progress) {
+            if (updater != null) {
+              updater.setProgress(progress);
+            }
+            // we have to get the object because it's in separate thred
+            ReferenceGenome temp = getReferenceGenomeDao().getById(referenceGenome.getId());
+            temp.setDownloadProgress(progress);
+            getReferenceGenomeDao().update(temp);
+            getReferenceGenomeDao().commit();
+          }
+        });
+        return null;
+      } finally {
+        getDbUtils().closeSessionForCurrentThread();
+      }
+    }
+
+  }
+
+  @Override
+  public void downloadGenomeVersion(MiriamData organism, String version, IProgressUpdater updater, boolean async,
+      String customUrl) throws IOException, URISyntaxException, ReferenceGenomeConnectorException {
+    Callable<Void> computations = new DownloadGenomeVersionTask(organism, version, customUrl, updater);
+    if (async) {
+      getAsyncExecutorService().submit(computations);
+    } else {
+      Future<Void> task = getSyncExecutorService().submit(computations);
+      executeTask(task);
+    }
+  }
+
+  /**
+   * Returns local path on ftp server to folder with data about given organism and
+   * version.
+   * 
+   * @param organism
+   *          organism of reference genome
+   * @param version
+   *          of reference genome
+   * @return local path on ftp server to folder with data about reference genome
+   */
+  private String getGenomePath(MiriamData organism, String version) {
+    return "/goldenPath/" + version + "/bigZips/";
+  }
+
+  @Override
+  public String getGenomeVersionFile(MiriamData organism, String version) throws FileNotAvailableException {
+    String filename = super.getCacheValue(
+        FILENAME_BY_ORGANISM_VERSION_PREFIX + organism.getResource() + "\n" + version);
+    if (filename != null) {
+      return filename;
+    }
+    FTPClient ftp = createFtpClient();
+    try {
+      ftp.connect(SERVER);
+      // After connection attempt, you should check the reply code to verify
+      // success.
+      int reply = ftp.getReplyCode();
+
+      if (!FTPReply.isPositiveCompletion(reply)) {
+        throw new FileNotAvailableException(
+            "Cannot find file with genome for: " + organism + "; " + version + ". FTP server refused connection.");
+      } else {
+        ftp.enterLocalPassiveMode();
+        ftp.login("anonymous", "");
+        ftp.setFileType(FTP.BINARY_FILE_TYPE);
+
+        String remotePath = getGenomePath(organism, version);
+        FTPFile[] files = ftp.listFiles(remotePath);
+        for (FTPFile ftpFile : files) {
+          if (ftpFile.getName().endsWith(".2bit")) {
+            if (filename != null) {
+              logger.warn("More than one 2bit file found in a folder: " + remotePath + ". Using first: " + filename);
+            } else {
+              filename = ftpFile.getName();
+            }
+          }
+        }
+        ftp.logout();
+      }
+    } catch (IOException e) {
+      throw new FileNotAvailableException(e);
+    } finally {
+      if (ftp.isConnected()) {
+        try {
+          ftp.disconnect();
+        } catch (IOException ioe) {
+          throw new FileNotAvailableException(
+              "Cannot find file with genome for: " + organism + "; " + version + ". Problem with ftp connection.", ioe);
+        }
+      }
+    }
+    if (filename == null) {
+      throw new FileNotAvailableException("Cannot find file with genome for: " + organism + "; " + version);
+    }
+    String result = "ftp://" + SERVER + getGenomePath(organism, version) + filename;
+    super.setCacheValue(FILENAME_BY_ORGANISM_VERSION_PREFIX + organism.getResource() + "\n" + version, result);
+    return result;
+  }
+
+  /**
+   * Creates new instance of {@link FTPClient}.
+   * 
+   * @return new instance of {@link FTPClient}
+   */
+  FTPClient createFtpClient() {
+    FTPClient ftp = new FTPClient();
+    return ftp;
+  }
+
+  /**
+   * Extracts int from the version of the genome. The genome version look like
+   * follow: xxxx011.
+   * 
+   * @param s
+   *          genome version where suffix part is integer number that informs
+   *          about version
+   * @return {@link Integer} representing version of the genome from string that
+   *         describes genome version (it contains also some letters characters}
+   */
+  int extractInt(String s) {
+    int startIndex = 0;
+    int endIndex = s.length() - 2;
+    for (int i = 0; i < s.length(); i++) {
+      startIndex = i;
+      if (s.charAt(i) >= '0' && s.charAt(i) <= '9') {
+        break;
+      }
+    }
+    for (int i = startIndex; i < s.length(); i++) {
+      if (s.charAt(i) < '0' || s.charAt(i) > '9') {
+        break;
+      }
+      endIndex = i;
+    }
+    endIndex++;
+    if (startIndex >= endIndex) {
+      return 0;
+    } else {
+      return Integer.parseInt(s.substring(startIndex, endIndex));
+    }
+  }
+
+  @Override
+  protected WebPageDownloader getWebPageDownloader() {
+    return super.getWebPageDownloader();
+  }
+
+  @Override
+  protected void setWebPageDownloader(WebPageDownloader webPageDownloader) {
+    super.setWebPageDownloader(webPageDownloader);
+  }
 
 }
diff --git a/annotation/src/test/java/lcsb/mapviewer/annotation/services/genome/UcscReferenceGenomeConnectorTest.java b/annotation/src/test/java/lcsb/mapviewer/annotation/services/genome/UcscReferenceGenomeConnectorTest.java
index 5ff31b7460..0ae6336559 100644
--- a/annotation/src/test/java/lcsb/mapviewer/annotation/services/genome/UcscReferenceGenomeConnectorTest.java
+++ b/annotation/src/test/java/lcsb/mapviewer/annotation/services/genome/UcscReferenceGenomeConnectorTest.java
@@ -49,921 +49,926 @@ import lcsb.mapviewer.persist.dao.map.layout.ReferenceGenomeDao;
 
 public class UcscReferenceGenomeConnectorTest extends AnnotationTestFunctions {
 
-	Logger											 logger	= Logger.getLogger(UcscReferenceGenomeConnectorTest.class);
-
-	@Autowired
-	UcscReferenceGenomeConnector connector;
-
-	@Autowired
-	ReferenceGenomeDao					 referenceGenomeDao;
-
-	@Before
-	public void setUp() throws Exception {
-	}
-
-	@After
-	public void tearDown() throws Exception {
-	}
-
-	@Test
-	public void testRefreshCacheQuery() throws Exception {
-		try {
-			assertNotNull(connector.refreshCacheQuery("http://google.pl/"));
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGetAvailableGenomeVersion() throws Exception {
-		MiriamData human = new MiriamData(MiriamType.TAXONOMY, "9606");
-		try {
-			List<String> list = connector.getAvailableGenomeVersion(human);
-			assertTrue(list.size() >= 17);
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGetAvailableGenomeVersionWhenProblemWithTaxonomyServer() throws Exception {
-		MiriamData human = new MiriamData(MiriamType.TAXONOMY, "9606");
-		TaxonomyBackend taxonomyBackend = connector.getTaxonomyBackend();
-		try {
-			// simulate problems with taxonomy server
-			TaxonomyBackend taxonomyMock = Mockito.mock(TaxonomyBackend.class);
-			when(taxonomyMock.getByName(anyString())).thenThrow(new TaxonomySearchException(null, null));
-			connector.setTaxonomyBackend(taxonomyMock);
-
-			connector.getAvailableGenomeVersion(human);
-			fail("Exception expected");
-		} catch (ReferenceGenomeConnectorException e) {
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		} finally {
-			connector.setTaxonomyBackend(taxonomyBackend);
-		}
-	}
-
-	@Test
-	public void testGetAvailableGenomeVersionForInvalidTaxonomy() throws Exception {
-		MiriamData unknown = new MiriamData(MiriamType.TAXONOMY, "10101010101");
-		try {
-			List<String> list = connector.getAvailableGenomeVersion(unknown);
-			assertEquals(0, list.size());
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGetAvailableOrganisms() throws Exception {
-		try {
-			MiriamData human = new MiriamData(MiriamType.TAXONOMY, "9606");
-			MiriamData chicken = new MiriamData(MiriamType.TAXONOMY, "9031");
-			List<MiriamData> list = connector.getAvailableOrganisms();
-			assertTrue(list.size() > 40);
-
-			assertTrue(list.contains(human));
-			assertTrue(list.contains(chicken));
-
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGetAvailableOrganismsWhenUcscWebpageDown() throws Exception {
-		WebPageDownloader downloader = connector.getWebPageDownloader();
-		GeneralCacheInterface originalCache = connector.getCache();
-		try {
-			// exclude first cached value
-			connector.setCache(null);
-
-			WebPageDownloader mockDownloader = Mockito.mock(WebPageDownloader.class);
-			when(mockDownloader.getFromNetwork(anyString(), anyString(), anyString())).thenThrow(new IOException());
-			connector.setWebPageDownloader(mockDownloader);
-			connector.getAvailableOrganisms();
-			fail("Exception expected");
-		} catch (ReferenceGenomeConnectorException e) {
-			assertTrue(e.getMessage().contains("Problem with accessing UCSC database"));
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		} finally {
-			connector.setWebPageDownloader(downloader);
-			connector.setCache(originalCache);
-		}
-	}
-
-	@Test
-	public void testDownloadGenomeVersion() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "1570291");
-		String version = "eboVir3";
-
-		try {
-			dbUtils.createSessionForCurrentThread();
-			List<String> list = connector.getDownloadedGenomeVersions(yeast);
-			if (list.contains(version)) {
-				connector.removeGenomeVersion(yeast, version);
-			}
-			dbUtils.closeSessionForCurrentThread();
-			list = connector.getDownloadedGenomeVersions(yeast);
-
-			assertFalse(list.contains(version));
-
-			connector.downloadGenomeVersion(yeast, version, new IProgressUpdater() {
-				@Override
-				public void setProgress(double progress) {
-				}
-			}, false);
-
-			list = connector.getDownloadedGenomeVersions(yeast);
-			assertTrue(list.contains(version));
-
-			List<ReferenceGenome> genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
-			ReferenceGenome genome = null;
-			for (ReferenceGenome referenceGenome : genomes) {
-				if (referenceGenome.getVersion().equals(version)) {
-					genome = referenceGenome;
-				}
-			}
-			assertNotNull(genome);
-			connector.downloadGeneMappingGenomeVersion(genome, "test", null, false, "http://www.biodalliance.org/datasets/flyThickets.bb");
-
-			dbUtils.createSessionForCurrentThread();
-			genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
-			genome = null;
-			for (ReferenceGenome referenceGenome : genomes) {
-				if (referenceGenome.getVersion().equals(version)) {
-					genome = referenceGenome;
-				}
-			}
-			assertNotNull(genome);
-			assertEquals(1, genome.getGeneMapping().size());
-			dbUtils.closeSessionForCurrentThread();
-
-			try {
-				// try to download the same thing for the second time
-				connector.downloadGeneMappingGenomeVersion(genome, "test", new IProgressUpdater() {
-					@Override
-					public void setProgress(double progress) {
-					}
-				}, false, "http://www.biodalliance.org/datasets/flyThickets.bb");
-				fail("Exception expected");
-			} catch (ReferenceGenomeConnectorException e) {
-				assertTrue(e.getMessage().contains("already exists"));
-			}
-
-			try {
-				// try to download something that is not big bed
-				connector.downloadGeneMappingGenomeVersion(genome, "test2", new IProgressUpdater() {
-					@Override
-					public void setProgress(double progress) {
-					}
-				}, false, "http://www.biodalliance.org/datasets/flyThickets.txt");
-				fail("Exception expected");
-			} catch (ReferenceGenomeConnectorException e) {
-				assertTrue(e.getMessage().contains("Only big bed format files are supported"));
-			}
-
-//			connector.removeGeneMapping(genome.getGeneMapping().get(0));
-
-			int warningCount = getWarnings().size();
-
-			// try to remove invalid mapping
-			connector.removeGeneMapping(new ReferenceGenomeGeneMapping());
-
-			assertEquals(warningCount + 1, getWarnings().size());
-
-			dbUtils.createSessionForCurrentThread();
-			connector.removeGenomeVersion(yeast, version);
-			dbUtils.closeSessionForCurrentThread();
-
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@SuppressWarnings("unchecked")
-	@Test
-	public void testDownloadGenomeVersionWhenInternalUrlIsInvalid() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "1570291");
-		String version = "eboVir3";
-
-		ExecutorService originalExecutorService = connector.getSyncExecutorService();
-		try {
-			ExecutorService executorService = Mockito.mock(ExecutorService.class);
-			connector.setSyncExecutorService(executorService);
-			Future<?>  task = Mockito.mock(Future.class);
-			when(task.get()).thenThrow(new ExecutionException(new URISyntaxException("", "")));
-			when(executorService.submit(any(Callable.class))).thenReturn(task);
-
-			connector.downloadGenomeVersion(yeast, version, new IProgressUpdater() {
-				@Override
-				public void setProgress(double progress) {
-				}
-			}, false);
-			fail("Exception expected");
-
-		} catch (InvalidStateException e) {
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		} finally {
-			connector.setSyncExecutorService(originalExecutorService);
-		}
-	}
-
-	@Test
-	public void testDownloadGenomeVersionAsync() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "1570291");
-		String version = "eboVir3";
-
-		BigFileCache bigFileCache = connector.getBigFileCache();
-		try {
-			dbUtils.createSessionForCurrentThread();
-			List<String> list = connector.getDownloadedGenomeVersions(yeast);
-			if (list.contains(version)) {
-				connector.removeGenomeVersion(yeast, version);
-			}
-			list = connector.getDownloadedGenomeVersions(yeast);
-
-			dbUtils.closeSessionForCurrentThread();
-
-			// create a mock for ftp connections (so we speed up tests), the real
-			// connection is tested elsewhere
-			BigFileCache mockCache = Mockito.mock(BigFileCache.class);
-			Mockito.doAnswer(new Answer<Void>() {
-
-				@Override
-				public Void answer(InvocationOnMock invocation) throws Throwable {
-					IProgressUpdater updater = (IProgressUpdater) invocation.getArguments()[2];
-					updater.setProgress(100);
-					return null;
-				}
-			}).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
-
-			connector.setBigFileCache(mockCache);
-
-			assertFalse(list.contains(version));
-
-			connector.downloadGenomeVersion(yeast, version, null, true);
-
-			waitForDownload();
-
-			dbUtils.createSessionForCurrentThread();
-			list = connector.getDownloadedGenomeVersions(yeast);
-			assertTrue(list.contains(version));
-
-			List<ReferenceGenome> genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
-			ReferenceGenome genome = null;
-			for (ReferenceGenome referenceGenome : genomes) {
-				if (referenceGenome.getVersion().equals(version)) {
-					genome = referenceGenome;
-				}
-			}
-			dbUtils.closeSessionForCurrentThread();
-			assertNotNull(genome);
-			connector.downloadGeneMappingGenomeVersion(genome, "test", new IProgressUpdater() {
-				@Override
-				public void setProgress(double progress) {
-				}
-			}, true, "http://www.biodalliance.org/datasets/flyThickets.bb");
-
-			waitForDownload();
-
-			genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
-			genome = null;
-			for (ReferenceGenome referenceGenome : genomes) {
-				if (referenceGenome.getVersion().equals(version)) {
-					genome = referenceGenome;
-				}
-			}
-			assertNotNull(genome);
-			referenceGenomeDao.refresh(genome);
-			assertEquals(1, genome.getGeneMapping().size());
-
-			try {
-				// try to download the same thing for the second time
-				connector.downloadGeneMappingGenomeVersion(genome, "test", new IProgressUpdater() {
-					@Override
-					public void setProgress(double progress) {
-					}
-				}, false, "http://www.biodalliance.org/datasets/flyThickets.bb");
-				fail("Exception expected");
-			} catch (ReferenceGenomeConnectorException e) {
-				assertTrue(e.getMessage().contains("already exists"));
-			}
-
-			try {
-				// try to download something that is not big bed
-				connector.downloadGeneMappingGenomeVersion(genome, "test2", new IProgressUpdater() {
-					@Override
-					public void setProgress(double progress) {
-					}
-				}, false, "http://www.biodalliance.org/datasets/flyThickets.txt");
-				fail("Exception expected");
-			} catch (ReferenceGenomeConnectorException e) {
-				assertTrue(e.getMessage().contains("Only big bed format files are supported"));
-			}
-
-			connector.removeGeneMapping(genome.getGeneMapping().get(0));
-
-			int warningCount = getWarnings().size();
-
-			// try to remove invalid mapping
-			connector.removeGeneMapping(new ReferenceGenomeGeneMapping());
-
-			assertEquals(warningCount + 1, getWarnings().size());
-
-			connector.removeGenomeVersion(yeast, version);
-
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		} finally {
-			connector.setBigFileCache(bigFileCache);
-		}
-	}
-
-	@Test
-	public void testDownloadGenomeVersionAsyncWithException() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "1570291");
-		String version = "eboVir3";
-
-		BigFileCache bigFileCache = connector.getBigFileCache();
-		try {
-			dbUtils.createSessionForCurrentThread();
-			List<String> list = connector.getDownloadedGenomeVersions(yeast);
-			if (list.contains(version)) {
-				connector.removeGenomeVersion(yeast, version);
-			}
-			list = connector.getDownloadedGenomeVersions(yeast);
-
-			dbUtils.closeSessionForCurrentThread();
-
-			// create a mock for ftp connections (so we speed up tests), the real
-			// connection is tested elsewhere
-			BigFileCache mockCache = Mockito.mock(BigFileCache.class);
-
-			connector.setBigFileCache(mockCache);
-
-			assertFalse(list.contains(version));
-
-			connector.downloadGenomeVersion(yeast, version, new IProgressUpdater() {
-				@Override
-				public void setProgress(double progress) {
-				}
-			}, true);
-
-			waitForDownload();
-
-			dbUtils.createSessionForCurrentThread();
-			list = connector.getDownloadedGenomeVersions(yeast);
-			assertTrue(list.contains(version));
-
-			List<ReferenceGenome> genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
-			ReferenceGenome genome = null;
-			for (ReferenceGenome referenceGenome : genomes) {
-				if (referenceGenome.getVersion().equals(version)) {
-					genome = referenceGenome;
-				}
-			}
-			dbUtils.closeSessionForCurrentThread();
-			assertNotNull(genome);
-
-			Mockito.doThrow(new IOException()).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
-			assertEquals(0, getErrors().size());
-			connector.downloadGeneMappingGenomeVersion(genome, "test", new IProgressUpdater() {
-				@Override
-				public void setProgress(double progress) {
-				}
-			}, true, "http://www.biodalliance.org/datasets/flyThickets.bb");
-
-			waitForDownload();
-
-			logger.debug(getErrors());
-			// during this downlad we expect exception thrown by downloader
-			assertEquals(1, getErrors().size());
-
-			Mockito.doThrow(new OutOfMemoryError()).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
-			connector.downloadGeneMappingGenomeVersion(genome, "test2", new IProgressUpdater() {
-				@Override
-				public void setProgress(double progress) {
-				}
-			}, true, "http://www.biodalliance.org/datasets/flyThickets.bb");
-
-			waitForDownload();
-			// during this downlad we expect error, this shouldn't change list of
-			// errors
-			assertEquals(1, getErrors().size());
-
-			Mockito.doThrow(new URISyntaxException("", "")).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
-			connector.downloadGeneMappingGenomeVersion(genome, "test3", new IProgressUpdater() {
-				@Override
-				public void setProgress(double progress) {
-				}
-			}, true, "http://www.biodalliance.org/datasets/flyThickets.bb");
-
-			waitForDownload();
-
-			// during this downlad we expect error (exception thrown by downloader)
-			assertEquals(2, getErrors().size());
-
-			genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
-			genome = null;
-			for (ReferenceGenome referenceGenome : genomes) {
-				if (referenceGenome.getVersion().equals(version)) {
-					genome = referenceGenome;
-				}
-			}
-			assertNotNull(genome);
-			referenceGenomeDao.refresh(genome);
-			assertEquals(3, genome.getGeneMapping().size());
-
-			connector.removeGeneMapping(genome.getGeneMapping().get(2));
-			connector.removeGeneMapping(genome.getGeneMapping().get(1));
-			connector.removeGeneMapping(genome.getGeneMapping().get(0));
-			connector.removeGenomeVersion(yeast, version);
-
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		} finally {
-			connector.setBigFileCache(bigFileCache);
-		}
-	}
-
-	@Test
-	public void testDownloadGenomeVersionWithException() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "1570291");
-		String version = "eboVir3";
-
-		BigFileCache bigFileCache = connector.getBigFileCache();
-		try {
-			List<String> list = connector.getDownloadedGenomeVersions(yeast);
-			if (list.contains(version)) {
-				connector.removeGenomeVersion(yeast, version);
-			}
-			list = connector.getDownloadedGenomeVersions(yeast);
-
-			// create a mock for ftp connections (so we speed up tests), the real
-			// connection is tested elsewhere
-			BigFileCache mockCache = Mockito.mock(BigFileCache.class);
-
-			connector.setBigFileCache(mockCache);
-
-			assertFalse(list.contains(version));
-
-			connector.downloadGenomeVersion(yeast, version, new IProgressUpdater() {
-				@Override
-				public void setProgress(double progress) {
-				}
-			}, false);
-
-			list = connector.getDownloadedGenomeVersions(yeast);
-			assertTrue(list.contains(version));
-
-			List<ReferenceGenome> genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
-			ReferenceGenome genome = null;
-			for (ReferenceGenome referenceGenome : genomes) {
-				if (referenceGenome.getVersion().equals(version)) {
-					genome = referenceGenome;
-				}
-			}
-			assertNotNull(genome);
-
-			Mockito.doThrow(new IOException()).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
-			assertEquals(0, getErrors().size());
-			try {
-				connector.downloadGeneMappingGenomeVersion(genome, "test", new IProgressUpdater() {
-					@Override
-					public void setProgress(double progress) {
-					}
-				}, false, "http://www.biodalliance.org/datasets/flyThickets.bb");
-				fail("Exception expected");
-			} catch (IOException e) {
-
-			}
-
-			Mockito.doThrow(new RuntimeException()).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
-			try {
-				connector.downloadGeneMappingGenomeVersion(genome, "test2", new IProgressUpdater() {
-					@Override
-					public void setProgress(double progress) {
-					}
-				}, false, "http://www.biodalliance.org/datasets/flyThickets.bb");
-
-				fail("Exception expected");
-			} catch (InvalidStateException e) {
-
-			}
-
-			Mockito.doThrow(new URISyntaxException("", "")).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
-
-			try {
-				connector.downloadGeneMappingGenomeVersion(genome, "test3", new IProgressUpdater() {
-					@Override
-					public void setProgress(double progress) {
-					}
-				}, false, "http://www.biodalliance.org/datasets/flyThickets.bb");
-				fail("Exception expected");
-			} catch (URISyntaxException e) {
-
-			}
-
-			genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
-			genome = null;
-			for (ReferenceGenome referenceGenome : genomes) {
-				if (referenceGenome.getVersion().equals(version)) {
-					genome = referenceGenome;
-				}
-			}
-			assertNotNull(genome);
-			referenceGenomeDao.refresh(genome);
-			assertEquals(3, genome.getGeneMapping().size());
-
-			connector.removeGeneMapping(genome.getGeneMapping().get(2));
-			connector.removeGeneMapping(genome.getGeneMapping().get(1));
-			connector.removeGeneMapping(genome.getGeneMapping().get(0));
-			connector.removeGenomeVersion(yeast, version);
-
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		} finally {
-			connector.setBigFileCache(bigFileCache);
-		}
-	}
-
-	@Test
-	public void testGetGenomeVersionFile() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
-		String version = "sacCer3";
-		try {
-			String url = connector.getGenomeVersionFile(yeast, version);
-			assertNotNull(url);
-			url.contains("sacCer3.2bit");
-			url.contains("ftp");
-
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGetUnknownGenomeVersionFile() {
-		MiriamData human = new MiriamData(MiriamType.TAXONOMY, "9606");
-		String version = "hg8";
-		try {
-			connector.getGenomeVersionFile(human, version);
-			fail("Exception expected");
-		} catch (FileNotAvailableException e) {
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testRefreshInvalid() throws Exception {
-		try {
-			connector.refreshCacheQuery("invalid query");
-			fail("Exception expected");
-		} catch (InvalidArgumentException e) {
-			assertTrue(e.getMessage().contains("Don't know what to do"));
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testRefreshInvalid2() throws Exception {
-		try {
-			connector.refreshCacheQuery(new Object());
-			fail("Exception expected");
-		} catch (InvalidArgumentException e) {
-			assertTrue(e.getMessage().contains("Don't know what to do"));
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testRefresh() throws Exception {
-		try {
-			MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
-			String version = "sacCer3";
-
-			String query = UcscReferenceGenomeConnector.FILENAME_BY_ORGANISM_VERSION_PREFIX + yeast.getResource() + "\n" + version;
-			Object res = connector.refreshCacheQuery(query);
-			assertNotNull(res);
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testRefreshUnknownFile() throws Exception {
-		try {
-			MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
-			String version = "unkVer";
-
-			String query = UcscReferenceGenomeConnector.FILENAME_BY_ORGANISM_VERSION_PREFIX + yeast.getResource() + "\n" + version;
-			connector.refreshCacheQuery(query);
-		} catch (SourceNotAvailable e) {
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGetters() throws Exception {
-		try {
-			UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector();
-			connectorUnderTest.setBigFileCache(connector.getBigFileCache());
-			connectorUnderTest.setDbUtils(connector.getDbUtils());
-			connectorUnderTest.setReferenceGenomeDao(connector.getReferenceGenomeDao());
-			connectorUnderTest.setReferenceGenomeGeneMappingDao(connector.getReferenceGenomeGeneMappingDao());
-
-			assertEquals(connector.getBigFileCache(), connectorUnderTest.getBigFileCache());
-			assertEquals(connector.getDbUtils(), connectorUnderTest.getDbUtils());
-			assertEquals(connector.getReferenceGenomeDao(), connectorUnderTest.getReferenceGenomeDao());
-			assertEquals(connector.getReferenceGenomeGeneMappingDao(), connectorUnderTest.getReferenceGenomeGeneMappingDao());
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGenomeVersionExtractorForInvalidGenome() throws Exception {
-		try {
-			int version = connector.extractInt("");
-			assertEquals(0, version);
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGenomeVersionExtractorForInvalidGenome2() throws Exception {
-		try {
-			int version = connector.extractInt("x");
-			assertEquals(0, version);
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGenomeVersionExtractorForValid() throws Exception {
-		try {
-			int version = connector.extractInt("x1");
-			assertEquals(1, version);
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGenomeVersionExtractorForValid2() throws Exception {
-		try {
-			int version = connector.extractInt("xy12z");
-			assertEquals(12, version);
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testRefreshCacheQueryNotAvailable() throws Exception {
-		WebPageDownloader downloader = connector.getWebPageDownloader();
-		GeneralCacheInterface originalCache = connector.getCache();
-		try {
-			// exclude first cached value
-			connector.setCache(null);
-			WebPageDownloader mockDownloader = Mockito.mock(WebPageDownloader.class);
-			when(mockDownloader.getFromNetwork(anyString(), anyString(), anyString())).thenThrow(new IOException());
-			connector.setWebPageDownloader(mockDownloader);
-			connector.refreshCacheQuery("http://google.pl/");
-			fail("Exception expected");
-		} catch (SourceNotAvailable e) {
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		} finally {
-			connector.setWebPageDownloader(downloader);
-			connector.setCache(originalCache);
-		}
-	}
-
-	@Test
-	public void testRefreshCacheQueryWhenFtpConnectionFails() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
-		String version = "sacCer3";
-
-		UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
-			@Override
-			FTPClient createFtpClient() {
-				FTPClient mockClient = Mockito.mock(FTPClient.class);
-				Mockito.doReturn(FTPReply.REQUEST_DENIED).when(mockClient).getReplyCode();
-				return mockClient;
-			}
-		};
-
-		try {
-			// exclude first cached value
-			connectorUnderTest.getGenomeVersionFile(yeast, version);
-			fail("Exception expected");
-		} catch (FileNotAvailableException e) {
-			assertTrue(e.getMessage().contains("FTP server refused connection"));
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGetGenomeVersionFileWhenFtpConnectionFails2() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
-		String version = "sacCer3";
-
-		UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
-			@Override
-			FTPClient createFtpClient() {
-				FTPClient mockClient = Mockito.mock(FTPClient.class);
-				try {
-					Mockito.doThrow(new IOException()).when(mockClient).connect(anyString());
-					Mockito.doReturn(false).when(mockClient).isConnected();
-				} catch (Exception e) {
-					fail();
-				}
-				return mockClient;
-			}
-		};
-
-		try {
-			// exclude first cached value
-			connectorUnderTest.getGenomeVersionFile(yeast, version);
-			fail("Exception expected");
-		} catch (FileNotAvailableException e) {
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGetGenomeVersionFileWhenFtpConnectionFails() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
-		String version = "sacCer3";
-
-		UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
-			@Override
-			FTPClient createFtpClient() {
-				FTPClient mockClient = Mockito.mock(FTPClient.class);
-				try {
-					Mockito.doThrow(new IOException()).when(mockClient).connect(anyString());
-					Mockito.doReturn(true).when(mockClient).isConnected();
-				} catch (Exception e) {
-					fail();
-				}
-				return mockClient;
-			}
-		};
-
-		try {
-			// exclude first cached value
-			connectorUnderTest.getGenomeVersionFile(yeast, version);
-			fail("Exception expected");
-		} catch (FileNotAvailableException e) {
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGetGenomeVersionFileWhenFtpConnectionFails3() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
-		String version = "sacCer3";
-
-		UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
-			@Override
-			FTPClient createFtpClient() {
-				FTPClient mockClient = Mockito.mock(FTPClient.class);
-				try {
-					Mockito.doThrow(new IOException()).when(mockClient).connect(anyString());
-					Mockito.doReturn(true).when(mockClient).isConnected();
-					Mockito.doThrow(new IOException()).when(mockClient).disconnect();
-				} catch (Exception e) {
-					fail();
-				}
-				return mockClient;
-			}
-		};
-
-		try {
-			// exclude first cached value
-			connectorUnderTest.getGenomeVersionFile(yeast, version);
-			fail("Exception expected");
-		} catch (FileNotAvailableException e) {
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGetGenomeVersionFileWhenFtpDisconnectConnectionFails() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
-		String version = "sacCer3";
-
-		UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
-			@Override
-			FTPClient createFtpClient() {
-				FTPClient mockClient = Mockito.mock(FTPClient.class);
-				try {
-					Mockito.doThrow(new IOException()).when(mockClient).disconnect();
-					Mockito.doReturn(new FTPFile[] {}).when(mockClient).listFiles(anyString());
-					Mockito.doReturn(FTPReply.COMMAND_OK).when(mockClient).getReplyCode();
-					Mockito.doReturn(true).when(mockClient).isConnected();
-				} catch (Exception e) {
-					fail();
-				}
-				return mockClient;
-			}
-		};
-
-		try {
-			// exclude first cached value
-			connectorUnderTest.getGenomeVersionFile(yeast, version);
-			fail("Exception expected");
-		} catch (FileNotAvailableException e) {
-			assertTrue(e.getMessage().contains("Problem with ftp connection"));
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	@Test
-	public void testGetGenomeVersionFileWithMore2bitFiles() throws Exception {
-		MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
-		String version = "sacCer3";
-
-		UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
-			@Override
-			FTPClient createFtpClient() {
-				FTPClient mockClient = Mockito.mock(FTPClient.class);
-				try {
-					FTPFile file1 = new FTPFile();
-					file1.setName("1.2bit");
-					FTPFile file2 = new FTPFile();
-					file2.setName("2.2bit");
-					FTPFile[] files = new FTPFile[] { file1, file2 };
-
-					Mockito.doReturn(files).when(mockClient).listFiles(anyString());
-					Mockito.doReturn(FTPReply.COMMAND_OK).when(mockClient).getReplyCode();
-				} catch (Exception e) {
-					fail();
-				}
-				return mockClient;
-			}
-		};
-
-		try {
-			// exclude first cached value
-			String url = connectorUnderTest.getGenomeVersionFile(yeast, version);
-			assertEquals(1, getWarnings().size());
-			assertTrue(url.endsWith("1.2bit"));
-		} catch (Exception e) {
-			e.printStackTrace();
-			throw e;
-		}
-	}
-
-	private void waitForDownload() throws InterruptedException {
-		while (connector.getDownloadThreadCount() > 0) {
-			logger.debug("Waiting for download to finish");
-			Thread.sleep(100);
-		}
-	}
+  Logger logger = Logger.getLogger(UcscReferenceGenomeConnectorTest.class);
+
+  @Autowired
+  UcscReferenceGenomeConnector connector;
+
+  @Autowired
+  ReferenceGenomeDao referenceGenomeDao;
+
+  @Before
+  public void setUp() throws Exception {
+  }
+
+  @After
+  public void tearDown() throws Exception {
+  }
+
+  @Test
+  public void testRefreshCacheQuery() throws Exception {
+    try {
+      assertNotNull(connector.refreshCacheQuery("http://google.pl/"));
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGetAvailableGenomeVersion() throws Exception {
+    MiriamData human = new MiriamData(MiriamType.TAXONOMY, "9606");
+    try {
+      List<String> list = connector.getAvailableGenomeVersion(human);
+      logger.debug(list);
+      assertTrue(list.size() >= 17);
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGetAvailableGenomeVersionWhenProblemWithTaxonomyServer() throws Exception {
+    MiriamData human = new MiriamData(MiriamType.TAXONOMY, "9606");
+    TaxonomyBackend taxonomyBackend = connector.getTaxonomyBackend();
+    try {
+      // simulate problems with taxonomy server
+      TaxonomyBackend taxonomyMock = Mockito.mock(TaxonomyBackend.class);
+      when(taxonomyMock.getByName(anyString())).thenThrow(new TaxonomySearchException(null, null));
+      connector.setTaxonomyBackend(taxonomyMock);
+
+      connector.getAvailableGenomeVersion(human);
+      fail("Exception expected");
+    } catch (ReferenceGenomeConnectorException e) {
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    } finally {
+      connector.setTaxonomyBackend(taxonomyBackend);
+    }
+  }
+
+  @Test
+  public void testGetAvailableGenomeVersionForInvalidTaxonomy() throws Exception {
+    MiriamData unknown = new MiriamData(MiriamType.TAXONOMY, "10101010101");
+    try {
+      List<String> list = connector.getAvailableGenomeVersion(unknown);
+      assertEquals(0, list.size());
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGetAvailableOrganisms() throws Exception {
+    try {
+      MiriamData human = new MiriamData(MiriamType.TAXONOMY, "9606");
+      MiriamData chicken = new MiriamData(MiriamType.TAXONOMY, "9031");
+      List<MiriamData> list = connector.getAvailableOrganisms();
+      logger.debug(list);
+      assertTrue(list.size() > 40);
+
+      assertTrue(list.contains(human));
+      assertTrue(list.contains(chicken));
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGetAvailableOrganismsWhenUcscWebpageDown() throws Exception {
+    WebPageDownloader downloader = connector.getWebPageDownloader();
+    GeneralCacheInterface originalCache = connector.getCache();
+    try {
+      // exclude first cached value
+      connector.setCache(null);
+
+      WebPageDownloader mockDownloader = Mockito.mock(WebPageDownloader.class);
+      when(mockDownloader.getFromNetwork(anyString(), anyString(), anyString())).thenThrow(new IOException());
+      connector.setWebPageDownloader(mockDownloader);
+      connector.getAvailableOrganisms();
+      fail("Exception expected");
+    } catch (ReferenceGenomeConnectorException e) {
+      assertTrue(e.getMessage().contains("Problem with accessing UCSC database"));
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    } finally {
+      connector.setWebPageDownloader(downloader);
+      connector.setCache(originalCache);
+    }
+  }
+
+  @Test
+  public void testDownloadGenomeVersion() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "1570291");
+    String version = "eboVir3";
+
+    try {
+      dbUtils.createSessionForCurrentThread();
+      List<String> list = connector.getDownloadedGenomeVersions(yeast);
+      if (list.contains(version)) {
+        connector.removeGenomeVersion(yeast, version);
+      }
+      dbUtils.closeSessionForCurrentThread();
+      list = connector.getDownloadedGenomeVersions(yeast);
+
+      assertFalse(list.contains(version));
+
+      connector.downloadGenomeVersion(yeast, version, new IProgressUpdater() {
+        @Override
+        public void setProgress(double progress) {
+        }
+      }, false);
+
+      list = connector.getDownloadedGenomeVersions(yeast);
+      assertTrue(list.contains(version));
+
+      List<ReferenceGenome> genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
+      ReferenceGenome genome = null;
+      for (ReferenceGenome referenceGenome : genomes) {
+        if (referenceGenome.getVersion().equals(version)) {
+          genome = referenceGenome;
+        }
+      }
+      assertNotNull(genome);
+      connector.downloadGeneMappingGenomeVersion(genome, "test", null, false,
+          "http://www.biodalliance.org/datasets/flyThickets.bb");
+
+      dbUtils.createSessionForCurrentThread();
+      genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
+      genome = null;
+      for (ReferenceGenome referenceGenome : genomes) {
+        if (referenceGenome.getVersion().equals(version)) {
+          genome = referenceGenome;
+        }
+      }
+      assertNotNull(genome);
+      assertEquals(1, genome.getGeneMapping().size());
+      dbUtils.closeSessionForCurrentThread();
+
+      try {
+        // try to download the same thing for the second time
+        connector.downloadGeneMappingGenomeVersion(genome, "test", new IProgressUpdater() {
+          @Override
+          public void setProgress(double progress) {
+          }
+        }, false, "http://www.biodalliance.org/datasets/flyThickets.bb");
+        fail("Exception expected");
+      } catch (ReferenceGenomeConnectorException e) {
+        assertTrue(e.getMessage().contains("already exists"));
+      }
+
+      try {
+        // try to download something that is not big bed
+        connector.downloadGeneMappingGenomeVersion(genome, "test2", new IProgressUpdater() {
+          @Override
+          public void setProgress(double progress) {
+          }
+        }, false, "http://www.biodalliance.org/datasets/flyThickets.txt");
+        fail("Exception expected");
+      } catch (ReferenceGenomeConnectorException e) {
+        assertTrue(e.getMessage().contains("Only big bed format files are supported"));
+      }
+
+      // connector.removeGeneMapping(genome.getGeneMapping().get(0));
+
+      int warningCount = getWarnings().size();
+
+      // try to remove invalid mapping
+      connector.removeGeneMapping(new ReferenceGenomeGeneMapping());
+
+      assertEquals(warningCount + 1, getWarnings().size());
+
+      dbUtils.createSessionForCurrentThread();
+      connector.removeGenomeVersion(yeast, version);
+      dbUtils.closeSessionForCurrentThread();
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  @Test
+  public void testDownloadGenomeVersionWhenInternalUrlIsInvalid() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "1570291");
+    String version = "eboVir3";
+
+    ExecutorService originalExecutorService = connector.getSyncExecutorService();
+    try {
+      ExecutorService executorService = Mockito.mock(ExecutorService.class);
+      connector.setSyncExecutorService(executorService);
+      Future<?> task = Mockito.mock(Future.class);
+      when(task.get()).thenThrow(new ExecutionException(new URISyntaxException("", "")));
+      when(executorService.submit(any(Callable.class))).thenReturn(task);
+
+      connector.downloadGenomeVersion(yeast, version, new IProgressUpdater() {
+        @Override
+        public void setProgress(double progress) {
+        }
+      }, false);
+      fail("Exception expected");
+
+    } catch (InvalidStateException e) {
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    } finally {
+      connector.setSyncExecutorService(originalExecutorService);
+    }
+  }
+
+  @Test
+  public void testDownloadGenomeVersionAsync() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "1570291");
+    String version = "eboVir3";
+
+    BigFileCache bigFileCache = connector.getBigFileCache();
+    try {
+      dbUtils.createSessionForCurrentThread();
+      List<String> list = connector.getDownloadedGenomeVersions(yeast);
+      if (list.contains(version)) {
+        connector.removeGenomeVersion(yeast, version);
+      }
+      list = connector.getDownloadedGenomeVersions(yeast);
+
+      dbUtils.closeSessionForCurrentThread();
+
+      // create a mock for ftp connections (so we speed up tests), the real
+      // connection is tested elsewhere
+      BigFileCache mockCache = Mockito.mock(BigFileCache.class);
+      Mockito.doAnswer(new Answer<Void>() {
+
+        @Override
+        public Void answer(InvocationOnMock invocation) throws Throwable {
+          IProgressUpdater updater = (IProgressUpdater) invocation.getArguments()[2];
+          updater.setProgress(100);
+          return null;
+        }
+      }).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
+
+      connector.setBigFileCache(mockCache);
+
+      assertFalse(list.contains(version));
+
+      connector.downloadGenomeVersion(yeast, version, null, true);
+
+      waitForDownload();
+
+      dbUtils.createSessionForCurrentThread();
+      list = connector.getDownloadedGenomeVersions(yeast);
+      assertTrue(list.contains(version));
+
+      List<ReferenceGenome> genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
+      ReferenceGenome genome = null;
+      for (ReferenceGenome referenceGenome : genomes) {
+        if (referenceGenome.getVersion().equals(version)) {
+          genome = referenceGenome;
+        }
+      }
+      dbUtils.closeSessionForCurrentThread();
+      assertNotNull(genome);
+      connector.downloadGeneMappingGenomeVersion(genome, "test", new IProgressUpdater() {
+        @Override
+        public void setProgress(double progress) {
+        }
+      }, true, "http://www.biodalliance.org/datasets/flyThickets.bb");
+
+      waitForDownload();
+
+      genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
+      genome = null;
+      for (ReferenceGenome referenceGenome : genomes) {
+        if (referenceGenome.getVersion().equals(version)) {
+          genome = referenceGenome;
+        }
+      }
+      assertNotNull(genome);
+      referenceGenomeDao.refresh(genome);
+      assertEquals(1, genome.getGeneMapping().size());
+
+      try {
+        // try to download the same thing for the second time
+        connector.downloadGeneMappingGenomeVersion(genome, "test", new IProgressUpdater() {
+          @Override
+          public void setProgress(double progress) {
+          }
+        }, false, "http://www.biodalliance.org/datasets/flyThickets.bb");
+        fail("Exception expected");
+      } catch (ReferenceGenomeConnectorException e) {
+        assertTrue(e.getMessage().contains("already exists"));
+      }
+
+      try {
+        // try to download something that is not big bed
+        connector.downloadGeneMappingGenomeVersion(genome, "test2", new IProgressUpdater() {
+          @Override
+          public void setProgress(double progress) {
+          }
+        }, false, "http://www.biodalliance.org/datasets/flyThickets.txt");
+        fail("Exception expected");
+      } catch (ReferenceGenomeConnectorException e) {
+        assertTrue(e.getMessage().contains("Only big bed format files are supported"));
+      }
+
+      connector.removeGeneMapping(genome.getGeneMapping().get(0));
+
+      int warningCount = getWarnings().size();
+
+      // try to remove invalid mapping
+      connector.removeGeneMapping(new ReferenceGenomeGeneMapping());
+
+      assertEquals(warningCount + 1, getWarnings().size());
+
+      connector.removeGenomeVersion(yeast, version);
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    } finally {
+      connector.setBigFileCache(bigFileCache);
+    }
+  }
+
+  @Test
+  public void testDownloadGenomeVersionAsyncWithException() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "1570291");
+    String version = "eboVir3";
+
+    BigFileCache bigFileCache = connector.getBigFileCache();
+    try {
+      dbUtils.createSessionForCurrentThread();
+      List<String> list = connector.getDownloadedGenomeVersions(yeast);
+      if (list.contains(version)) {
+        connector.removeGenomeVersion(yeast, version);
+      }
+      list = connector.getDownloadedGenomeVersions(yeast);
+
+      dbUtils.closeSessionForCurrentThread();
+
+      // create a mock for ftp connections (so we speed up tests), the real
+      // connection is tested elsewhere
+      BigFileCache mockCache = Mockito.mock(BigFileCache.class);
+
+      connector.setBigFileCache(mockCache);
+
+      assertFalse(list.contains(version));
+
+      connector.downloadGenomeVersion(yeast, version, new IProgressUpdater() {
+        @Override
+        public void setProgress(double progress) {
+        }
+      }, true);
+
+      waitForDownload();
+
+      dbUtils.createSessionForCurrentThread();
+      list = connector.getDownloadedGenomeVersions(yeast);
+      assertTrue(list.contains(version));
+
+      List<ReferenceGenome> genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
+      ReferenceGenome genome = null;
+      for (ReferenceGenome referenceGenome : genomes) {
+        if (referenceGenome.getVersion().equals(version)) {
+          genome = referenceGenome;
+        }
+      }
+      dbUtils.closeSessionForCurrentThread();
+      assertNotNull(genome);
+
+      Mockito.doThrow(new IOException()).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
+      assertEquals(0, getErrors().size());
+      connector.downloadGeneMappingGenomeVersion(genome, "test", new IProgressUpdater() {
+        @Override
+        public void setProgress(double progress) {
+        }
+      }, true, "http://www.biodalliance.org/datasets/flyThickets.bb");
+
+      waitForDownload();
+
+      logger.debug(getErrors());
+      // during this downlad we expect exception thrown by downloader
+      assertEquals(1, getErrors().size());
+
+      Mockito.doThrow(new OutOfMemoryError()).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
+      connector.downloadGeneMappingGenomeVersion(genome, "test2", new IProgressUpdater() {
+        @Override
+        public void setProgress(double progress) {
+        }
+      }, true, "http://www.biodalliance.org/datasets/flyThickets.bb");
+
+      waitForDownload();
+      // during this downlad we expect error, this shouldn't change list of
+      // errors
+      assertEquals(1, getErrors().size());
+
+      Mockito.doThrow(new URISyntaxException("", "")).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
+      connector.downloadGeneMappingGenomeVersion(genome, "test3", new IProgressUpdater() {
+        @Override
+        public void setProgress(double progress) {
+        }
+      }, true, "http://www.biodalliance.org/datasets/flyThickets.bb");
+
+      waitForDownload();
+
+      // during this downlad we expect error (exception thrown by downloader)
+      assertEquals(2, getErrors().size());
+
+      genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
+      genome = null;
+      for (ReferenceGenome referenceGenome : genomes) {
+        if (referenceGenome.getVersion().equals(version)) {
+          genome = referenceGenome;
+        }
+      }
+      assertNotNull(genome);
+      referenceGenomeDao.refresh(genome);
+      assertEquals(3, genome.getGeneMapping().size());
+
+      connector.removeGeneMapping(genome.getGeneMapping().get(2));
+      connector.removeGeneMapping(genome.getGeneMapping().get(1));
+      connector.removeGeneMapping(genome.getGeneMapping().get(0));
+      connector.removeGenomeVersion(yeast, version);
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    } finally {
+      connector.setBigFileCache(bigFileCache);
+    }
+  }
+
+  @Test
+  public void testDownloadGenomeVersionWithException() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "1570291");
+    String version = "eboVir3";
+
+    BigFileCache bigFileCache = connector.getBigFileCache();
+    try {
+      List<String> list = connector.getDownloadedGenomeVersions(yeast);
+      if (list.contains(version)) {
+        connector.removeGenomeVersion(yeast, version);
+      }
+      list = connector.getDownloadedGenomeVersions(yeast);
+
+      // create a mock for ftp connections (so we speed up tests), the real
+      // connection is tested elsewhere
+      BigFileCache mockCache = Mockito.mock(BigFileCache.class);
+
+      connector.setBigFileCache(mockCache);
+
+      assertFalse(list.contains(version));
+
+      connector.downloadGenomeVersion(yeast, version, new IProgressUpdater() {
+        @Override
+        public void setProgress(double progress) {
+        }
+      }, false);
+
+      list = connector.getDownloadedGenomeVersions(yeast);
+      assertTrue(list.contains(version));
+
+      List<ReferenceGenome> genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
+      ReferenceGenome genome = null;
+      for (ReferenceGenome referenceGenome : genomes) {
+        if (referenceGenome.getVersion().equals(version)) {
+          genome = referenceGenome;
+        }
+      }
+      assertNotNull(genome);
+
+      Mockito.doThrow(new IOException()).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
+      assertEquals(0, getErrors().size());
+      try {
+        connector.downloadGeneMappingGenomeVersion(genome, "test", new IProgressUpdater() {
+          @Override
+          public void setProgress(double progress) {
+          }
+        }, false, "http://www.biodalliance.org/datasets/flyThickets.bb");
+        fail("Exception expected");
+      } catch (IOException e) {
+
+      }
+
+      Mockito.doThrow(new RuntimeException()).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
+      try {
+        connector.downloadGeneMappingGenomeVersion(genome, "test2", new IProgressUpdater() {
+          @Override
+          public void setProgress(double progress) {
+          }
+        }, false, "http://www.biodalliance.org/datasets/flyThickets.bb");
+
+        fail("Exception expected");
+      } catch (InvalidStateException e) {
+
+      }
+
+      Mockito.doThrow(new URISyntaxException("", "")).when(mockCache).downloadFile(anyString(), anyBoolean(), any());
+
+      try {
+        connector.downloadGeneMappingGenomeVersion(genome, "test3", new IProgressUpdater() {
+          @Override
+          public void setProgress(double progress) {
+          }
+        }, false, "http://www.biodalliance.org/datasets/flyThickets.bb");
+        fail("Exception expected");
+      } catch (URISyntaxException e) {
+
+      }
+
+      genomes = referenceGenomeDao.getByType(ReferenceGenomeType.UCSC);
+      genome = null;
+      for (ReferenceGenome referenceGenome : genomes) {
+        if (referenceGenome.getVersion().equals(version)) {
+          genome = referenceGenome;
+        }
+      }
+      assertNotNull(genome);
+      referenceGenomeDao.refresh(genome);
+      assertEquals(3, genome.getGeneMapping().size());
+
+      connector.removeGeneMapping(genome.getGeneMapping().get(2));
+      connector.removeGeneMapping(genome.getGeneMapping().get(1));
+      connector.removeGeneMapping(genome.getGeneMapping().get(0));
+      connector.removeGenomeVersion(yeast, version);
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    } finally {
+      connector.setBigFileCache(bigFileCache);
+    }
+  }
+
+  @Test
+  public void testGetGenomeVersionFile() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
+    String version = "sacCer3";
+    try {
+      String url = connector.getGenomeVersionFile(yeast, version);
+      assertNotNull(url);
+      url.contains("sacCer3.2bit");
+      url.contains("ftp");
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGetUnknownGenomeVersionFile() {
+    MiriamData human = new MiriamData(MiriamType.TAXONOMY, "9606");
+    String version = "hg8";
+    try {
+      connector.getGenomeVersionFile(human, version);
+      fail("Exception expected");
+    } catch (FileNotAvailableException e) {
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testRefreshInvalid() throws Exception {
+    try {
+      connector.refreshCacheQuery("invalid query");
+      fail("Exception expected");
+    } catch (InvalidArgumentException e) {
+      assertTrue(e.getMessage().contains("Don't know what to do"));
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testRefreshInvalid2() throws Exception {
+    try {
+      connector.refreshCacheQuery(new Object());
+      fail("Exception expected");
+    } catch (InvalidArgumentException e) {
+      assertTrue(e.getMessage().contains("Don't know what to do"));
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testRefresh() throws Exception {
+    try {
+      MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
+      String version = "sacCer3";
+
+      String query = UcscReferenceGenomeConnector.FILENAME_BY_ORGANISM_VERSION_PREFIX + yeast.getResource() + "\n"
+          + version;
+      Object res = connector.refreshCacheQuery(query);
+      assertNotNull(res);
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testRefreshUnknownFile() throws Exception {
+    try {
+      MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
+      String version = "unkVer";
+
+      String query = UcscReferenceGenomeConnector.FILENAME_BY_ORGANISM_VERSION_PREFIX + yeast.getResource() + "\n"
+          + version;
+      connector.refreshCacheQuery(query);
+    } catch (SourceNotAvailable e) {
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGetters() throws Exception {
+    try {
+      UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector();
+      connectorUnderTest.setBigFileCache(connector.getBigFileCache());
+      connectorUnderTest.setDbUtils(connector.getDbUtils());
+      connectorUnderTest.setReferenceGenomeDao(connector.getReferenceGenomeDao());
+      connectorUnderTest.setReferenceGenomeGeneMappingDao(connector.getReferenceGenomeGeneMappingDao());
+
+      assertEquals(connector.getBigFileCache(), connectorUnderTest.getBigFileCache());
+      assertEquals(connector.getDbUtils(), connectorUnderTest.getDbUtils());
+      assertEquals(connector.getReferenceGenomeDao(), connectorUnderTest.getReferenceGenomeDao());
+      assertEquals(connector.getReferenceGenomeGeneMappingDao(), connectorUnderTest.getReferenceGenomeGeneMappingDao());
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGenomeVersionExtractorForInvalidGenome() throws Exception {
+    try {
+      int version = connector.extractInt("");
+      assertEquals(0, version);
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGenomeVersionExtractorForInvalidGenome2() throws Exception {
+    try {
+      int version = connector.extractInt("x");
+      assertEquals(0, version);
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGenomeVersionExtractorForValid() throws Exception {
+    try {
+      int version = connector.extractInt("x1");
+      assertEquals(1, version);
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGenomeVersionExtractorForValid2() throws Exception {
+    try {
+      int version = connector.extractInt("xy12z");
+      assertEquals(12, version);
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testRefreshCacheQueryNotAvailable() throws Exception {
+    WebPageDownloader downloader = connector.getWebPageDownloader();
+    GeneralCacheInterface originalCache = connector.getCache();
+    try {
+      // exclude first cached value
+      connector.setCache(null);
+      WebPageDownloader mockDownloader = Mockito.mock(WebPageDownloader.class);
+      when(mockDownloader.getFromNetwork(anyString(), anyString(), anyString())).thenThrow(new IOException());
+      connector.setWebPageDownloader(mockDownloader);
+      connector.refreshCacheQuery("http://google.pl/");
+      fail("Exception expected");
+    } catch (SourceNotAvailable e) {
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    } finally {
+      connector.setWebPageDownloader(downloader);
+      connector.setCache(originalCache);
+    }
+  }
+
+  @Test
+  public void testRefreshCacheQueryWhenFtpConnectionFails() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
+    String version = "sacCer3";
+
+    UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
+      @Override
+      FTPClient createFtpClient() {
+        FTPClient mockClient = Mockito.mock(FTPClient.class);
+        Mockito.doReturn(FTPReply.REQUEST_DENIED).when(mockClient).getReplyCode();
+        return mockClient;
+      }
+    };
+
+    try {
+      // exclude first cached value
+      connectorUnderTest.getGenomeVersionFile(yeast, version);
+      fail("Exception expected");
+    } catch (FileNotAvailableException e) {
+      assertTrue(e.getMessage().contains("FTP server refused connection"));
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGetGenomeVersionFileWhenFtpConnectionFails2() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
+    String version = "sacCer3";
+
+    UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
+      @Override
+      FTPClient createFtpClient() {
+        FTPClient mockClient = Mockito.mock(FTPClient.class);
+        try {
+          Mockito.doThrow(new IOException()).when(mockClient).connect(anyString());
+          Mockito.doReturn(false).when(mockClient).isConnected();
+        } catch (Exception e) {
+          fail();
+        }
+        return mockClient;
+      }
+    };
+
+    try {
+      // exclude first cached value
+      connectorUnderTest.getGenomeVersionFile(yeast, version);
+      fail("Exception expected");
+    } catch (FileNotAvailableException e) {
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGetGenomeVersionFileWhenFtpConnectionFails() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
+    String version = "sacCer3";
+
+    UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
+      @Override
+      FTPClient createFtpClient() {
+        FTPClient mockClient = Mockito.mock(FTPClient.class);
+        try {
+          Mockito.doThrow(new IOException()).when(mockClient).connect(anyString());
+          Mockito.doReturn(true).when(mockClient).isConnected();
+        } catch (Exception e) {
+          fail();
+        }
+        return mockClient;
+      }
+    };
+
+    try {
+      // exclude first cached value
+      connectorUnderTest.getGenomeVersionFile(yeast, version);
+      fail("Exception expected");
+    } catch (FileNotAvailableException e) {
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGetGenomeVersionFileWhenFtpConnectionFails3() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
+    String version = "sacCer3";
+
+    UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
+      @Override
+      FTPClient createFtpClient() {
+        FTPClient mockClient = Mockito.mock(FTPClient.class);
+        try {
+          Mockito.doThrow(new IOException()).when(mockClient).connect(anyString());
+          Mockito.doReturn(true).when(mockClient).isConnected();
+          Mockito.doThrow(new IOException()).when(mockClient).disconnect();
+        } catch (Exception e) {
+          fail();
+        }
+        return mockClient;
+      }
+    };
+
+    try {
+      // exclude first cached value
+      connectorUnderTest.getGenomeVersionFile(yeast, version);
+      fail("Exception expected");
+    } catch (FileNotAvailableException e) {
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGetGenomeVersionFileWhenFtpDisconnectConnectionFails() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
+    String version = "sacCer3";
+
+    UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
+      @Override
+      FTPClient createFtpClient() {
+        FTPClient mockClient = Mockito.mock(FTPClient.class);
+        try {
+          Mockito.doThrow(new IOException()).when(mockClient).disconnect();
+          Mockito.doReturn(new FTPFile[] {}).when(mockClient).listFiles(anyString());
+          Mockito.doReturn(FTPReply.COMMAND_OK).when(mockClient).getReplyCode();
+          Mockito.doReturn(true).when(mockClient).isConnected();
+        } catch (Exception e) {
+          fail();
+        }
+        return mockClient;
+      }
+    };
+
+    try {
+      // exclude first cached value
+      connectorUnderTest.getGenomeVersionFile(yeast, version);
+      fail("Exception expected");
+    } catch (FileNotAvailableException e) {
+      assertTrue(e.getMessage().contains("Problem with ftp connection"));
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  @Test
+  public void testGetGenomeVersionFileWithMore2bitFiles() throws Exception {
+    MiriamData yeast = new MiriamData(MiriamType.TAXONOMY, "4932");
+    String version = "sacCer3";
+
+    UcscReferenceGenomeConnector connectorUnderTest = new UcscReferenceGenomeConnector() {
+      @Override
+      FTPClient createFtpClient() {
+        FTPClient mockClient = Mockito.mock(FTPClient.class);
+        try {
+          FTPFile file1 = new FTPFile();
+          file1.setName("1.2bit");
+          FTPFile file2 = new FTPFile();
+          file2.setName("2.2bit");
+          FTPFile[] files = new FTPFile[] { file1, file2 };
+
+          Mockito.doReturn(files).when(mockClient).listFiles(anyString());
+          Mockito.doReturn(FTPReply.COMMAND_OK).when(mockClient).getReplyCode();
+        } catch (Exception e) {
+          fail();
+        }
+        return mockClient;
+      }
+    };
+
+    try {
+      // exclude first cached value
+      String url = connectorUnderTest.getGenomeVersionFile(yeast, version);
+      assertEquals(1, getWarnings().size());
+      assertTrue(url.endsWith("1.2bit"));
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+
+  private void waitForDownload() throws InterruptedException {
+    while (connector.getDownloadThreadCount() > 0) {
+      logger.debug("Waiting for download to finish");
+      Thread.sleep(100);
+    }
+  }
 }
-- 
GitLab