diff --git a/annotation/src/main/java/lcsb/mapviewer/annotation/cache/BigFileCache.java b/annotation/src/main/java/lcsb/mapviewer/annotation/cache/BigFileCache.java
index 3592481abe7510344f9d9779be234283b507e7eb..11c9daa14e79f986a40b9aa9ec44f94f08043090 100644
--- a/annotation/src/main/java/lcsb/mapviewer/annotation/cache/BigFileCache.java
+++ b/annotation/src/main/java/lcsb/mapviewer/annotation/cache/BigFileCache.java
@@ -48,848 +48,848 @@ import lcsb.mapviewer.persist.dao.cache.BigFileEntryDao;
*/
public class BigFileCache {
- /**
- * Buffer size used for downloading single chunk of a file.
- */
- private static final int DOWNLOAD_BUFFER_SIZE = 1024;
-
- /**
- * Default class logger.
- */
- private Logger logger = Logger.getLogger(BigFileCache.class);
-
- /**
- * Access class for database objects storing statuses of the files.
- */
- @Autowired
- private BigFileEntryDao bigFileEntryDao;
-
- /**
- * Access class for database objects storing some configuration information.
- */
- @Autowired
- private ConfigurationDao configurationDao;
-
- /**
- * Utils that help to manage the sessions in custom multithreaded
- * implementation.
- */
- @Autowired
- private DbUtils dbUtils;
-
- /**
- * Service used for executing tasks in separate thread.
- */
- private ExecutorService asyncExecutorService;
-
- /**
- * Service used for executing tasks immediately.
- */
- private ExecutorService syncExecutorService;
-
- /**
- * Factory class used to create {@link FTPClient} objects.
- */
- private FtpClientFactory ftpClientFactory = new FtpClientFactory();
-
- /**
- * Default constructor.
- */
- public BigFileCache() {
- // the executor is a daemon thread so that it will get killed automatically
- // when the main program exits
- asyncExecutorService = Executors.newScheduledThreadPool(10, new ThreadFactory() {
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new Thread(r);
- t.setDaemon(true);
- return t;
- }
- });
- syncExecutorService = Executors.newScheduledThreadPool(1, new ThreadFactory() {
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new Thread(r);
- t.setDaemon(true);
- return t;
- }
- });
-
- // put in the queue empty task to make sure that everything was initialized
- // (additional managing thread was createed)
- asyncExecutorService.submit(new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- return null;
- }
- });
- syncExecutorService.submit(new Callable<Object>() {
- @Override
- public Object call() throws Exception {
- return null;
- }
- });
- }
-
- /**
- * Returns a path for a file in local file system.
- *
- * @param url
- * ftp url to the file
- * @return a path for a file in local file systeml
- * @throws FileNotFoundException
- * thrown when file should be in file system, but couldn't be found
- * there (somebody manually removed it)
- */
- public String getAbsolutePathForFile(String url) throws FileNotFoundException {
- BigFileEntry entry = bigFileEntryDao.getByUrl(url);
- if (entry == null) {
- return null;
- }
- if (entry.getDownloadProgress() == null || entry.getDownloadProgress() < 100.0) {
- throw new FileNotFoundException("File is not complete: " + entry.getLocalPath() + ". Downloaded from: " + url);
- }
- File f = new File(Configuration.getWebAppDir() + entry.getLocalPath());
- if (!f.exists()) {
- throw new FileNotFoundException("Missing big file: " + Configuration.getWebAppDir() + entry.getLocalPath() + ". Downloaded from: " + url);
- }
-
- return f.getAbsolutePath();
- }
-
- /**
- * Returns a path for a file in local file system for an ftp url.
- *
- * @param url
- * ftp url to the file
- * @return a path for a file in local file system for an ftp url
- * @throws FileNotFoundException
- * thrown when file should be in file system, but couldn't be found
- * there (somebody manually removed it)
- */
- public String getLocalPathForFile(String url) throws FileNotFoundException {
- BigFileEntry entry = bigFileEntryDao.getByUrl(url);
- if (entry == null) {
- return null;
- }
- if (entry.getDownloadProgress() == null || entry.getDownloadProgress() < 100.0) {
- throw new FileNotFoundException("File is not complete: " + entry.getLocalPath() + ". Downloaded from: " + url);
- }
- File f = new File(Configuration.getWebAppDir() + entry.getLocalPath());
- if (!f.exists()) {
- throw new FileNotFoundException("Missing big file: " + Configuration.getWebAppDir() + entry.getLocalPath() + ". Downloaded from: " + url);
- }
-
- return entry.getLocalPath();
- }
-
- /**
- * Download big file from ftp server.
- *
- * @param url
- * url for file to download
- * @param async
- * <code>true</code> if should the download be asynchronous
- * @param updater
- * callback emthod that will be called with info updates about
- * downloading process
- * @throws IOException
- * thrown when there is a problem with downloading file
- * @throws URISyntaxException
- * thrown when url is invalid
- */
- public void downloadFile(String url, boolean async, IProgressUpdater updater) throws IOException, URISyntaxException {
- Callable<Void> computations = null;
- if (url.toLowerCase().startsWith("ftp:")) {
- computations = new GetFtpFileTask(url, updater);
- } else if (url.toLowerCase().startsWith("http:") || url.toLowerCase().startsWith("https:")) {
- computations = new GetHttpFileTask(url, updater);
- } else {
- throw new URISyntaxException(url, "Unknown protocol for url");
- }
-
- if (async) {
- asyncExecutorService.submit(computations);
- } else {
- Future<Void> task = syncExecutorService.submit(computations);
- executeTask(task);
- }
- }
-
- /**
- * Creates new {@link BigFileEntry} for given url.
- *
- * @param url
- * url for which entry will be created
- * @return new {@link BigFileEntry} for given url
- * @throws URISyntaxException
- * thrown when url is invalid
- */
- private BigFileEntry createEntryForBigFile(String url) throws URISyntaxException {
- String localPath = configurationDao.getValueByType(ConfigurationElementType.BIG_FILE_STORAGE_DIR);
- BigFileEntry entry = new BigFileEntry();
- entry.setDownloadDate(Calendar.getInstance());
- entry.setLocalPath(localPath);
- entry.setUrl(url);
- bigFileEntryDao.add(entry);
-
- localPath = entry.getLocalPath() + "/" + entry.getId() + "/";
- new File(Configuration.getWebAppDir() + localPath).mkdirs();
- localPath += getFileName(url);
-
- entry.setLocalPath(localPath);
- bigFileEntryDao.update(entry);
- bigFileEntryDao.flush();
- return entry;
- }
-
- /**
- * Task that will be able to fetch file from ftp server.
- *
- * @author Piotr Gawron
- *
- */
- private final class GetFtpFileTask implements Callable<Void> {
-
- /**
- * Url to the file that we want to download.
- *
- */
- private String url;
-
- /**
- * Callback listener that will receive information about upload progress.
- *
- */
- private IProgressUpdater updater;
-
- /**
- * Default constructor.
- *
- * @param url
- * {@link #url}
- * @param updater
- * {@link #updater}
- */
- private GetFtpFileTask(String url, IProgressUpdater updater) {
- this.url = url;
- if (updater != null) {
- this.updater = updater;
- } else {
- this.updater = new IProgressUpdater() {
- @Override
- public void setProgress(double progress) {
- }
- };
- }
- }
-
- @Override
- public Void call() throws Exception {
- dbUtils.createSessionForCurrentThread();
- FTPClient ftp = ftpClientFactory.createFtpClient();
- try {
- try {
- if (getAbsolutePathForFile(url) != null) {
- logger.warn("File already downloaded. Skipping...");
- return null;
- }
- } catch (FileNotFoundException e) {
- removeFile(url);
- }
-
- BigFileEntry entry = createEntryForBigFile(url);
- entry.setDownloadThreadId(Thread.currentThread().getId());
- bigFileEntryDao.update(entry);
-
- String server = getDomainName(url);
- ftp.connect(server);
- // After connection attempt, you should check the reply code to verify
- // success.
- int reply = ftp.getReplyCode();
-
- if (!FTPReply.isPositiveCompletion(reply)) {
- throw new IOException("FTP server refused connection.");
- }
- ftp.enterLocalPassiveMode();
- ftp.login("anonymous", "");
- ftp.setFileType(FTP.BINARY_FILE_TYPE);
-
- long fileSize = -1;
- String remotePath = getFilePath(url);
- FTPFile[] files = ftp.listFiles(remotePath);
- if (files.length == 1 && files[0].isFile()) {
- fileSize = files[0].getSize();
- }
- final long size = fileSize;
-
- OutputStream output = new FileOutputStream(Configuration.getWebAppDir() + entry.getLocalPath());
- CountingOutputStream cos = new CountingOutputStream(output) {
- private double lastProgress = -1;
-
- protected void beforeWrite(int n) {
- super.beforeWrite(n);
- double newProgress = ((double) getCount()) / ((double) size) * IProgressUpdater.MAX_PROGRESS;
- if (newProgress - lastProgress >= IProgressUpdater.PROGRESS_BAR_UPDATE_RESOLUTION) {
- lastProgress = newProgress;
- entry.setDownloadProgress(lastProgress);
- bigFileEntryDao.update(entry);
- bigFileEntryDao.commit();
- updater.setProgress(lastProgress);
- }
- }
- };
- ftp.retrieveFile(remotePath, cos);
-
- entry.setDownloadProgress(IProgressUpdater.MAX_PROGRESS);
- bigFileEntryDao.update(entry);
- bigFileEntryDao.commit();
- updater.setProgress(IProgressUpdater.MAX_PROGRESS);
-
- output.close();
-
- ftp.logout();
-
- return null;
- } finally {
- bigFileEntryDao.commit();
- // close the transaction for this thread
- dbUtils.closeSessionForCurrentThread();
- if (ftp.isConnected()) {
- ftp.disconnect();
- }
-
- }
- }
-
- }
-
- /**
- * Class that describes task of downloading http file.
- *
- * @author Piotr Gawron
- *
- */
- private final class GetHttpFileTask implements Callable<Void> {
-
- /**
- * Url to the file that we want to download.
- *
- */
- private String url;
-
- /**
- * Callback listener that will receive information about upload progress.
- *
- */
- private IProgressUpdater updater;
-
- /**
- * Default constructor.
- *
- * @param url
- * {@link #url}
- * @param updater
- * {@link #updater}
- */
- private GetHttpFileTask(String url, IProgressUpdater updater) {
- this.url = url;
- if (updater != null) {
- this.updater = updater;
- } else {
- this.updater = new IProgressUpdater() {
- @Override
- public void setProgress(double progress) {
- }
- };
- }
- }
-
- @Override
- public Void call() throws Exception {
- dbUtils.createSessionForCurrentThread();
- BufferedInputStream in = null;
- CountingOutputStream cos = null;
- try {
- try {
- if (getAbsolutePathForFile(url) != null) {
- logger.warn("File already downloaded. Skipping...");
- return null;
- }
- } catch (FileNotFoundException e) {
- removeFile(url);
- }
-
- BigFileEntry entry = createEntryForBigFile(url);
- entry.setDownloadThreadId(Thread.currentThread().getId());
- bigFileEntryDao.update(entry);
-
- final long size = getRemoteHttpFileSize(url);
-
- OutputStream output = new FileOutputStream(Configuration.getWebAppDir() + entry.getLocalPath());
- cos = new CountingOutputStream(output) {
- private double lastProgress = -1;
-
- protected void beforeWrite(int n) {
- super.beforeWrite(n);
- double newProgress = ((double) getCount()) / ((double) size) * IProgressUpdater.MAX_PROGRESS;
- if (newProgress - lastProgress >= IProgressUpdater.PROGRESS_BAR_UPDATE_RESOLUTION) {
- lastProgress = newProgress;
- entry.setDownloadProgress(lastProgress);
- bigFileEntryDao.update(entry);
- bigFileEntryDao.commit();
- updater.setProgress(lastProgress);
- }
- }
- };
- URL website = new URL(url);
- in = new BufferedInputStream(website.openStream());
-
- final byte[] data = new byte[DOWNLOAD_BUFFER_SIZE];
- int count;
- while ((count = in.read(data, 0, DOWNLOAD_BUFFER_SIZE)) != -1) {
- cos.write(data, 0, count);
- }
-
- entry.setDownloadProgress(IProgressUpdater.MAX_PROGRESS);
- bigFileEntryDao.update(entry);
- bigFileEntryDao.commit();
- updater.setProgress(IProgressUpdater.MAX_PROGRESS);
-
- output.close();
-
- return null;
- } finally {
- bigFileEntryDao.commit();
- // close the transaction for this thread
- dbUtils.closeSessionForCurrentThread();
- // close opened streams
- if (in != null) {
- in.close();
- }
- if (cos != null) {
- cos.close();
- }
- }
- }
- }
-
- /**
- * Checks if local file for given url is up to date. The check is based on
- * file size check (ftp doesn't provide checksums for files, so we cannot
- * compare it differently without downloading file).
- *
- * @param url
- * url to ftp file
- * @return <code>true</code> if file is up to date, <code>false</code> in
- * other case
- * @throws URISyntaxException
- * thrown when url is invalid
- * @throws IOException
- * thrown when there is a problem with accessing local or remote
- * file
- */
- public boolean isLocalFileUpToDate(String url) throws URISyntaxException, IOException {
- if (url.toLowerCase().startsWith("ftp")) {
- return isLocalFtpFileUpToDate(url);
- } else if (url.toLowerCase().startsWith("http")) {
- return isLocalHttpFileUpToDate(url);
- } else {
- throw new URISyntaxException(url, "Unknown protocol");
- }
- }
-
- /**
- * Checks if local file fetched from ftp is up to date.
- *
- * @param url
- * url to remote file
- * @return <code>true</code> if file is up to date, <code>false</code>
- * otherwise
- * @throws FileNotFoundException
- * thrown when local file cannot be found
- * @throws URISyntaxException
- * thrown when url is invalid
- * @throws IOException
- * thrown when there is a problem with connection to remote server
- */
- public boolean isLocalFtpFileUpToDate(String url) throws FileNotFoundException, IOException, URISyntaxException {
- boolean result = false;
- BigFileEntry entry = bigFileEntryDao.getByUrl(url);
- if (entry == null) {
- throw new FileNotFoundException("File wasn't downloaded: " + url);
- }
- String path = Configuration.getWebAppDir() + entry.getLocalPath();
- File f = new File(path);
- if (!f.exists()) {
- throw new FileNotFoundException("Missing file: " + path + ". Downloaded from: " + url);
- }
- long localSize = f.length();
-
- FTPClient ftp = ftpClientFactory.createFtpClient();
- try {
- String server = getDomainName(url);
- ftp.connect(server);
- // After connection attempt, you should check the reply code to verify
- // success.
- int reply = ftp.getReplyCode();
-
- if (!FTPReply.isPositiveCompletion(reply)) {
- throw new IOException("FTP server refused connection.");
- }
- ftp.enterLocalPassiveMode();
- ftp.login("anonymous", "");
- ftp.setFileType(FTP.BINARY_FILE_TYPE);
-
- long fileSize = -1;
- String remotePath = getFilePath(url);
- FTPFile[] files = ftp.listFiles(remotePath);
- if (files.length == 1 && files[0].isFile()) {
- fileSize = files[0].getSize();
- }
-
- result = (fileSize == localSize);
- ftp.logout();
- } finally {
- if (ftp.isConnected()) {
- ftp.disconnect();
- }
- }
- return result;
- }
-
- /**
- * Checks if local file fetched from http is up to date.
- *
- * @param url
- * url to remote file
- * @return <code>true</code> if file is up to date, <code>false</code>
- * otherwise
- * @throws IOException
- * thrown when local file cannot be found or there is a problem with
- * accesing remote file
- */
- public boolean isLocalHttpFileUpToDate(String url) throws IOException {
- boolean result = false;
- BigFileEntry entry = bigFileEntryDao.getByUrl(url);
- if (entry == null) {
- throw new FileNotFoundException("File wasn't downloaded: " + url);
- }
- String path = Configuration.getWebAppDir() + entry.getLocalPath();
- File f = new File(path);
- if (!f.exists()) {
- throw new FileNotFoundException("Missing file: " + path + ". Downloaded from: " + url);
- }
- long localSize = f.length();
-
- long remoteSize = getRemoteHttpFileSize(url);
-
- result = (localSize == remoteSize);
- return result;
- }
-
- /**
- * Returns size of the remote file access via http url.
- *
- * @param url
- * url address to the file
- * @return size of the remote file access via http url
- * @throws IOException
- * thrown when there is a problem with accessing remote file
- */
- long getRemoteHttpFileSize(String url) throws IOException {
- long remoteSize = -1;
- HttpURLConnection conn = null;
- try {
- URL website = new URL(url);
- conn = (HttpURLConnection) website.openConnection();
- conn.setRequestMethod("HEAD");
- conn.getInputStream();
- remoteSize = conn.getContentLength();
- } finally {
- if (conn != null) {
- conn.disconnect();
- }
- }
- return remoteSize;
- }
-
- /**
- * Removes local file copy of a file given in a parameter.
- *
- * @param url
- * ftp url of a file
- * @throws IOException
- * thrown when there is a problem with deleting file
- */
- public void removeFile(String url) throws IOException {
- BigFileEntry entry = bigFileEntryDao.getByUrl(url);
- if (entry == null) {
- throw new InvalidArgumentException("Cannot remove file. File wasn't downloaded: " + url);
- }
- String path = Configuration.getWebAppDir() + entry.getLocalPath();
- File f = new File(path);
- if (!f.exists()) {
- logger.warn("Missing file: " + path + ". Downloaded from: " + url);
- }
- String dirPath = FilenameUtils.getFullPath(path);
-
- FileUtils.deleteDirectory(new File(dirPath));
-
- bigFileEntryDao.delete(entry);
- }
-
- /**
- * Task that describes updating file from ftp.
- *
- * @author Piotr Gawron
- *
- */
- private final class UpdateFtpFileTask implements Callable<Void> {
-
- /**
- * Url to file.
- */
- private String url;
-
- /**
- * Default constructor.
- *
- * @param url
- * {@link #url}
- */
- private UpdateFtpFileTask(String url) {
- this.url = url;
- }
-
- @Override
- public Void call() throws Exception {
- dbUtils.createSessionForCurrentThread();
- FTPClient ftp = ftpClientFactory.createFtpClient();
- try {
- BigFileEntry localEntryCopy = bigFileEntryDao.getByUrl(url);
- localEntryCopy.setDownloadThreadId(Thread.currentThread().getId());
- localEntryCopy.setDownloadProgress(0.0);
- bigFileEntryDao.update(localEntryCopy);
-
- // remove current version
- new File(Configuration.getWebAppDir() + localEntryCopy.getLocalPath()).delete();
-
- String server = getDomainName(url);
- ftp.connect(server);
- // After connection attempt, you should check the reply code to
- // verify success.
- int reply = ftp.getReplyCode();
-
- if (!FTPReply.isPositiveCompletion(reply)) {
- throw new IOException("FTP server refused connection.");
- }
- ftp.enterLocalPassiveMode();
- ftp.login("anonymous", "");
- ftp.setFileType(FTP.BINARY_FILE_TYPE);
-
- long fileSize = -1;
- String remotePath = getFilePath(url);
- FTPFile[] files = ftp.listFiles(remotePath);
- if (files.length == 1 && files[0].isFile()) {
- fileSize = files[0].getSize();
- }
- final long size = fileSize;
-
- OutputStream output = new FileOutputStream(localEntryCopy.getLocalPath());
- CountingOutputStream cos = new CountingOutputStream(output) {
- private double lastProgress = -1;
-
- protected void beforeWrite(int n) {
- super.beforeWrite(n);
- double newProgress = ((double) getCount()) / ((double) size) * IProgressUpdater.MAX_PROGRESS;
- if (newProgress - lastProgress >= IProgressUpdater.PROGRESS_BAR_UPDATE_RESOLUTION) {
- lastProgress = newProgress;
- localEntryCopy.setDownloadProgress(lastProgress);
- bigFileEntryDao.update(localEntryCopy);
- }
- }
- };
- ftp.retrieveFile(remotePath, cos);
-
- output.close();
-
- ftp.logout();
- return null;
- } finally {
- dbUtils.closeSessionForCurrentThread();
- if (ftp.isConnected()) {
- ftp.disconnect();
- }
- }
- }
-
- }
-
- /**
- * Updates local copy of a file that can be downloaded from a url given in
- * parameter.
- *
- * @param url
- * url to an ftp file to be updated
- * @param async
- * <code>true</code> if update should be done asynchronously,
- * <code>false</code> otherwise
- * @throws IOException
- * thrown when there is a problem with accessing remote file
- * @throws URISyntaxException
- * thrown when url is invalid
- */
- public void updateFile(String url, boolean async) throws URISyntaxException, IOException {
- if (isLocalFileUpToDate(url)) {
- logger.warn("File is up to date. Skipping...");
- return;
- }
- UpdateFtpFileTask updateTask = new UpdateFtpFileTask(url);
- if (async) {
- asyncExecutorService.submit(updateTask);
- } else {
- Future<Void> task = syncExecutorService.submit(updateTask);
- executeTask(task);
- }
- }
-
- /**
- * Executes download/update task.
- *
- * @param task
- * task to be executed
- * @throws URISyntaxException
- * thrown when task finsihed with {@link URISyntaxException}
- * @throws IOException
- * thrown when task finsihed with {@link IOException}
- */
- void executeTask(Future<?> task) throws URISyntaxException, IOException {
- try {
- task.get();
- } catch (InterruptedException e) {
- logger.error(e, e);
- } catch (ExecutionException e) {
- if (e.getCause() instanceof URISyntaxException) {
- throw (URISyntaxException) e.getCause();
- } else if (e.getCause() instanceof IOException) {
- throw new IOException((IOException) e.getCause());
- } else {
- throw new InvalidStateException(e);
- }
- }
- }
-
- /**
- * Returns server domain name from url.
- *
- * @param url
- * url to be processed
- * @return server domain name from url
- * @throws URISyntaxException
- * thrown when url is invalid
- */
- String getDomainName(String url) throws URISyntaxException {
- URI uri = new URI(url);
- String domain = uri.getHost();
- if (domain.startsWith("www.")) {
- domain = domain.substring("www.".length());
- }
-
- return domain;
- }
-
- /**
- * Returns path to file on server without server domain name from url.
- *
- * @param url
- * url to be processed
- * @return path to file on server without server domain name from url
- * @throws URISyntaxException
- * thrown when url is invalid
- */
- public String getFilePath(String url) throws URISyntaxException {
- URI uri = new URI(url);
- return uri.getPath();
- }
-
- /**
- * Returns simple file name from url.
- *
- * @param url
- * url to be processed
- * @return simple file name from url
- * @throws URISyntaxException
- * thrown when url is invalid
- */
- public String getFileName(String url) throws URISyntaxException {
- URI uri = new URI(url);
- return FilenameUtils.getName(uri.getPath());
- }
-
- /**
- * Checks if the file identified by url is cached.
- *
- * @param sourceUrl
- * url that identifies file
- * @return <code>true</code> if the file is cached, <code>false</code>
- * otherwise
- */
- public boolean isCached(String sourceUrl) {
- BigFileEntry entry = bigFileEntryDao.getByUrl(sourceUrl);
- if (entry != null) {
- File f = new File(Configuration.getWebAppDir() + entry.getLocalPath());
- if (!f.exists()) {
- logger.warn("File is supposed to be cached but it's not there... " + sourceUrl);
- return false;
- }
- }
- return entry != null;
- }
-
- /**
- * @param bigFileEntryDao
- * the bigFileEntryDao to set
- * @see #bigFileEntryDao
- */
- public void setBigFileEntryDao(BigFileEntryDao bigFileEntryDao) {
- this.bigFileEntryDao = bigFileEntryDao;
- }
-
- /**
- * Return number of tasks that are executed or are waiting for execution.
- *
- * @return number of tasks that are executed or are waiting for execution
- */
- public int getDownloadThreadCount() {
- return ((ScheduledThreadPoolExecutor) asyncExecutorService).getQueue().size() + ((ScheduledThreadPoolExecutor) asyncExecutorService).getActiveCount()
- + ((ScheduledThreadPoolExecutor) syncExecutorService).getQueue().size() + ((ScheduledThreadPoolExecutor) syncExecutorService).getActiveCount();
- }
-
- /**
- * @param ftpClientFactory
- * the ftpClientFactory to set
- * @see #ftpClientFactory
- */
- protected void setFtpClientFactory(FtpClientFactory ftpClientFactory) {
- this.ftpClientFactory = ftpClientFactory;
- }
-
- /**
- * @return the configurationDao
- * @see #configurationDao
- */
- protected ConfigurationDao getConfigurationDao() {
- return configurationDao;
- }
-
- /**
- * @param configurationDao
- * the configurationDao to set
- * @see #configurationDao
- */
- protected void setConfigurationDao(ConfigurationDao configurationDao) {
- this.configurationDao = configurationDao;
- }
+ /**
+ * Buffer size used for downloading single chunk of a file.
+ */
+ private static final int DOWNLOAD_BUFFER_SIZE = 1024;
+
+ /**
+ * Default class logger.
+ */
+ private Logger logger = Logger.getLogger(BigFileCache.class);
+
+ /**
+ * Access class for database objects storing statuses of the files.
+ */
+ @Autowired
+ private BigFileEntryDao bigFileEntryDao;
+
+ /**
+ * Access class for database objects storing some configuration information.
+ */
+ @Autowired
+ private ConfigurationDao configurationDao;
+
+ /**
+ * Utils that help to manage the sessions in custom multithreaded
+ * implementation.
+ */
+ @Autowired
+ private DbUtils dbUtils;
+
+ /**
+ * Service used for executing tasks in separate thread.
+ */
+ private ExecutorService asyncExecutorService;
+
+ /**
+ * Service used for executing tasks immediately.
+ */
+ private ExecutorService syncExecutorService;
+
+ /**
+ * Factory class used to create {@link FTPClient} objects.
+ */
+ private FtpClientFactory ftpClientFactory = new FtpClientFactory();
+
+ /**
+ * Default constructor.
+ */
+ public BigFileCache() {
+ // the executor is a daemon thread so that it will get killed automatically
+ // when the main program exits
+ asyncExecutorService = Executors.newScheduledThreadPool(10, new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setDaemon(true);
+ return t;
+ }
+ });
+ syncExecutorService = Executors.newScheduledThreadPool(1, new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setDaemon(true);
+ return t;
+ }
+ });
+
+ // put in the queue empty task to make sure that everything was initialized
+ // (additional managing thread was createed)
+ asyncExecutorService.submit(new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ return null;
+ }
+ });
+ syncExecutorService.submit(new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ return null;
+ }
+ });
+ }
+
+ /**
+ * Returns a path for a file in local file system.
+ *
+ * @param url
+ * url to the file
+ * @return a path for a file in local file system
+ * @throws FileNotFoundException
+ * thrown when file should be in file system, but couldn't be found
+ * there (somebody manually removed it)
+ */
+ public String getAbsolutePathForFile(String url) throws FileNotFoundException {
+ BigFileEntry entry = bigFileEntryDao.getByUrl(url);
+ if (entry == null) {
+ return null;
+ }
+ if (entry.getDownloadProgress() == null || entry.getDownloadProgress() < 100.0) {
+ throw new FileNotFoundException("File is not complete: " + entry.getLocalPath() + ". Downloaded from: " + url);
+ }
+ File f = new File(Configuration.getWebAppDir() + entry.getLocalPath());
+ if (!f.exists()) {
+ throw new FileNotFoundException(
+ "Missing big file: " + Configuration.getWebAppDir() + entry.getLocalPath() + ". Downloaded from: " + url);
+ }
+
+ return f.getAbsolutePath();
+ }
+
+ /**
+ * Returns a path for a file in local file system for an ftp url.
+ *
+ * @param url
+ * ftp url to the file
+ * @return a path for a file in local file system for an ftp url
+ * @throws FileNotFoundException
+ * thrown when file should be in file system, but couldn't be found
+ * there (somebody manually removed it)
+ */
+ public String getLocalPathForFile(String url) throws FileNotFoundException {
+ BigFileEntry entry = bigFileEntryDao.getByUrl(url);
+ if (entry == null) {
+ return null;
+ }
+ if (entry.getDownloadProgress() == null || entry.getDownloadProgress() < 100.0) {
+ throw new FileNotFoundException("File is not complete: " + entry.getLocalPath() + ". Downloaded from: " + url);
+ }
+ File f = new File(Configuration.getWebAppDir() + entry.getLocalPath());
+ if (!f.exists()) {
+ throw new FileNotFoundException(
+ "Missing big file: " + Configuration.getWebAppDir() + entry.getLocalPath() + ". Downloaded from: " + url);
+ }
+
+ return entry.getLocalPath();
+ }
+
+ /**
+ * Download big file from ftp server.
+ *
+ * @param url
+ * url for file to download
+ * @param async
+ * <code>true</code> if should the download be asynchronous
+ * @param updater
+ * callback method that will be called with info updates about
+ * downloading process
+ * @throws IOException
+ * thrown when there is a problem with downloading file
+ * @throws URISyntaxException
+ * thrown when url is invalid
+ */
+ public void downloadFile(String url, boolean async, IProgressUpdater updater) throws IOException, URISyntaxException {
+ Callable<Void> computations = null;
+ if (url.toLowerCase().startsWith("ftp:")) {
+ computations = new GetFtpFileTask(url, updater);
+ } else if (url.toLowerCase().startsWith("http:") || url.toLowerCase().startsWith("https:")) {
+ computations = new GetHttpFileTask(url, updater);
+ } else {
+ throw new URISyntaxException(url, "Unknown protocol for url");
+ }
+
+ if (async) {
+ asyncExecutorService.submit(computations);
+ } else {
+ Future<Void> task = syncExecutorService.submit(computations);
+ executeTask(task);
+ }
+ }
+
+ /**
+ * Creates new {@link BigFileEntry} for given url.
+ *
+ * @param url
+ * url for which entry will be created
+ * @return new {@link BigFileEntry} for given url
+ * @throws URISyntaxException
+ * thrown when url is invalid
+ */
+ private BigFileEntry createEntryForBigFile(String url) throws URISyntaxException {
+ String localPath = configurationDao.getValueByType(ConfigurationElementType.BIG_FILE_STORAGE_DIR);
+ BigFileEntry entry = new BigFileEntry();
+ entry.setDownloadDate(Calendar.getInstance());
+ entry.setLocalPath(localPath);
+ entry.setUrl(url);
+ bigFileEntryDao.add(entry);
+
+ localPath = entry.getLocalPath() + "/" + entry.getId() + "/";
+ new File(Configuration.getWebAppDir() + localPath).mkdirs();
+ localPath += getFileName(url);
+
+ entry.setLocalPath(localPath);
+ bigFileEntryDao.update(entry);
+ bigFileEntryDao.flush();
+ return entry;
+ }
+
+ /**
+ * Task that will be able to fetch file from ftp server.
+ *
+ * @author Piotr Gawron
+ *
+ */
+ private final class GetFtpFileTask implements Callable<Void> {
+
+ /**
+ * Url to the file that we want to download.
+ *
+ */
+ private String url;
+
+ /**
+ * Callback listener that will receive information about upload progress.
+ *
+ */
+ private IProgressUpdater updater;
+
+ /**
+ * Default constructor.
+ *
+ * @param url
+ * {@link #url}
+ * @param updater
+ * {@link #updater}
+ */
+ private GetFtpFileTask(String url, IProgressUpdater updater) {
+ this.url = url;
+ if (updater != null) {
+ this.updater = updater;
+ } else {
+ this.updater = new IProgressUpdater() {
+ @Override
+ public void setProgress(double progress) {
+ }
+ };
+ }
+ }
+
+ @Override
+ public Void call() throws Exception {
+ dbUtils.createSessionForCurrentThread();
+ FTPClient ftp = ftpClientFactory.createFtpClient();
+ try {
+ try {
+ if (getAbsolutePathForFile(url) != null) {
+ logger.warn("File already downloaded. Skipping...");
+ return null;
+ }
+ } catch (FileNotFoundException e) {
+ removeFile(url);
+ }
+
+ BigFileEntry entry = createEntryForBigFile(url);
+ entry.setDownloadThreadId(Thread.currentThread().getId());
+ bigFileEntryDao.update(entry);
+
+ String server = getDomainName(url);
+ ftp.connect(server);
+ // After connection attempt, you should check the reply code to verify
+ // success.
+ int reply = ftp.getReplyCode();
+
+ if (!FTPReply.isPositiveCompletion(reply)) {
+ throw new IOException("FTP server refused connection.");
+ }
+ ftp.enterLocalPassiveMode();
+ ftp.login("anonymous", "");
+ ftp.setFileType(FTP.BINARY_FILE_TYPE);
+
+ long fileSize = -1;
+ String remotePath = getFilePath(url);
+ FTPFile[] files = ftp.listFiles(remotePath);
+ if (files.length == 1 && files[0].isFile()) {
+ fileSize = files[0].getSize();
+ }
+ final long size = fileSize;
+
+ OutputStream output = new FileOutputStream(Configuration.getWebAppDir() + entry.getLocalPath());
+ CountingOutputStream cos = new CountingOutputStream(output) {
+ private double lastProgress = -1;
+
+ protected void beforeWrite(int n) {
+ super.beforeWrite(n);
+ double newProgress = ((double) getCount()) / ((double) size) * IProgressUpdater.MAX_PROGRESS;
+ if (newProgress - lastProgress >= IProgressUpdater.PROGRESS_BAR_UPDATE_RESOLUTION) {
+ lastProgress = newProgress;
+ entry.setDownloadProgress(lastProgress);
+ bigFileEntryDao.update(entry);
+ bigFileEntryDao.commit();
+ updater.setProgress(lastProgress);
+ }
+ }
+ };
+ ftp.retrieveFile(remotePath, cos);
+
+ entry.setDownloadProgress(IProgressUpdater.MAX_PROGRESS);
+ bigFileEntryDao.update(entry);
+ bigFileEntryDao.commit();
+ updater.setProgress(IProgressUpdater.MAX_PROGRESS);
+
+ output.close();
+
+ ftp.logout();
+
+ return null;
+ } finally {
+ bigFileEntryDao.commit();
+ // close the transaction for this thread
+ dbUtils.closeSessionForCurrentThread();
+ if (ftp.isConnected()) {
+ ftp.disconnect();
+ }
+
+ }
+ }
+
+ }
+
+ /**
+ * Class that describes task of downloading http file.
+ *
+ * @author Piotr Gawron
+ *
+ */
+ private final class GetHttpFileTask implements Callable<Void> {
+
+ /**
+ * Url to the file that we want to download.
+ *
+ */
+ private String url;
+
+ /**
+ * Callback listener that will receive information about upload progress.
+ *
+ */
+ private IProgressUpdater updater;
+
+ /**
+ * Default constructor.
+ *
+ * @param url
+ * {@link #url}
+ * @param updater
+ * {@link #updater}
+ */
+ private GetHttpFileTask(String url, IProgressUpdater updater) {
+ this.url = url;
+ if (updater != null) {
+ this.updater = updater;
+ } else {
+ this.updater = new IProgressUpdater() {
+ @Override
+ public void setProgress(double progress) {
+ }
+ };
+ }
+ }
+
+ @Override
+ public Void call() throws Exception {
+ dbUtils.createSessionForCurrentThread();
+ BufferedInputStream in = null;
+ CountingOutputStream cos = null;
+ try {
+ try {
+ if (getAbsolutePathForFile(url) != null) {
+ logger.warn("File already downloaded. Skipping...");
+ return null;
+ }
+ } catch (FileNotFoundException e) {
+ removeFile(url);
+ }
+ BigFileEntry entry = createEntryForBigFile(url);
+ entry.setDownloadThreadId(Thread.currentThread().getId());
+ bigFileEntryDao.update(entry);
+
+ final long size = getRemoteHttpFileSize(url);
+
+ OutputStream output = new FileOutputStream(Configuration.getWebAppDir() + entry.getLocalPath());
+ cos = new CountingOutputStream(output) {
+ private double lastProgress = -1;
+
+ protected void beforeWrite(int n) {
+ super.beforeWrite(n);
+ double newProgress = ((double) getCount()) / ((double) size) * IProgressUpdater.MAX_PROGRESS;
+ if (newProgress - lastProgress >= IProgressUpdater.PROGRESS_BAR_UPDATE_RESOLUTION) {
+ lastProgress = newProgress;
+ entry.setDownloadProgress(lastProgress);
+ bigFileEntryDao.update(entry);
+ bigFileEntryDao.commit();
+ updater.setProgress(lastProgress);
+ }
+ }
+ };
+ URL website = new URL(url);
+ in = new BufferedInputStream(website.openStream());
+
+ final byte[] data = new byte[DOWNLOAD_BUFFER_SIZE];
+ int count;
+ while ((count = in.read(data, 0, DOWNLOAD_BUFFER_SIZE)) != -1) {
+ cos.write(data, 0, count);
+ }
+
+ entry.setDownloadProgress(IProgressUpdater.MAX_PROGRESS);
+ bigFileEntryDao.update(entry);
+ bigFileEntryDao.commit();
+ updater.setProgress(IProgressUpdater.MAX_PROGRESS);
+
+ output.close();
+
+ return null;
+ } finally {
+ bigFileEntryDao.commit();
+ // close the transaction for this thread
+ dbUtils.closeSessionForCurrentThread();
+ // close opened streams
+ if (in != null) {
+ in.close();
+ }
+ if (cos != null) {
+ cos.close();
+ }
+ }
+ }
+ }
+
+ /**
+ * Checks if local file for given url is up to date. The check is based on file
+ * size check (ftp doesn't provide checksums for files, so we cannot compare it
+ * differently without downloading file).
+ *
+ * @param url
+ * url to ftp file
+ * @return <code>true</code> if file is up to date, <code>false</code> in other
+ * case
+ * @throws URISyntaxException
+ * thrown when url is invalid
+ * @throws IOException
+ * thrown when there is a problem with accessing local or remote file
+ */
+ public boolean isLocalFileUpToDate(String url) throws URISyntaxException, IOException {
+ if (url.toLowerCase().startsWith("ftp")) {
+ return isLocalFtpFileUpToDate(url);
+ } else if (url.toLowerCase().startsWith("http")) {
+ return isLocalHttpFileUpToDate(url);
+ } else {
+ throw new URISyntaxException(url, "Unknown protocol");
+ }
+ }
+
+ /**
+ * Checks if local file fetched from ftp is up to date.
+ *
+ * @param url
+ * url to remote file
+ * @return <code>true</code> if file is up to date, <code>false</code> otherwise
+ * @throws FileNotFoundException
+ * thrown when local file cannot be found
+ * @throws URISyntaxException
+ * thrown when url is invalid
+ * @throws IOException
+ * thrown when there is a problem with connection to remote server
+ */
+ public boolean isLocalFtpFileUpToDate(String url) throws FileNotFoundException, IOException, URISyntaxException {
+ boolean result = false;
+ BigFileEntry entry = bigFileEntryDao.getByUrl(url);
+ if (entry == null) {
+ throw new FileNotFoundException("File wasn't downloaded: " + url);
+ }
+ String path = Configuration.getWebAppDir() + entry.getLocalPath();
+ File f = new File(path);
+ if (!f.exists()) {
+ throw new FileNotFoundException("Missing file: " + path + ". Downloaded from: " + url);
+ }
+ long localSize = f.length();
+
+ FTPClient ftp = ftpClientFactory.createFtpClient();
+ try {
+ String server = getDomainName(url);
+ ftp.connect(server);
+ // After connection attempt, you should check the reply code to verify
+ // success.
+ int reply = ftp.getReplyCode();
+
+ if (!FTPReply.isPositiveCompletion(reply)) {
+ throw new IOException("FTP server refused connection.");
+ }
+ ftp.enterLocalPassiveMode();
+ ftp.login("anonymous", "");
+ ftp.setFileType(FTP.BINARY_FILE_TYPE);
+
+ long fileSize = -1;
+ String remotePath = getFilePath(url);
+ FTPFile[] files = ftp.listFiles(remotePath);
+ if (files.length == 1 && files[0].isFile()) {
+ fileSize = files[0].getSize();
+ }
+
+ result = (fileSize == localSize);
+ ftp.logout();
+ } finally {
+ if (ftp.isConnected()) {
+ ftp.disconnect();
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Checks if local file fetched from http is up to date.
+ *
+ * @param url
+ * url to remote file
+ * @return <code>true</code> if file is up to date, <code>false</code> otherwise
+ * @throws IOException
+ * thrown when local file cannot be found or there is a problem with
+ * accessing remote file
+ */
+ public boolean isLocalHttpFileUpToDate(String url) throws IOException {
+ boolean result = false;
+ BigFileEntry entry = bigFileEntryDao.getByUrl(url);
+ if (entry == null) {
+ throw new FileNotFoundException("File wasn't downloaded: " + url);
+ }
+ String path = Configuration.getWebAppDir() + entry.getLocalPath();
+ File f = new File(path);
+ if (!f.exists()) {
+ throw new FileNotFoundException("Missing file: " + path + ". Downloaded from: " + url);
+ }
+ long localSize = f.length();
+
+ long remoteSize = getRemoteHttpFileSize(url);
+
+ result = (localSize == remoteSize);
+ return result;
+ }
+
+ /**
+ * Returns size of the remote file access via http url.
+ *
+ * @param url
+ * url address to the file
+ * @return size of the remote file access via http url
+ * @throws IOException
+ * thrown when there is a problem with accessing remote file
+ */
+ long getRemoteHttpFileSize(String url) throws IOException {
+ long remoteSize = -1;
+ HttpURLConnection conn = null;
+ try {
+ URL website = new URL(url);
+ conn = (HttpURLConnection) website.openConnection();
+ conn.setRequestMethod("HEAD");
+ conn.getInputStream();
+ remoteSize = conn.getContentLength();
+ } finally {
+ if (conn != null) {
+ conn.disconnect();
+ }
+ }
+ return remoteSize;
+ }
+
+ /**
+ * Removes local file copy of a file given in a parameter.
+ *
+ * @param url
+ * ftp url of a file
+ * @throws IOException
+ * thrown when there is a problem with deleting file
+ */
+ public void removeFile(String url) throws IOException {
+ BigFileEntry entry = bigFileEntryDao.getByUrl(url);
+ if (entry == null) {
+ throw new InvalidArgumentException("Cannot remove file. File wasn't downloaded: " + url);
+ }
+ String path = Configuration.getWebAppDir() + entry.getLocalPath();
+ File f = new File(path);
+ if (!f.exists()) {
+ logger.warn("Missing file: " + path + ". Downloaded from: " + url);
+ }
+ String dirPath = FilenameUtils.getFullPath(path);
+
+ FileUtils.deleteDirectory(new File(dirPath));
+
+ bigFileEntryDao.delete(entry);
+ }
+
+ /**
+ * Task that describes updating file from ftp.
+ *
+ * @author Piotr Gawron
+ *
+ */
+ private final class UpdateFtpFileTask implements Callable<Void> {
+
+ /**
+ * Url to file.
+ */
+ private String url;
+
+ /**
+ * Default constructor.
+ *
+ * @param url
+ * {@link #url}
+ */
+ private UpdateFtpFileTask(String url) {
+ this.url = url;
+ }
+
+ @Override
+ public Void call() throws Exception {
+ dbUtils.createSessionForCurrentThread();
+ FTPClient ftp = ftpClientFactory.createFtpClient();
+ try {
+ BigFileEntry localEntryCopy = bigFileEntryDao.getByUrl(url);
+ localEntryCopy.setDownloadThreadId(Thread.currentThread().getId());
+ localEntryCopy.setDownloadProgress(0.0);
+ bigFileEntryDao.update(localEntryCopy);
+
+ // remove current version
+ new File(Configuration.getWebAppDir() + localEntryCopy.getLocalPath()).delete();
+
+ String server = getDomainName(url);
+ ftp.connect(server);
+ // After connection attempt, you should check the reply code to
+ // verify success.
+ int reply = ftp.getReplyCode();
+
+ if (!FTPReply.isPositiveCompletion(reply)) {
+ throw new IOException("FTP server refused connection.");
+ }
+ ftp.enterLocalPassiveMode();
+ ftp.login("anonymous", "");
+ ftp.setFileType(FTP.BINARY_FILE_TYPE);
+
+ long fileSize = -1;
+ String remotePath = getFilePath(url);
+ FTPFile[] files = ftp.listFiles(remotePath);
+ if (files.length == 1 && files[0].isFile()) {
+ fileSize = files[0].getSize();
+ }
+ final long size = fileSize;
+
+ OutputStream output = new FileOutputStream(localEntryCopy.getLocalPath());
+ CountingOutputStream cos = new CountingOutputStream(output) {
+ private double lastProgress = -1;
+
+ protected void beforeWrite(int n) {
+ super.beforeWrite(n);
+ double newProgress = ((double) getCount()) / ((double) size) * IProgressUpdater.MAX_PROGRESS;
+ if (newProgress - lastProgress >= IProgressUpdater.PROGRESS_BAR_UPDATE_RESOLUTION) {
+ lastProgress = newProgress;
+ localEntryCopy.setDownloadProgress(lastProgress);
+ bigFileEntryDao.update(localEntryCopy);
+ }
+ }
+ };
+ ftp.retrieveFile(remotePath, cos);
+
+ output.close();
+
+ ftp.logout();
+ return null;
+ } finally {
+ dbUtils.closeSessionForCurrentThread();
+ if (ftp.isConnected()) {
+ ftp.disconnect();
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Updates local copy of a file that can be downloaded from a url given in
+ * parameter.
+ *
+ * @param url
+ * url to an ftp file to be updated
+ * @param async
+ * <code>true</code> if update should be done asynchronously,
+ * <code>false</code> otherwise
+ * @throws IOException
+ * thrown when there is a problem with accessing remote file
+ * @throws URISyntaxException
+ * thrown when url is invalid
+ */
+ public void updateFile(String url, boolean async) throws URISyntaxException, IOException {
+ if (isLocalFileUpToDate(url)) {
+ logger.warn("File is up to date. Skipping...");
+ return;
+ }
+ UpdateFtpFileTask updateTask = new UpdateFtpFileTask(url);
+ if (async) {
+ asyncExecutorService.submit(updateTask);
+ } else {
+ Future<Void> task = syncExecutorService.submit(updateTask);
+ executeTask(task);
+ }
+ }
+
+ /**
+ * Executes download/update task.
+ *
+ * @param task
+ * task to be executed
+ * @throws URISyntaxException
+ * thrown when task finished with {@link URISyntaxException}
+ * @throws IOException
+ * thrown when task finished with {@link IOException}
+ */
+ void executeTask(Future<?> task) throws URISyntaxException, IOException {
+ try {
+ task.get();
+ } catch (InterruptedException e) {
+ logger.error(e, e);
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof URISyntaxException) {
+ throw (URISyntaxException) e.getCause();
+ } else if (e.getCause() instanceof IOException) {
+ throw new IOException((IOException) e.getCause());
+ } else {
+ throw new InvalidStateException(e);
+ }
+ }
+ }
+
+ /**
+ * Returns server domain name from url.
+ *
+ * @param url
+ * url to be processed
+ * @return server domain name from url
+ * @throws URISyntaxException
+ * thrown when url is invalid
+ */
+ String getDomainName(String url) throws URISyntaxException {
+ URI uri = new URI(url);
+ String domain = uri.getHost();
+ if (domain.startsWith("www.")) {
+ domain = domain.substring("www.".length());
+ }
+
+ return domain;
+ }
+
+ /**
+ * Returns path to file on server without server domain name from url.
+ *
+ * @param url
+ * url to be processed
+ * @return path to file on server without server domain name from url
+ * @throws URISyntaxException
+ * thrown when url is invalid
+ */
+ public String getFilePath(String url) throws URISyntaxException {
+ URI uri = new URI(url);
+ return uri.getPath();
+ }
+
+ /**
+ * Returns simple file name from url.
+ *
+ * @param url
+ * url to be processed
+ * @return simple file name from url
+ * @throws URISyntaxException
+ * thrown when url is invalid
+ */
+ public String getFileName(String url) throws URISyntaxException {
+ URI uri = new URI(url);
+ return FilenameUtils.getName(uri.getPath());
+ }
+
+ /**
+ * Checks if the file identified by url is cached.
+ *
+ * @param sourceUrl
+ * url that identifies file
+ * @return <code>true</code> if the file is cached, <code>false</code> otherwise
+ */
+ public boolean isCached(String sourceUrl) {
+ BigFileEntry entry = bigFileEntryDao.getByUrl(sourceUrl);
+ if (entry != null) {
+ File f = new File(Configuration.getWebAppDir() + entry.getLocalPath());
+ if (!f.exists()) {
+ logger.warn("File is supposed to be cached but it's not there... " + sourceUrl);
+ bigFileEntryDao.delete(entry);
+ return false;
+ }
+ }
+ return entry != null;
+ }
+
+ /**
+ * @param bigFileEntryDao
+ * the bigFileEntryDao to set
+ * @see #bigFileEntryDao
+ */
+ public void setBigFileEntryDao(BigFileEntryDao bigFileEntryDao) {
+ this.bigFileEntryDao = bigFileEntryDao;
+ }
+
+ /**
+ * Return number of tasks that are executed or are waiting for execution.
+ *
+ * @return number of tasks that are executed or are waiting for execution
+ */
+ public int getDownloadThreadCount() {
+ return ((ScheduledThreadPoolExecutor) asyncExecutorService).getQueue().size()
+ + ((ScheduledThreadPoolExecutor) asyncExecutorService).getActiveCount()
+ + ((ScheduledThreadPoolExecutor) syncExecutorService).getQueue().size()
+ + ((ScheduledThreadPoolExecutor) syncExecutorService).getActiveCount();
+ }
+
+ /**
+ * @param ftpClientFactory
+ * the ftpClientFactory to set
+ * @see #ftpClientFactory
+ */
+ protected void setFtpClientFactory(FtpClientFactory ftpClientFactory) {
+ this.ftpClientFactory = ftpClientFactory;
+ }
+
+ /**
+ * @return the configurationDao
+ * @see #configurationDao
+ */
+ protected ConfigurationDao getConfigurationDao() {
+ return configurationDao;
+ }
+
+ /**
+ * @param configurationDao
+ * the configurationDao to set
+ * @see #configurationDao
+ */
+ protected void setConfigurationDao(ConfigurationDao configurationDao) {
+ this.configurationDao = configurationDao;
+ }
}
\ No newline at end of file
diff --git a/annotation/src/test/java/lcsb/mapviewer/annotation/cache/BigFileCacheTest.java b/annotation/src/test/java/lcsb/mapviewer/annotation/cache/BigFileCacheTest.java
index 87ca09b4d0e98ad99823475f6e9f9fa28dd17f07..fef9e3780344364b4741803c8e97a8f7691894e0 100644
--- a/annotation/src/test/java/lcsb/mapviewer/annotation/cache/BigFileCacheTest.java
+++ b/annotation/src/test/java/lcsb/mapviewer/annotation/cache/BigFileCacheTest.java
@@ -41,544 +41,544 @@ import lcsb.mapviewer.persist.dao.cache.BigFileEntryDao;
public class BigFileCacheTest extends AnnotationTestFunctions {
- Logger logger = Logger.getLogger(BigFileCacheTest.class);
-
- String ftpUrl = "ftp://ftp.informatik.rwth-aachen.de/pub/Linux/debian-cd/current/i386/bt-cd/MD5SUMS";
-
- String httpUrl = "http://minerva.uni.lu/humans.txt";
-
- @Autowired
- BigFileCache bigFileCache;
-
- @Before
- public void setUp() throws Exception {
- }
-
- @After
- public void tearDown() throws Exception {
- }
-
- @Test
- public void testDownloadFile() throws Exception {
- try {
- bigFileCache.downloadFile(ftpUrl, false, null);
- String localFile = bigFileCache.getAbsolutePathForFile(ftpUrl);
- assertTrue(bigFileCache.isCached(ftpUrl));
- assertTrue(bigFileCache.isLocalFileUpToDate(ftpUrl));
- File f = new File(localFile);
- assertTrue(f.exists());
- String content = super.readFile(localFile);
- assertTrue(content.contains("iso"));
-
- // when we try to download file that is already downloaded there should be
- // a warning
- bigFileCache.downloadFile(ftpUrl, false, null);
- assertEquals(1, getWarnings().size());
-
- // however when due to some reason the file will be removed from file
- // system
- f.delete();
- // the file should be downloaded again
- bigFileCache.downloadFile(ftpUrl, false, null);
- // additional warning about it
- assertEquals(2, getWarnings().size());
-
- localFile = bigFileCache.getAbsolutePathForFile(ftpUrl);
- f = new File(localFile);
- assertTrue(f.exists());
-
- // when we are trying to update file that is up to date then we will have
- // a warning about it (but file shouldn't be changed)
- bigFileCache.updateFile(ftpUrl, false);
- assertEquals(3, getWarnings().size());
-
- bigFileCache.removeFile(ftpUrl);
-
- assertNull(bigFileCache.getAbsolutePathForFile(ftpUrl));
-
- assertFalse(f.exists());
-
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testDownloadFileWithUnknownProtocol() throws Exception {
- try {
- bigFileCache.downloadFile("wtf://file.com", false, null);
-
- } catch (URISyntaxException e) {
- assertTrue(e.getMessage().contains("Unknown protocol for url"));
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testDownloadAsyncFileFromHttp() throws Exception {
- try {
- bigFileCache.downloadFile(httpUrl, true, null);
- waitForDownload();
- String localFile = bigFileCache.getAbsolutePathForFile(httpUrl);
- assertTrue(bigFileCache.isLocalFileUpToDate(httpUrl));
- File f = new File(localFile);
- assertTrue(f.exists());
- String content = super.readFile(localFile);
- assertTrue(content.contains("Contact"));
- bigFileCache.removeFile(httpUrl);
-
- assertNull(bigFileCache.getAbsolutePathForFile(httpUrl));
-
- assertFalse(f.exists());
-
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- private void waitForDownload() throws InterruptedException {
- while (bigFileCache.getDownloadThreadCount() > 0) {
- logger.debug("Waiting for download to finish");
- Thread.sleep(100);
- }
- }
-
- @Test
- public void testIsLocalHttpFileUpToDateWhenMissing() throws Exception {
- try {
- bigFileCache.isLocalFileUpToDate(httpUrl);
- fail("Exception expected");
- } catch (FileNotFoundException e) {
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testIsLocalHttpFileUpToDateWhenFileRemovedManually() throws Exception {
- try {
- bigFileCache.downloadFile(httpUrl, false, new IProgressUpdater() {
- @Override
- public void setProgress(double progress) {
- }
- });
- new File(bigFileCache.getAbsolutePathForFile(httpUrl)).delete();
- try {
- bigFileCache.isLocalFileUpToDate(httpUrl);
- fail("Exception expected");
- } catch (FileNotFoundException e) {
- }
- bigFileCache.removeFile(httpUrl);
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testIsLocalFtpFileUpToDateWhenMissing() throws Exception {
- try {
- bigFileCache.isLocalFileUpToDate(ftpUrl);
- fail("Exception expected");
- } catch (FileNotFoundException e) {
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testRefreshCacheQueryWhenFtpConnectionFails() throws Exception {
-
- try {
- bigFileCache.downloadFile(ftpUrl, false, new IProgressUpdater() {
- @Override
- public void setProgress(double progress) {
- }
- });
- assertTrue(bigFileCache.isLocalFileUpToDate(ftpUrl));
-
- FTPClient mockClient = Mockito.mock(FTPClient.class);
- Mockito.doAnswer(new Answer<Integer>() {
-
- @Override
- public Integer answer(InvocationOnMock invocation) throws Throwable {
- return FTPReply.REQUEST_DENIED;
- }
- }).when(mockClient).getReplyCode();
-
- FtpClientFactory factory = Mockito.mock(FtpClientFactory.class);
- when(factory.createFtpClient()).thenReturn(mockClient);
-
- bigFileCache.setFtpClientFactory(factory);
-
- bigFileCache.isLocalFileUpToDate(ftpUrl);
- fail("Exception expected");
- } catch (IOException e) {
- assertTrue(e.getMessage().contains("FTP server refused connection"));
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- } finally {
- bigFileCache.setFtpClientFactory(new FtpClientFactory());
- bigFileCache.removeFile(ftpUrl);
- }
- }
-
- @Test
- public void testDownloadWhenFtpConnectionFails() throws Exception {
- try {
- FTPClient mockClient = Mockito.mock(FTPClient.class);
- Mockito.doAnswer(new Answer<Integer>() {
- @Override
- public Integer answer(InvocationOnMock invocation) throws Throwable {
- return FTPReply.REQUEST_DENIED;
- }
- }).when(mockClient).getReplyCode();
-
- FtpClientFactory factory = Mockito.mock(FtpClientFactory.class);
- when(factory.createFtpClient()).thenReturn(mockClient);
-
- bigFileCache.setFtpClientFactory(factory);
- bigFileCache.downloadFile(ftpUrl, false, null);
-
- fail("Exception expected");
- } catch (IOException e) {
- assertTrue(e.getMessage().contains("FTP server refused connection"));
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- } finally {
- bigFileCache.setFtpClientFactory(new FtpClientFactory());
- bigFileCache.removeFile(ftpUrl);
- }
- }
-
- @Test
- public void testDownloadHttpWhenDownloadFails() throws Exception {
- ConfigurationDao configurationDao = bigFileCache.getConfigurationDao();
-
- try {
-
- ConfigurationDao mockDao = Mockito.mock(ConfigurationDao.class);
- when(mockDao.getValueByType(any())).thenThrow(new RuntimeException());
- bigFileCache.setConfigurationDao(mockDao);
- bigFileCache.downloadFile(httpUrl, false, null);
-
- fail("Exception expected");
- } catch (InvalidStateException e) {
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- } finally {
- bigFileCache.setConfigurationDao(configurationDao);
- }
- }
-
- @Test
- public void testDownloadedFileRemovedException() throws Exception {
- try {
- String url = ftpUrl;
- bigFileCache.downloadFile(url, false, new IProgressUpdater() {
- @Override
- public void setProgress(double progress) {
- }
- });
- String localFile = bigFileCache.getAbsolutePathForFile(url);
- File f = new File(localFile);
- assertTrue(f.exists());
- f.delete();
-
- try {
- bigFileCache.getAbsolutePathForFile(url);
- fail("Exception expected");
- } catch (FileNotFoundException e) {
-
- }
- try {
- bigFileCache.isLocalFileUpToDate(url);
- fail("Exception expected");
- } catch (FileNotFoundException e) {
-
- }
-
- bigFileCache.removeFile(url);
-
- assertNull(bigFileCache.getAbsolutePathForFile(url));
-
- assertFalse(f.exists());
-
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testGetPathForFileBeingDownloaded() throws Exception {
- try {
- String url = ftpUrl;
- BigFileCache bigFileCacheUnderTest = new BigFileCache();
- BigFileEntryDao mockDao = Mockito.mock(BigFileEntryDao.class);
- BigFileEntry entry = new BigFileEntry();
- when(mockDao.getByUrl(anyString())).thenReturn(entry);
-
- bigFileCacheUnderTest.setBigFileEntryDao(mockDao);
- try {
- bigFileCacheUnderTest.getAbsolutePathForFile(url);
- fail("Exception expected");
- } catch (FileNotFoundException e) {
- assertTrue(e.getMessage().contains("File is not complete"));
- }
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testUpdateFile() throws Exception {
- try {
- bigFileCache.downloadFile(ftpUrl, false, new IProgressUpdater() {
- @Override
- public void setProgress(double progress) {
- }
- });
- String localFile = bigFileCache.getAbsolutePathForFile(ftpUrl);
- File f = new File(localFile);
- f.delete();
- PrintWriter out = new PrintWriter(localFile);
- out.println("tesxt");
- out.close();
-
- String content = super.readFile(localFile);
- assertFalse(content.contains("iso"));
-
- assertFalse(bigFileCache.isLocalFileUpToDate(ftpUrl));
- bigFileCache.updateFile(ftpUrl, false);
-
- localFile = bigFileCache.getAbsolutePathForFile(ftpUrl);
-
- content = super.readFile(localFile);
- assertTrue(content.contains("iso"));
-
- // now try update asynchronously
- f = new File(localFile);
- f.delete();
- out = new PrintWriter(localFile);
- out.println("tesxt");
- out.close();
-
- assertFalse(bigFileCache.isLocalFileUpToDate(ftpUrl));
- bigFileCache.updateFile(ftpUrl, true);
-
- waitForDownload();
-
- assertTrue(bigFileCache.isLocalFileUpToDate(ftpUrl));
-
- // check when connection fails
- f = new File(localFile);
- f.delete();
- out = new PrintWriter(localFile);
- out.println("tesxt");
- out.close();
-
- FTPClient mockClient = Mockito.mock(FTPClient.class);
- Mockito.doAnswer(new Answer<Integer>() {
-
- //first answer (for checking if the file should be updated)
- @Override
- public Integer answer(InvocationOnMock invocation) throws Throwable {
- return FTPReply.COMMAND_OK;
- }
- }).doAnswer(new Answer<Integer>() {
- //second answer (for downloading)
- @Override
- public Integer answer(InvocationOnMock invocation) throws Throwable {
- return FTPReply.REQUEST_DENIED;
- }
- }).when(mockClient).getReplyCode();
-
- Mockito.doAnswer(new Answer<FTPFile[]>() {
-
- @Override
- public FTPFile[] answer(InvocationOnMock invocation) throws Throwable {
- return new FTPFile[]{};
- }
- }).when(mockClient).listFiles(anyString());
-
- FtpClientFactory factory = Mockito.mock(FtpClientFactory.class);
- when(factory.createFtpClient()).thenReturn(mockClient);
-
- bigFileCache.setFtpClientFactory(factory);
- try {
- bigFileCache.updateFile(ftpUrl, false);
- fail("Exception expected");
- } catch (IOException e) {
- assertTrue(e.getMessage().contains("FTP server refused connection"));
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- } finally {
- bigFileCache.setFtpClientFactory(new FtpClientFactory());
- }
-
- bigFileCache.removeFile(ftpUrl);
-
- assertNull(bigFileCache.getAbsolutePathForFile(ftpUrl));
-
- assertFalse(f.exists());
-
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testGetRemoteHttpFileSizeForInvalidUrl() throws Exception {
- try {
- bigFileCache.getRemoteHttpFileSize("invalidurl");
- fail("Exception expected");
- } catch (IOException e) {
-
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testRemoveFileThatDoesntExist() throws Exception {
- try {
- bigFileCache.removeFile("unexisting file");
- fail("Exception expected");
- } catch (InvalidArgumentException e) {
- assertTrue(e.getMessage().contains("Cannot remove file. File wasn't downloaded"));
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testIsLocalFileUpToDateForInvalidURI() throws Exception {
- try {
- bigFileCache.isLocalFileUpToDate("unexistingFileProtocol://bla");
- fail("Exception expected");
- } catch (URISyntaxException e) {
- assertTrue(e.getMessage().contains("Unknown protocol"));
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testGetDomainName() throws Exception {
- try {
- String domain = bigFileCache.getDomainName("http://www.eskago.pl/radio/eska-rock");
- assertEquals("eskago.pl", domain);
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testExecuteProblematicTask() throws Exception {
- try {
- Future<?> task = Executors.newScheduledThreadPool(10, new ThreadFactory() {
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new Thread(r);
- t.setDaemon(true);
- return t;
- }
- }).submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- throw new Exception();
- }
- });
- bigFileCache.executeTask(task);
- fail("Exception expected");
- } catch (InvalidStateException e) {
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testExecuteProblematicTask2() throws Exception {
- try {
- Future<?> task = Executors.newScheduledThreadPool(10, new ThreadFactory() {
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new Thread(r);
- t.setDaemon(true);
- return t;
- }
- }).submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- throw new URISyntaxException("","");
- }
- });
- bigFileCache.executeTask(task);
- fail("Exception expected");
- } catch (URISyntaxException e) {
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- @Test
- public void testExecuteInterruptedTask() throws Exception {
- try {
- Future<?> task = Executors.newScheduledThreadPool(10, new ThreadFactory() {
- @Override
- public Thread newThread(Runnable r) {
- Thread t = new Thread(r);
- t.setDaemon(true);
- return t;
- }
- }).submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- Thread.sleep(100);
- return null;
- }
- });
- Thread t = new Thread(new Runnable() {
- @Override
- public void run() {
- try {
- bigFileCache.executeTask(task);
- } catch (Exception e) {
- logger.fatal(e,e);
- }
- }
- });
-
- t.start();
- // interrupt thread
- t.interrupt();
- t.join();
-
- assertEquals(1, getErrors().size());
- assertEquals(0, getFatals().size());
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
+ Logger logger = Logger.getLogger(BigFileCacheTest.class);
+
+ String ftpUrl = "ftp://ftp.informatik.rwth-aachen.de/pub/Linux/debian-cd/current/i386/bt-cd/MD5SUMS";
+
+ String httpUrl = "https://www.google.pl/humans.txt";
+
+ @Autowired
+ BigFileCache bigFileCache;
+
+ @Before
+ public void setUp() throws Exception {
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public void testDownloadFile() throws Exception {
+ try {
+ bigFileCache.downloadFile(ftpUrl, false, null);
+ String localFile = bigFileCache.getAbsolutePathForFile(ftpUrl);
+ assertTrue(bigFileCache.isCached(ftpUrl));
+ assertTrue(bigFileCache.isLocalFileUpToDate(ftpUrl));
+ File f = new File(localFile);
+ assertTrue(f.exists());
+ String content = super.readFile(localFile);
+ assertTrue(content.contains("iso"));
+
+ // when we try to download file that is already downloaded there should be
+ // a warning
+ bigFileCache.downloadFile(ftpUrl, false, null);
+ assertEquals(1, getWarnings().size());
+
+ // however when due to some reason the file will be removed from file
+ // system
+ f.delete();
+ // the file should be downloaded again
+ bigFileCache.downloadFile(ftpUrl, false, null);
+ // additional warning about it
+ assertEquals(2, getWarnings().size());
+
+ localFile = bigFileCache.getAbsolutePathForFile(ftpUrl);
+ f = new File(localFile);
+ assertTrue(f.exists());
+
+ // when we are trying to update file that is up to date then we will have
+ // a warning about it (but file shouldn't be changed)
+ bigFileCache.updateFile(ftpUrl, false);
+ assertEquals(3, getWarnings().size());
+
+ bigFileCache.removeFile(ftpUrl);
+
+ assertNull(bigFileCache.getAbsolutePathForFile(ftpUrl));
+
+ assertFalse(f.exists());
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testDownloadFileWithUnknownProtocol() throws Exception {
+ try {
+ bigFileCache.downloadFile("wtf://file.com", false, null);
+
+ } catch (URISyntaxException e) {
+ assertTrue(e.getMessage().contains("Unknown protocol for url"));
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testDownloadAsyncFileFromHttp() throws Exception {
+ try {
+ bigFileCache.downloadFile(httpUrl, true, null);
+ waitForDownload();
+ String localFile = bigFileCache.getAbsolutePathForFile(httpUrl);
+ assertTrue(bigFileCache.isLocalFileUpToDate(httpUrl));
+ File f = new File(localFile);
+ assertTrue(f.exists());
+ String content = super.readFile(localFile);
+ assertTrue(content.contains("Google"));
+ bigFileCache.removeFile(httpUrl);
+
+ assertNull(bigFileCache.getAbsolutePathForFile(httpUrl));
+
+ assertFalse(f.exists());
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ private void waitForDownload() throws InterruptedException {
+ while (bigFileCache.getDownloadThreadCount() > 0) {
+ logger.debug("Waiting for download to finish");
+ Thread.sleep(100);
+ }
+ }
+
+ @Test
+ public void testIsLocalHttpFileUpToDateWhenMissing() throws Exception {
+ try {
+ bigFileCache.isLocalFileUpToDate(httpUrl + "?some_get_param");
+ fail("Exception expected");
+ } catch (FileNotFoundException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testIsLocalHttpFileUpToDateWhenFileRemovedManually() throws Exception {
+ try {
+ bigFileCache.downloadFile(httpUrl, false, new IProgressUpdater() {
+ @Override
+ public void setProgress(double progress) {
+ }
+ });
+ new File(bigFileCache.getAbsolutePathForFile(httpUrl)).delete();
+ try {
+ bigFileCache.isLocalFileUpToDate(httpUrl);
+ fail("Exception expected");
+ } catch (FileNotFoundException e) {
+ }
+ bigFileCache.removeFile(httpUrl);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testIsLocalFtpFileUpToDateWhenMissing() throws Exception {
+ try {
+ bigFileCache.isLocalFileUpToDate(ftpUrl);
+ fail("Exception expected");
+ } catch (FileNotFoundException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testRefreshCacheQueryWhenFtpConnectionFails() throws Exception {
+
+ try {
+ bigFileCache.downloadFile(ftpUrl, false, new IProgressUpdater() {
+ @Override
+ public void setProgress(double progress) {
+ }
+ });
+ assertTrue(bigFileCache.isLocalFileUpToDate(ftpUrl));
+
+ FTPClient mockClient = Mockito.mock(FTPClient.class);
+ Mockito.doAnswer(new Answer<Integer>() {
+
+ @Override
+ public Integer answer(InvocationOnMock invocation) throws Throwable {
+ return FTPReply.REQUEST_DENIED;
+ }
+ }).when(mockClient).getReplyCode();
+
+ FtpClientFactory factory = Mockito.mock(FtpClientFactory.class);
+ when(factory.createFtpClient()).thenReturn(mockClient);
+
+ bigFileCache.setFtpClientFactory(factory);
+
+ bigFileCache.isLocalFileUpToDate(ftpUrl);
+ fail("Exception expected");
+ } catch (IOException e) {
+ assertTrue(e.getMessage().contains("FTP server refused connection"));
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ } finally {
+ bigFileCache.setFtpClientFactory(new FtpClientFactory());
+ bigFileCache.removeFile(ftpUrl);
+ }
+ }
+
+ @Test
+ public void testDownloadWhenFtpConnectionFails() throws Exception {
+ try {
+ FTPClient mockClient = Mockito.mock(FTPClient.class);
+ Mockito.doAnswer(new Answer<Integer>() {
+ @Override
+ public Integer answer(InvocationOnMock invocation) throws Throwable {
+ return FTPReply.REQUEST_DENIED;
+ }
+ }).when(mockClient).getReplyCode();
+
+ FtpClientFactory factory = Mockito.mock(FtpClientFactory.class);
+ when(factory.createFtpClient()).thenReturn(mockClient);
+
+ bigFileCache.setFtpClientFactory(factory);
+ bigFileCache.downloadFile(ftpUrl, false, null);
+
+ fail("Exception expected");
+ } catch (IOException e) {
+ assertTrue(e.getMessage().contains("FTP server refused connection"));
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ } finally {
+ bigFileCache.setFtpClientFactory(new FtpClientFactory());
+ bigFileCache.removeFile(ftpUrl);
+ }
+ }
+
+ @Test
+ public void testDownloadHttpWhenDownloadFails() throws Exception {
+ ConfigurationDao configurationDao = bigFileCache.getConfigurationDao();
+
+ try {
+
+ ConfigurationDao mockDao = Mockito.mock(ConfigurationDao.class);
+ when(mockDao.getValueByType(any())).thenThrow(new RuntimeException());
+ bigFileCache.setConfigurationDao(mockDao);
+ bigFileCache.downloadFile(httpUrl, false, null);
+
+ fail("Exception expected");
+ } catch (InvalidStateException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ } finally {
+ bigFileCache.setConfigurationDao(configurationDao);
+ }
+ }
+
+ @Test
+ public void testDownloadedFileRemovedException() throws Exception {
+ try {
+ String url = ftpUrl;
+ bigFileCache.downloadFile(url, false, new IProgressUpdater() {
+ @Override
+ public void setProgress(double progress) {
+ }
+ });
+ String localFile = bigFileCache.getAbsolutePathForFile(url);
+ File f = new File(localFile);
+ assertTrue(f.exists());
+ f.delete();
+
+ try {
+ bigFileCache.getAbsolutePathForFile(url);
+ fail("Exception expected");
+ } catch (FileNotFoundException e) {
+
+ }
+ try {
+ bigFileCache.isLocalFileUpToDate(url);
+ fail("Exception expected");
+ } catch (FileNotFoundException e) {
+
+ }
+
+ bigFileCache.removeFile(url);
+
+ assertNull(bigFileCache.getAbsolutePathForFile(url));
+
+ assertFalse(f.exists());
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testGetPathForFileBeingDownloaded() throws Exception {
+ try {
+ String url = ftpUrl;
+ BigFileCache bigFileCacheUnderTest = new BigFileCache();
+ BigFileEntryDao mockDao = Mockito.mock(BigFileEntryDao.class);
+ BigFileEntry entry = new BigFileEntry();
+ when(mockDao.getByUrl(anyString())).thenReturn(entry);
+
+ bigFileCacheUnderTest.setBigFileEntryDao(mockDao);
+ try {
+ bigFileCacheUnderTest.getAbsolutePathForFile(url);
+ fail("Exception expected");
+ } catch (FileNotFoundException e) {
+ assertTrue(e.getMessage().contains("File is not complete"));
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testUpdateFile() throws Exception {
+ try {
+ bigFileCache.downloadFile(ftpUrl, false, new IProgressUpdater() {
+ @Override
+ public void setProgress(double progress) {
+ }
+ });
+ String localFile = bigFileCache.getAbsolutePathForFile(ftpUrl);
+ File f = new File(localFile);
+ f.delete();
+ PrintWriter out = new PrintWriter(localFile);
+ out.println("tesxt");
+ out.close();
+
+ String content = super.readFile(localFile);
+ assertFalse(content.contains("iso"));
+
+ assertFalse(bigFileCache.isLocalFileUpToDate(ftpUrl));
+ bigFileCache.updateFile(ftpUrl, false);
+
+ localFile = bigFileCache.getAbsolutePathForFile(ftpUrl);
+
+ content = super.readFile(localFile);
+ assertTrue(content.contains("iso"));
+
+ // now try update asynchronously
+ f = new File(localFile);
+ f.delete();
+ out = new PrintWriter(localFile);
+ out.println("tesxt");
+ out.close();
+
+ assertFalse(bigFileCache.isLocalFileUpToDate(ftpUrl));
+ bigFileCache.updateFile(ftpUrl, true);
+
+ waitForDownload();
+
+ assertTrue(bigFileCache.isLocalFileUpToDate(ftpUrl));
+
+ // check when connection fails
+ f = new File(localFile);
+ f.delete();
+ out = new PrintWriter(localFile);
+ out.println("tesxt");
+ out.close();
+
+ FTPClient mockClient = Mockito.mock(FTPClient.class);
+ Mockito.doAnswer(new Answer<Integer>() {
+
+ // first answer (for checking if the file should be updated)
+ @Override
+ public Integer answer(InvocationOnMock invocation) throws Throwable {
+ return FTPReply.COMMAND_OK;
+ }
+ }).doAnswer(new Answer<Integer>() {
+ // second answer (for downloading)
+ @Override
+ public Integer answer(InvocationOnMock invocation) throws Throwable {
+ return FTPReply.REQUEST_DENIED;
+ }
+ }).when(mockClient).getReplyCode();
+
+ Mockito.doAnswer(new Answer<FTPFile[]>() {
+
+ @Override
+ public FTPFile[] answer(InvocationOnMock invocation) throws Throwable {
+ return new FTPFile[] {};
+ }
+ }).when(mockClient).listFiles(anyString());
+
+ FtpClientFactory factory = Mockito.mock(FtpClientFactory.class);
+ when(factory.createFtpClient()).thenReturn(mockClient);
+
+ bigFileCache.setFtpClientFactory(factory);
+ try {
+ bigFileCache.updateFile(ftpUrl, false);
+ fail("Exception expected");
+ } catch (IOException e) {
+ assertTrue(e.getMessage().contains("FTP server refused connection"));
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ } finally {
+ bigFileCache.setFtpClientFactory(new FtpClientFactory());
+ }
+
+ bigFileCache.removeFile(ftpUrl);
+
+ assertNull(bigFileCache.getAbsolutePathForFile(ftpUrl));
+
+ assertFalse(f.exists());
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testGetRemoteHttpFileSizeForInvalidUrl() throws Exception {
+ try {
+ bigFileCache.getRemoteHttpFileSize("invalidurl");
+ fail("Exception expected");
+ } catch (IOException e) {
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testRemoveFileThatDoesntExist() throws Exception {
+ try {
+ bigFileCache.removeFile("unexisting file");
+ fail("Exception expected");
+ } catch (InvalidArgumentException e) {
+ assertTrue(e.getMessage().contains("Cannot remove file. File wasn't downloaded"));
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testIsLocalFileUpToDateForInvalidURI() throws Exception {
+ try {
+ bigFileCache.isLocalFileUpToDate("unexistingFileProtocol://bla");
+ fail("Exception expected");
+ } catch (URISyntaxException e) {
+ assertTrue(e.getMessage().contains("Unknown protocol"));
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testGetDomainName() throws Exception {
+ try {
+ String domain = bigFileCache.getDomainName("http://www.eskago.pl/radio/eska-rock");
+ assertEquals("eskago.pl", domain);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testExecuteProblematicTask() throws Exception {
+ try {
+ Future<?> task = Executors.newScheduledThreadPool(10, new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setDaemon(true);
+ return t;
+ }
+ }).submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ throw new Exception();
+ }
+ });
+ bigFileCache.executeTask(task);
+ fail("Exception expected");
+ } catch (InvalidStateException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testExecuteProblematicTask2() throws Exception {
+ try {
+ Future<?> task = Executors.newScheduledThreadPool(10, new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setDaemon(true);
+ return t;
+ }
+ }).submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ throw new URISyntaxException("", "");
+ }
+ });
+ bigFileCache.executeTask(task);
+ fail("Exception expected");
+ } catch (URISyntaxException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testExecuteInterruptedTask() throws Exception {
+ try {
+ Future<?> task = Executors.newScheduledThreadPool(10, new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(r);
+ t.setDaemon(true);
+ return t;
+ }
+ }).submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ Thread.sleep(100);
+ return null;
+ }
+ });
+ Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ bigFileCache.executeTask(task);
+ } catch (Exception e) {
+ logger.fatal(e, e);
+ }
+ }
+ });
+
+ t.start();
+ // interrupt thread
+ t.interrupt();
+ t.join();
+
+ assertEquals(1, getErrors().size());
+ assertEquals(0, getFatals().size());
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
}