From d7110f16fa4e7253dfea581d61fc37081f6be6ba Mon Sep 17 00:00:00 2001 From: Michael Petretti Date: Wed, 2 Jul 2014 16:33:29 +0200 Subject: Adappted to the new Filetransfer class. Ready for testing. --- .../satellitedaemon/ftp/FtpDownloadWorker.java | 126 ++++++++++++-------- .../satellitedaemon/ftp/FtpUploadWorker.java | 104 ++++++++++------ .../satellitedaemon/ftp/ThriftConnection.java | 131 +++++++++------------ 3 files changed, 201 insertions(+), 160 deletions(-) diff --git a/src/main/java/org/openslx/satellitedaemon/ftp/FtpDownloadWorker.java b/src/main/java/org/openslx/satellitedaemon/ftp/FtpDownloadWorker.java index bcf217f..76f03cf 100644 --- a/src/main/java/org/openslx/satellitedaemon/ftp/FtpDownloadWorker.java +++ b/src/main/java/org/openslx/satellitedaemon/ftp/FtpDownloadWorker.java @@ -1,24 +1,22 @@ package org.openslx.satellitedaemon.ftp; -import java.io.File; import java.io.FileInputStream; -import java.io.FileOutputStream; +import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; +import java.security.KeyManagementException; import java.security.KeyStore; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.List; -import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; -import org.apache.commons.net.ftp.FTP; -import org.apache.commons.net.ftp.FTPSClient; import org.apache.log4j.Logger; -import org.openslx.imagemaster.thrift.iface.FtpCredentials; +import org.openslx.filetransfer.Downloader; +import org.openslx.imagemaster.thrift.iface.DownloadInfos; import org.openslx.satellitedaemon.Globals; import org.openslx.satellitedaemon.Globals.PropInt; import org.openslx.satellitedaemon.Globals.PropString; @@ -36,50 +34,86 @@ public class FtpDownloadWorker implements Runnable log.info( "FtpDownloadWorker: imageList Contains " + imageList.size() + " items." ); for ( DbImage image : imageList ) { - FtpCredentials ftpc = ThriftConnection.getFtpCredentials( image.guid ); - if ( ftpc == null ) { - log.error( "The FtpCredentials are null" ); + DownloadInfos downInfos = ThriftConnection.getDownloadInfos( image ); + if ( downInfos == null ) { + log.error( "The DownloadInfos returned by ThriftConnection class are null" ); } + char[] passphrase = Globals.getPropertyString( PropString.FTPSKEYSTOREPWD ).toCharArray(); + KeyStore keystore; try { - TrustManagerFactory trustManagerFactory = TrustManagerFactory - .getInstance( KeyManagerFactory.getDefaultAlgorithm() ); - KeyStore keystore = KeyStore.getInstance( Globals.getPropertyString( PropString.KEYSTORETYPE ) ); - keystore.load( new FileInputStream( new File( - Globals.getPropertyString( PropString.FTPSKEYSTOREPATH ) ) ), - Globals.getPropertyString( PropString.FTPSKEYSTOREPWD ).toCharArray() ); - trustManagerFactory.init( keystore ); - TrustManager trustManager = trustManagerFactory.getTrustManagers()[0]; - FTPSClient ftpClient = new FTPSClient( "SSL", true ); - ftpClient.setTrustManager( trustManager ); - try { - ftpClient.connect( Globals.getPropertyString( PropString.FTPSERVERIP ), Globals.getPropertyInt( PropInt.FTPPORT ) ); - if ( !ftpClient.login( ftpc.username, ftpc.password ) ) { - log.error( "FTP problem. Coundn't log in!" ); - } - File file = new File( "/tmp/" + image.guid + ".vmdk"); - ftpClient.setFileType( FTP.BINARY_FILE_TYPE ); - log.info( "FtpDownloadWorker: ftpc.filename: " + ftpc.filename ); - InputStream is = ftpClient.retrieveFileStream( ftpc.filename ); - FileOutputStream fos = new FileOutputStream( file ); - int b; - while ((b = is.read()) != -1) { - fos.write( b ); - } - is.close(); - fos.close(); - ThriftConnection.finishedDownload( ftpc.username); + keystore = KeyStore.getInstance( "JKS" ); + keystore.load( new FileInputStream( Globals.getPropertyString( PropString.FTPSKEYSTOREPATH ) ), passphrase ); + TrustManagerFactory tmf = TrustManagerFactory.getInstance( TrustManagerFactory.getDefaultAlgorithm() ); + tmf.init( keystore ); + SSLContext context = SSLContext.getInstance( "SSLv3" ); + TrustManager[] trustManagers = tmf.getTrustManagers(); + context.init( null, trustManagers, null ); - } catch (IOException e) { - log.error("FtpDownloadWorker: Error creating the FileInputStream"); - } - finally { - ftpClient.disconnect(); - log.info( "FtpDownloadWorker: ftpClient disconnected" ); - } - } catch ( NoSuchAlgorithmException | KeyStoreException | CertificateException | IOException e ) { - log.debug( "FtpDownloadWorker: Problem with Keystore ore FtpsClient creation." ); + Downloader d = new Downloader( Globals.getPropertyString( PropString.FTPSERVERIP ), Globals.getPropertyInt( PropInt.FTPPORT ), context ); + d.sendToken( downInfos.token ); + while ( d.readMetaData() ) + d.readBinary(); + } catch ( NoSuchAlgorithmException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( CertificateException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( FileNotFoundException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( IOException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( KeyStoreException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( KeyManagementException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); } + + + // try { + // TrustManagerFactory trustManagerFactory = TrustManagerFactory + // .getInstance( KeyManagerFactory.getDefaultAlgorithm() ); + // KeyStore keystore = KeyStore.getInstance( Globals.getPropertyString( PropString.KEYSTORETYPE ) ); + // keystore.load( new FileInputStream( new File( + // Globals.getPropertyString( PropString.FTPSKEYSTOREPATH ) ) ), + // Globals.getPropertyString( PropString.FTPSKEYSTOREPWD ).toCharArray() ); + // trustManagerFactory.init( keystore ); + // TrustManager trustManager = trustManagerFactory.getTrustManagers()[0]; + // FTPSClient ftpClient = new FTPSClient( "SSL", true ); + // ftpClient.setTrustManager( trustManager ); + // try { + // ftpClient.connect( Globals.getPropertyString( PropString.FTPSERVERIP ), Globals.getPropertyInt( PropInt.FTPPORT ) ); + // if ( !ftpClient.login( ftpc.username, ftpc.password ) ) { + // log.error( "FTP problem. Coundn't log in!" ); + // } + // File file = new File( "/tmp/" + image.guid + ".vmdk"); + // ftpClient.setFileType( FTP.BINARY_FILE_TYPE ); + // log.info( "FtpDownloadWorker: ftpc.filename: " + ftpc.filename ); + // InputStream is = ftpClient.retrieveFileStream( ftpc.filename ); + // FileOutputStream fos = new FileOutputStream( file ); + // int b; + // while ((b = is.read()) != -1) { + // fos.write( b ); + // } + // is.close(); + // fos.close(); + // ThriftConnection.finishedDownload( ftpc.username); + // + // } catch (IOException e) { + // log.error("FtpDownloadWorker: Error creating the FileInputStream"); + // } + // finally { + // ftpClient.disconnect(); + // log.info( "FtpDownloadWorker: ftpClient disconnected" ); + // } + // } catch ( NoSuchAlgorithmException | KeyStoreException | CertificateException | IOException e ) { + // log.debug( "FtpDownloadWorker: Problem with Keystore ore FtpsClient creation." ); + // } } try { Thread.sleep( 5 * 60 * 1000 ); diff --git a/src/main/java/org/openslx/satellitedaemon/ftp/FtpUploadWorker.java b/src/main/java/org/openslx/satellitedaemon/ftp/FtpUploadWorker.java index 13f3ae8..3ee624e 100644 --- a/src/main/java/org/openslx/satellitedaemon/ftp/FtpUploadWorker.java +++ b/src/main/java/org/openslx/satellitedaemon/ftp/FtpUploadWorker.java @@ -1,23 +1,23 @@ package org.openslx.satellitedaemon.ftp; -import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; +import java.security.KeyManagementException; import java.security.KeyStore; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.List; -import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; -import org.apache.commons.net.ftp.FTP; -import org.apache.commons.net.ftp.FTPSClient; import org.apache.log4j.Logger; -import org.openslx.imagemaster.thrift.iface.FtpCredentials; +import org.openslx.filetransfer.Uploader; import org.openslx.imagemaster.thrift.iface.ImageData; +import org.openslx.imagemaster.thrift.iface.UploadInfos; import org.openslx.satellitedaemon.Globals; import org.openslx.satellitedaemon.Globals.PropInt; import org.openslx.satellitedaemon.Globals.PropString; @@ -31,50 +31,78 @@ public class FtpUploadWorker implements Runnable public void run() { while ( true ) { + // This List contains all Images in the Database that should be uploaded. List imageList = DbImage.getAllMarkedForUpload(); log.info( "FtpUploadWorker: imageList Contains " + imageList.size() + " items." ); + + // Upload one Image after the other. for ( DbImage image : imageList ) { // TODO: still some fields for ImageData, which i can't fill with info from DbImage. ImageData imDat = new ImageData( image.guid, image.rid, image.name, System.currentTimeMillis(), System.currentTimeMillis(), image.creator, "anyThing", true, false, "best", "theVeryBest", image.fileSize ); - - FtpCredentials ftpc = ThriftConnection.getFtpCredentials( imDat ); - if ( ftpc == null ) { - log.error( "The FtpCredentials are null" ); - } - + char[] passphrase = Globals.getPropertyString( PropString.FTPSKEYSTOREPWD ).toCharArray(); + KeyStore keystore; try { - TrustManagerFactory trustManagerFactory = TrustManagerFactory - .getInstance( KeyManagerFactory.getDefaultAlgorithm() ); - KeyStore keystore = KeyStore.getInstance( Globals.getPropertyString( PropString.KEYSTORETYPE ) ); - keystore.load( new FileInputStream( new File( - Globals.getPropertyString( PropString.FTPSKEYSTOREPATH ) ) ), - Globals.getPropertyString( PropString.FTPSKEYSTOREPWD ).toCharArray() ); - trustManagerFactory.init( keystore ); - TrustManager trustManager = trustManagerFactory.getTrustManagers()[0]; - FTPSClient ftpClient = new FTPSClient( "SSL", true ); - ftpClient.setTrustManager( trustManager ); - try { - ftpClient.connect( Globals.getPropertyString( PropString.FTPSERVERIP ), Globals.getPropertyInt( PropInt.FTPPORT ) ); - if ( !ftpClient.login( ftpc.username, ftpc.password ) ) { - log.error( "FTP problem. Coundn't log in!" ); - } - File file = new File( image.path ); - FileInputStream fis = new FileInputStream( file ); - ftpClient.setFileType( FTP.BINARY_FILE_TYPE ); - ftpClient.storeFile( image.name, fis ); - ThriftConnection.finishedUpload( ftpc.username, imDat ); + + // All the necessary KeyStore handling for the "context"-item. + keystore = KeyStore.getInstance( "JKS" ); + keystore.load( new FileInputStream( Globals.getPropertyString( PropString.FTPSKEYSTOREPATH ) ), passphrase ); + TrustManagerFactory tmf = TrustManagerFactory.getInstance( TrustManagerFactory.getDefaultAlgorithm() ); + tmf.init( keystore ); + SSLContext context = SSLContext.getInstance( "SSLv3" ); + TrustManager[] trustManagers = tmf.getTrustManagers(); + context.init( null, trustManagers, null ); - } catch ( IOException e ) { - log.error( "FtpUploadWorker: Error creating the FileInputStream" ); - } finally { - ftpClient.disconnect(); - log.info( "FtpUploadWorker: ftpClient disconnected" ); + // uploadInfo and ThriftAuthentication + UploadInfos upInfos = ThriftConnection.getUploadInfos( imDat ); + if ( upInfos == null ) { + log.error( "The UploadInfos returned by ThriftConnection Class are null" ); } - } catch ( NoSuchAlgorithmException | KeyStoreException | CertificateException | IOException e ) { - log.debug( "FtpUploadWorker: Problem with Keystore ore FtpsClient creation." ); + + // creating the uploader with the "context"-item. + Uploader u = new Uploader( Globals.getPropertyString( PropString.FTPSERVERIP ), Globals.getPropertyInt( PropInt.FTPPORT ), context ); + u.sendToken( upInfos.token ); + + // continue sending Blocks until getMissingBlocks is empty. + while ( !upInfos.getMissingBlocks().isEmpty() ) { + // Send all Blocks from upInfos.getMissingBlocks() in ranges. + List blocks = upInfos.getMissingBlocks(); + int start = 0; + for ( int i = 0; i < blocks.size() - 1; i++ ) { + if ( blocks.get( i ) != ( blocks.get( i + 1 ) - 1 ) ) { + u.sendRange( start, i ); + u.sendFile( image.path ); + start = i + 1; + } + if ( i == blocks.size() - 2 ) { + u.sendRange( start, blocks.size() - 1 ); + u.sendFile( image.path ); + } + } + } + upInfos = ThriftConnection.getUploadInfos( imDat ); + + } catch ( NoSuchAlgorithmException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( CertificateException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( FileNotFoundException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( IOException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( KeyStoreException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( KeyManagementException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); } + } try { Thread.sleep( 5 * 60 * 1000 ); diff --git a/src/main/java/org/openslx/satellitedaemon/ftp/ThriftConnection.java b/src/main/java/org/openslx/satellitedaemon/ftp/ThriftConnection.java index 251e2b3..2fbc471 100644 --- a/src/main/java/org/openslx/satellitedaemon/ftp/ThriftConnection.java +++ b/src/main/java/org/openslx/satellitedaemon/ftp/ThriftConnection.java @@ -16,15 +16,16 @@ import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; -import org.openslx.imagemaster.thrift.iface.FtpCredentials; +import org.openslx.imagemaster.thrift.iface.DownloadInfos; import org.openslx.imagemaster.thrift.iface.ImageData; -import org.openslx.imagemaster.thrift.iface.ImageDataException; import org.openslx.imagemaster.thrift.iface.ImageServer; import org.openslx.imagemaster.thrift.iface.ServerAuthenticationException; import org.openslx.imagemaster.thrift.iface.ServerSessionData; +import org.openslx.imagemaster.thrift.iface.UploadInfos; import org.openslx.satellitedaemon.Globals; import org.openslx.satellitedaemon.Globals.PropInt; import org.openslx.satellitedaemon.Globals.PropString; +import org.openslx.satellitedaemon.db.DbImage; import org.openslx.satellitedaemon.util.EncryptWithServerIdPublicKey; /** @@ -37,13 +38,14 @@ public class ThriftConnection private static ServerSessionData sSD = null; private static Logger log = Logger.getLogger( ThriftConnection.class ); + /** * The method calls getConnection() to check if the connection is ok * and to get the ServerSessionData. If connection is ok, it returns ftpCredential. * * @return returns 'null' if there is a problem. */ - public static FtpCredentials getFtpCredentials( ImageData imDat ) + public static UploadInfos getUploadInfos( ImageData imDat ) { ImageServer.Client theClient = null; try { @@ -85,53 +87,55 @@ public class ThriftConnection return null; } - /** - * The method calls getConnection() to check if the connection is ok - * and to get the ServerSessionData. If connection is ok, it returns ftpCredential. - * - * @return returns 'null' if there is a problem. - */ - public static FtpCredentials getFtpCredentials( String uUID ) - { - ImageServer.Client theClient = null; - try { - theClient = getConnection(); - if ( theClient == null ) { - log.error( "Client was null!" ); - return null; - } - - return theClient.getImage( uUID, sSD.sessionId ); - } catch ( TException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch ( UnrecoverableKeyException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch ( InvalidKeyException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch ( NoSuchAlgorithmException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch ( CertificateException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch ( FileNotFoundException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch ( KeyStoreException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch ( SignatureException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch ( IOException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); + /** + * The method calls getConnection() to check if the connection is ok + * and to get the DownloadeInfos. If connection is ok, it returns ftpCredential. + * + * @return returns 'null' if there is a problem. + */ + public static DownloadInfos getDownloadInfos( DbImage imDat ) + { + ImageServer.Client theClient = null; + try { + theClient = getConnection(); + if ( theClient == null ) { + log.error( "Client was null!" ); + return null; } - return null; + + return theClient.getImage( imDat.guid, sSD.sessionId ); + } catch ( TException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( UnrecoverableKeyException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( InvalidKeyException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( NoSuchAlgorithmException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( CertificateException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( FileNotFoundException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( KeyStoreException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( SignatureException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch ( IOException e ) { + // TODO Auto-generated catch block + e.printStackTrace(); } + return null; + } + + /** * This method checks if there is already a working connection. If not, @@ -141,8 +145,8 @@ public class ThriftConnection * @return returns the client if successful. */ private static ImageServer.Client getConnection() - throws UnrecoverableKeyException, NoSuchAlgorithmException, CertificateException, FileNotFoundException, KeyStoreException, - IOException, InvalidKeyException, SignatureException + throws UnrecoverableKeyException, NoSuchAlgorithmException, CertificateException, + FileNotFoundException, KeyStoreException, IOException, InvalidKeyException, SignatureException { ImageServer.Client theClient = null; boolean isAuthenticated = false; @@ -202,7 +206,7 @@ public class ThriftConnection { ImageServer.Client newClient = null; try { - TTransport transport; // Is it really always the same IP:Port as from FTPServer? + TTransport transport; transport = new TSocket( Globals.getPropertyString( PropString.FTPSERVERIP ), Globals.getPropertyInt( PropInt.THRIFTPORT ) ); transport.open(); TProtocol protocol = new TBinaryProtocol( transport ); @@ -215,29 +219,4 @@ public class ThriftConnection } return newClient; } - - public static void finishedUpload(String ftpUser, ImageData imageDescription) { - try { - client.finishedUpload( ftpUser, imageDescription ); - } catch ( ImageDataException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch ( TException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - public static void finishedDownload(String ftpUser) { - try { - client.finishedDownload( ftpUser ); - } catch ( ImageDataException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch ( TException e ) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } -- cgit v1.2.3-55-g7522