From 8b2ffa1bf284bb9a14b451b8736b3aff1c88ee2d Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Wed, 9 Oct 2019 17:41:48 +0200 Subject: [DownloadFiles] Task for batch downloads Supports using custom GPG pubkey for verification of downloaded files --- .../openslx/taskmanager/tasks/DownloadFiles.java | 289 +++++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 src/main/java/org/openslx/taskmanager/tasks/DownloadFiles.java diff --git a/src/main/java/org/openslx/taskmanager/tasks/DownloadFiles.java b/src/main/java/org/openslx/taskmanager/tasks/DownloadFiles.java new file mode 100644 index 0000000..c274462 --- /dev/null +++ b/src/main/java/org/openslx/taskmanager/tasks/DownloadFiles.java @@ -0,0 +1,289 @@ +package org.openslx.taskmanager.tasks; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.openslx.satserver.util.Exec; +import org.openslx.satserver.util.ProxyHandler; +import org.openslx.satserver.util.Util; +import org.openslx.taskmanager.api.AbstractTask; + +import com.google.gson.annotations.Expose; + +public class DownloadFiles extends AbstractTask +{ + + @Expose + private Task[] files = null; + + @Expose + private String baseDir; + + @Expose + private String gpgPubKey; + + private Output statusObject = new Output(); + + private static final String[] ALLOWED_DIRS = + { "/srv/openslx/www/boot/", "/tmp/" }; + + @Override + protected boolean initTask() + { + this.setStatusObject( statusObject ); + if ( this.files == null || this.files.length == 0 ) { + statusObject.error = "No files given."; + return false; + } + if ( Util.isEmpty( this.baseDir ) ) { + statusObject.error = "No baseDir given."; + return false; + } + if ( this.baseDir.endsWith( "/" ) ) { + this.baseDir = this.baseDir.replaceAll( "/+$", "" ); + } + for ( Task t : files ) { + String path = FilenameUtils.normalize( this.baseDir + "/" + t.fileName ); + if ( path == null || !Util.startsWith( path, ALLOWED_DIRS ) + || path.endsWith( "/" ) || !Util.startsWith( path, this.baseDir ) ) { + statusObject.error = "File '" + t.fileName + "' not in allowed directory"; + return false; + } + } + return true; + } + + @Override + protected boolean execute() + { + // Keyring + final File gpgDir; + if ( this.gpgPubKey != null ) { + Path pubkeyPath; + try { + pubkeyPath = Files.createTempDirectory( "bwlp-", + PosixFilePermissions.asFileAttribute( PosixFilePermissions.fromString( "rwx------" ) ) ); + } catch ( Exception e ) { + statusObject.error = "Cannot create temp dir for gpg pubkey"; + return false; + } + gpgDir = pubkeyPath.toFile(); + File pubkey = new File( gpgDir, "import-file.asc" ); + try { + FileUtils.write( pubkey, this.gpgPubKey, StandardCharsets.UTF_8 ); + } catch ( IOException e ) { + statusObject.error = "Cannot write gpg pubkey to tempfile"; + return false; + } + int ret = Exec.sync( 3, "gpg", "--batch", "--homedir", gpgDir.getAbsolutePath(), "--import", pubkey.getAbsolutePath() ); + if ( ret != 0 ) { + statusObject.error = "gpg key import failed with exit code " + ret; + return false; + } + } else { + gpgDir = null; + } + // Create temp dir + Path td = null; + Exception ex = null; + for(int x = 0; x < 10; ++x) { + String rnd = Double.toString( Math.random() ); + try { + td = Files.createDirectories( Paths.get( this.baseDir + "-" + rnd + ".tmp" ) ); + break; + } catch (Exception e) { + ex = e; + } + } + if ( td == null ) { + statusObject.error = "Cannot create temporary directory: " + ex; + return false; + } + final Path tmpDir = td; + // Handle proxy settings before opening connection for downloading. + ProxyHandler.configProxy(); + ExecutorService tp = Executors.newFixedThreadPool( files.length > 3 ? 3 : files.length ); + final AtomicBoolean retval = new AtomicBoolean( true ); + for ( final Task t : this.files ) { + final FileStatus status = new FileStatus(); + status.id = t.id; + statusObject.files.add( status ); + tp.submit( new Runnable() { + public void run() + { + URLConnection connection = null; + InputStream in = null; + FileOutputStream fout = null; + try { + File tmpFile = new File( tmpDir.toFile(), t.fileName ); + connection = new URL( t.url ).openConnection(); + in = connection.getInputStream(); + fout = new FileOutputStream( tmpFile ); + status.size = connection.getContentLengthLong(); + if ( status.size <= 0 ) { // If size is unknown, fake progress... + status.progress = 10; + } + final byte data[] = new byte[ 90000 ]; + int count; + while ( ( count = in.read( data, 0, data.length ) ) != -1 ) { + fout.write( data, 0, count ); + status.complete += count; + if ( status.size > 0 ) + status.progress = (int) ( 100l * status.complete / status.size ); + else if ( status.progress < 95 && System.currentTimeMillis() % 50 == 0 ) + status.progress++; + } + // If we have a gpg sig, validate + if ( !Util.isEmpty( t.gpg ) ) { + if ( !checkSig( t, status, tmpFile, gpgDir ) ) { + retval.set( false ); + return; + } + status.progress = 100; + } + } catch ( Exception e ) { + status.error = "Download error: " + e.toString(); + retval.set( false ); + } finally { + Util.multiClose( fout ); + flushStream( in ); + if ( connection instanceof HttpURLConnection ) { + InputStream es = ( (HttpURLConnection)connection ).getErrorStream(); + flushStream( es ); + Util.multiClose( es ); + } + } + } + } ); + } + tp.shutdown(); + try { + tp.awaitTermination( 5, TimeUnit.MINUTES ); + } catch ( InterruptedException e ) { + Thread.currentThread().interrupt(); + } + try { + FileUtils.forceDelete( gpgDir ); + } catch ( Exception e1 ) { + } + if ( retval.get() ) { + // Move dir to proper location + File dest = new File( this.baseDir ); + try { + FileUtils.forceDelete( dest ); + } catch ( FileNotFoundException e ) { + // Ignore + } catch ( Exception e ) { + statusObject.error = "Cannot delete target dir " + this.baseDir + ": " + e.toString(); + retval.set( false ); + } + if ( retval.get() && !tmpDir.toFile().renameTo( dest ) ) { + statusObject.error = "Cannot rename temporary download directory " + tmpDir.toString() + " to " + dest.toString(); + retval.set( false ); + } + } + if ( !retval.get() ) { + try { + FileUtils.forceDelete( tmpDir.toFile() ); + } catch ( Exception e ) { + } + } + return retval.get(); + } + + private boolean checkSig( Task t, FileStatus status, File fileToCheck, File gpgDir ) + { + File gpgTempFile = null; + try { + try { + gpgTempFile = File.createTempFile( "bwlp-", ".gpg", null ); + Util.writeStringToFile( gpgTempFile, t.gpg ); + } catch ( Exception e ) { + status.error = "Could not create temporary file for gpg signature"; + return false; + } + ArrayList args = new ArrayList<>(); + args.add( "gpg" ); + if ( gpgDir != null ) { + args.add( "--homedir" ); + args.add( gpgDir.getAbsolutePath() ); + } + args.add( "--batch" ); + args.add( "--verify" ); + args.add( gpgTempFile.getAbsolutePath() ); + args.add( fileToCheck.getAbsolutePath() ); + if ( 0 != Exec.sync( 10, args.toArray( new String[args.size()] ) ) ) { + status.error = "GPG signature of downloaded file not valid!\n\n" + t.gpg; + return false; + } + return true; + } finally { + if ( gpgTempFile != null && gpgTempFile.exists() ) + gpgTempFile.delete(); + } + } + + private void flushStream( InputStream is ) + { + if ( is == null ) + return; + byte buffer[] = new byte[ 10000 ]; + try { + while ( is.read( buffer ) != -1 ); + } catch ( IOException e ) { + } + } + + /** + * Output - contains additional status data of this task + */ + @SuppressWarnings( "unused" ) + private static class Output + { + protected String error = null; + protected List files = new CopyOnWriteArrayList<>(); + } + + @SuppressWarnings( "unused" ) + private static class FileStatus + { + protected String error = null; + protected long size = -1; + protected long complete = 0; + protected int progress = 0; + protected String id; + } + + private static class Task + { + @Expose + public String id; + @Expose + public String url; + @Expose + public String fileName; + @Expose + public String gpg; + } + +} -- cgit v1.2.3-55-g7522