package org.openslx.taskmanager.tasks; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.openslx.satserver.util.Exec; import org.openslx.satserver.util.Util; import org.openslx.taskmanager.api.AbstractTask; import org.openslx.util.PrioThreadFactory; import com.google.gson.annotations.Expose; public class DownloadFiles extends AbstractTask { @Expose private Task[] files = null; @Expose private String baseDir; @Expose private String gpgPubKey; private Output statusObject = new Output(); private static final String[] ALLOWED_DIRS = { "/srv/openslx/www/boot/", "/tmp/" }; @Override protected boolean initTask() { this.setStatusObject( statusObject ); if ( this.files == null || this.files.length == 0 ) { statusObject.error = "No files given."; return false; } if ( Util.isEmpty( this.baseDir ) ) { statusObject.error = "No baseDir given."; return false; } if ( this.baseDir.endsWith( "/" ) ) { this.baseDir = this.baseDir.replaceAll( "/+$", "" ); } for ( Task t : files ) { String path = FilenameUtils.normalize( this.baseDir + "/" + t.fileName ); if ( path == null || !Util.startsWith( path, ALLOWED_DIRS ) || path.endsWith( "/" ) || !Util.startsWith( path, this.baseDir ) ) { statusObject.error = "File '" + t.fileName + "' not in allowed directory"; return false; } } return true; } @Override protected boolean execute() { // Keyring final File gpgDir; if ( this.gpgPubKey != null ) { Path pubkeyPath; try { pubkeyPath = Files.createTempDirectory( "bwlp-", PosixFilePermissions.asFileAttribute( PosixFilePermissions.fromString( "rwx------" ) ) ); } catch ( Exception e ) { statusObject.error = "Cannot create temp dir for gpg pubkey"; return false; } gpgDir = pubkeyPath.toFile(); File pubkey = new File( gpgDir, "import-file.asc" ); try { FileUtils.write( pubkey, this.gpgPubKey, StandardCharsets.UTF_8 ); } catch ( IOException e ) { statusObject.error = "Cannot write gpg pubkey to tempfile"; return false; } int ret = Exec.sync( 3, "gpg", "--batch", "--homedir", gpgDir.getAbsolutePath(), "--import", pubkey.getAbsolutePath() ); if ( ret != 0 ) { statusObject.error = "gpg key import failed with exit code " + ret; return false; } } else { gpgDir = null; } // Create temp dir Path td = null; Exception ex = null; for(int x = 0; x < 10; ++x) { String rnd = Double.toString( Math.random() ); try { td = Files.createDirectories( Paths.get( this.baseDir + "-" + rnd + ".tmp" ) ); break; } catch (Exception e) { ex = e; } } if ( td == null ) { statusObject.error = "Cannot create temporary directory: " + ex; return false; } final Path tmpDir = td; ExecutorService tp = Executors.newFixedThreadPool( files.length > 3 ? 3 : files.length, new PrioThreadFactory( "DL" ) ); final AtomicBoolean retval = new AtomicBoolean( true ); for ( final Task t : this.files ) { final FileStatus status = new FileStatus(); status.id = t.id; statusObject.files.add( status ); tp.submit( new Runnable() { public void run() { URLConnection connection = null; InputStream in = null; FileOutputStream fout = null; try { File tmpFile = new File( tmpDir.toFile(), t.fileName ); connection = new URL( t.url ).openConnection(); in = connection.getInputStream(); fout = new FileOutputStream( tmpFile ); status.size = connection.getContentLengthLong(); if ( status.size <= 0 ) { // If size is unknown, fake progress... status.progress = 10; } final byte data[] = new byte[ 90000 ]; int count; while ( ( count = in.read( data, 0, data.length ) ) != -1 ) { fout.write( data, 0, count ); status.complete += count; if ( status.size > 0 ) status.progress = (int) ( 100l * status.complete / status.size ); else if ( status.progress < 95 && System.currentTimeMillis() % 50 == 0 ) status.progress++; } // If we have a gpg sig, validate if ( !Util.isEmpty( t.gpg ) ) { if ( !checkSig( t, status, tmpFile, gpgDir ) ) { retval.set( false ); return; } status.progress = 100; } } catch ( Exception e ) { status.error = "Download error: " + e.toString(); retval.set( false ); } finally { Util.multiClose( fout ); flushStream( in ); if ( connection instanceof HttpURLConnection ) { InputStream es = ( (HttpURLConnection)connection ).getErrorStream(); flushStream( es ); Util.multiClose( es ); } } } } ); } tp.shutdown(); try { tp.awaitTermination( 5, TimeUnit.MINUTES ); } catch ( InterruptedException e ) { Thread.currentThread().interrupt(); } FileUtils.deleteQuietly( gpgDir ); if ( retval.get() ) { // Move dir to proper location File dest = new File( this.baseDir ); try { FileUtils.forceDelete( dest ); } catch ( FileNotFoundException e ) { // Ignore } catch ( Exception e ) { statusObject.error = "Cannot delete target dir " + this.baseDir + ": " + e.toString(); retval.set( false ); } if ( retval.get() && !tmpDir.toFile().renameTo( dest ) ) { statusObject.error = "Cannot rename temporary download directory " + tmpDir.toString() + " to " + dest.toString(); retval.set( false ); } } if ( !retval.get() ) { FileUtils.deleteQuietly( tmpDir.toFile() ); } return retval.get(); } private boolean checkSig( Task t, FileStatus status, File fileToCheck, File gpgDir ) { File gpgTempFile = null; try { try { gpgTempFile = File.createTempFile( "bwlp-", ".gpg", null ); Util.writeStringToFile( gpgTempFile, t.gpg ); } catch ( Exception e ) { status.error = "Could not create temporary file for gpg signature"; return false; } ArrayList args = new ArrayList<>(); args.add( "gpg" ); if ( gpgDir != null ) { args.add( "--homedir" ); args.add( gpgDir.getAbsolutePath() ); } args.add( "--batch" ); args.add( "--verify" ); args.add( gpgTempFile.getAbsolutePath() ); args.add( fileToCheck.getAbsolutePath() ); if ( 0 != Exec.sync( 10, args.toArray( new String[args.size()] ) ) ) { status.error = "GPG signature of downloaded file not valid!\n\n" + t.gpg; return false; } return true; } finally { if ( gpgTempFile != null && gpgTempFile.exists() ) gpgTempFile.delete(); } } private void flushStream( InputStream is ) { if ( is == null ) return; byte buffer[] = new byte[ 10000 ]; try { while ( is.read( buffer ) != -1 ); } catch ( IOException e ) { } } /** * Output - contains additional status data of this task */ @SuppressWarnings( "unused" ) private static class Output { protected String error = null; protected List files = new CopyOnWriteArrayList<>(); } @SuppressWarnings( "unused" ) private static class FileStatus { protected String error = null; protected long size = -1; protected long complete = 0; protected int progress = 0; protected String id; } private static class Task { @Expose public String id; @Expose public String url; @Expose public String fileName; @Expose public String gpg; } }