path: root/src/main/java/org/openslx/taskmanager/tasks/DownloadFiles.java

package org.openslx.taskmanager.tasks;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.openslx.satserver.util.Exec;
import org.openslx.satserver.util.Util;
import org.openslx.taskmanager.api.AbstractTask;
import org.openslx.util.PrioThreadFactory;

import com.google.gson.annotations.Expose;

public class DownloadFiles extends AbstractTask
{

	@Expose
	private Task[] files = null;
	
	@Expose
	private String baseDir;
	
	@Expose
	private String gpgPubKey;

	private Output statusObject = new Output();

	private static final String[] ALLOWED_DIRS =
	{ "/srv/openslx/www/boot/", "/tmp/" };

	@Override
	protected boolean initTask()
	{
		this.setStatusObject( statusObject );
		if ( this.files == null || this.files.length == 0 ) {
			statusObject.error = "No files given.";
			return false;
		}
		if ( Util.isEmpty( this.baseDir ) ) {
			statusObject.error = "No baseDir given.";
			return false;
		}
		if ( this.baseDir.endsWith( "/" ) ) {
			this.baseDir = this.baseDir.replaceAll( "/+$", "" );
		}
		for ( Task t : files ) {
			String path = FilenameUtils.normalize( this.baseDir + "/" + t.fileName );
			if ( path == null || !Util.startsWith( path, ALLOWED_DIRS )
					|| path.endsWith( "/" ) || !Util.startsWith( path, this.baseDir ) ) {
				statusObject.error = "File '" + t.fileName + "' not in allowed directory";
				return false;
			}
		}
		return true;
	}

	@Override
	protected boolean execute()
	{
		// Keyring
		final File gpgDir;
		if ( this.gpgPubKey != null ) {
			Path pubkeyPath;
			try {
				pubkeyPath = Files.createTempDirectory( "bwlp-",
						PosixFilePermissions.asFileAttribute( PosixFilePermissions.fromString( "rwx------" ) ) );
			} catch ( Exception e ) {
				statusObject.error = "Cannot create temp dir for gpg pubkey";
				return false;
			}
			gpgDir = pubkeyPath.toFile();
			File pubkey = new File( gpgDir, "import-file.asc" );
			try {
				FileUtils.write( pubkey, this.gpgPubKey, StandardCharsets.UTF_8 );
			} catch ( IOException e ) {
				statusObject.error = "Cannot write gpg pubkey to tempfile";
				return false;
			}
			int ret = Exec.sync( 3, "gpg", "--batch", "--homedir", gpgDir.getAbsolutePath(), "--import", pubkey.getAbsolutePath() );
			if ( ret != 0 ) {
				statusObject.error = "gpg key import failed with exit code " + ret;
				return false;
			}
		} else {
			gpgDir = null;
		}
		// Create temp dir
		Path td = null;
		Exception ex = null;
		for(int x = 0; x < 10; ++x) {
			String rnd = Double.toString( Math.random() );
			try {
				td = Files.createDirectories( Paths.get( this.baseDir + "-" + rnd + ".tmp" ) );
				break;
			} catch (Exception e) {
				ex = e;
			}
		}
		if ( td == null ) {
			statusObject.error = "Cannot create temporary directory: " + ex;
			return false;
		}
		final Path tmpDir = td;

		ExecutorService tp = Executors.newFixedThreadPool( files.length > 3 ? 3 : files.length,
				new PrioThreadFactory( "DL" ) );
		final AtomicBoolean retval = new AtomicBoolean( true );
		for ( final Task t : this.files ) {
			final FileStatus status = new FileStatus();
			status.id = t.id;
			statusObject.files.add( status );
			tp.submit( new Runnable() {
				public void run()
				{
					URLConnection connection = null;
					InputStream in = null;
					FileOutputStream fout = null;
					try {
						File tmpFile = new File( tmpDir.toFile(), t.fileName );
						connection = new URL( t.url ).openConnection();
						in = connection.getInputStream();
						fout = new FileOutputStream( tmpFile );
						status.size = connection.getContentLengthLong();
						if ( status.size <= 0 ) { // If size is unknown, fake progress...
							status.progress = 10;
						}
						final byte data[] = new byte[ 90000 ];
						int count;
						while ( ( count = in.read( data, 0, data.length ) ) != -1 ) {
							fout.write( data, 0, count );
							status.complete += count;
							if ( status.size > 0 )
								status.progress = (int) ( 100l * status.complete / status.size );
							else if ( status.progress < 95 && System.currentTimeMillis() % 50 == 0 )
								status.progress++;
						}
						// If we have a gpg sig, validate
						if ( !Util.isEmpty( t.gpg ) ) {
							if ( !checkSig( t, status, tmpFile, gpgDir ) ) {
								retval.set( false );
								return;
							}
							status.progress = 100;
						}
					} catch ( Exception e ) {
						status.error = "Download error: " + e.toString();
						retval.set( false );
					} finally {
						Util.multiClose( fout );
						flushStream( in );
						if ( connection instanceof HttpURLConnection ) {
							InputStream es = ( (HttpURLConnection)connection ).getErrorStream();
							flushStream( es );
							Util.multiClose( es );
						}
					}
				}
			} );
		}
		tp.shutdown();
		try {
			tp.awaitTermination( 5, TimeUnit.MINUTES );
		} catch ( InterruptedException e ) {
			Thread.currentThread().interrupt();
		}
		FileUtils.deleteQuietly( gpgDir );
		if ( retval.get() ) {
			// Move dir to proper location
			File dest = new File( this.baseDir );
			try {
				FileUtils.forceDelete( dest );
			} catch ( FileNotFoundException e ) {
				// Ignore
			} catch ( Exception e ) {
				statusObject.error = "Cannot delete target dir " + this.baseDir + ": " + e.toString();
				retval.set( false );
			}
			if ( retval.get() && !tmpDir.toFile().renameTo( dest ) ) {
				statusObject.error = "Cannot rename temporary download directory " + tmpDir.toString() + " to " + dest.toString();
				retval.set( false );
			}
		}
		if ( !retval.get() ) {
			FileUtils.deleteQuietly( tmpDir.toFile() );
		}
		return retval.get();
	}

	private boolean checkSig( Task t, FileStatus status, File fileToCheck, File gpgDir )
	{
		File gpgTempFile = null;
		try {
			try {
				gpgTempFile = File.createTempFile( "bwlp-", ".gpg", null );
				Util.writeStringToFile( gpgTempFile, t.gpg );
			} catch ( Exception e ) {
				status.error = "Could not create temporary file for gpg signature";
				return false;
			}
			ArrayList<String> args = new ArrayList<>();
			args.add( "gpg" );
			if ( gpgDir != null ) {
				args.add( "--homedir" );
				args.add( gpgDir.getAbsolutePath() );
			}
			args.add( "--batch" );
			args.add( "--verify" );
			args.add( gpgTempFile.getAbsolutePath() );
			args.add( fileToCheck.getAbsolutePath() );
			if ( 0 != Exec.sync( 10, args.toArray( new String[args.size()] ) ) ) {
				status.error = "GPG signature of downloaded file not valid!\n\n" + t.gpg;
				return false;
			}
			return true;
		} finally {
			if ( gpgTempFile != null && gpgTempFile.exists() )
				gpgTempFile.delete();
		}
	}
	
	private void flushStream( InputStream is )
	{
		if ( is == null )
			return;
		byte buffer[] = new byte[ 10000 ];
		try {
			while ( is.read( buffer ) != -1 );
		} catch ( IOException e ) {
		}
	}

	/**
	 * Output - contains additional status data of this task
	 */
	@SuppressWarnings( "unused" )
	private static class Output
	{
		protected String error = null;
		protected List<FileStatus> files = new CopyOnWriteArrayList<>();
	}
	
	@SuppressWarnings( "unused" )
	private static class FileStatus
	{
		protected String error = null;
		protected long size = -1;
		protected long complete = 0;
		protected int progress = 0;
		protected String id;
	}

	private static class Task
	{
		@Expose
		public String id;
		@Expose
		public String url;
		@Expose
		public String fileName;
		@Expose
		public String gpg;
	}

}