package org.openslx.taskmanager.tasks;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.openslx.satserver.util.Exec;
import org.openslx.satserver.util.Util;
import org.openslx.taskmanager.api.AbstractTask;
import org.openslx.util.PrioThreadFactory;
import com.google.gson.annotations.Expose;
public class DownloadFiles extends AbstractTask
{
@Expose
private Task[] files = null;
@Expose
private String baseDir;
@Expose
private String gpgPubKey;
private Output statusObject = new Output();
private static final String[] ALLOWED_DIRS =
{ "/srv/openslx/www/boot/", "/tmp/" };
@Override
protected boolean initTask()
{
this.setStatusObject( statusObject );
if ( this.files == null || this.files.length == 0 ) {
statusObject.error = "No files given.";
return false;
}
if ( Util.isEmpty( this.baseDir ) ) {
statusObject.error = "No baseDir given.";
return false;
}
if ( this.baseDir.endsWith( "/" ) ) {
this.baseDir = this.baseDir.replaceAll( "/+$", "" );
}
for ( Task t : files ) {
String path = FilenameUtils.normalize( this.baseDir + "/" + t.fileName );
if ( path == null || !Util.startsWith( path, ALLOWED_DIRS )
|| path.endsWith( "/" ) || !Util.startsWith( path, this.baseDir ) ) {
statusObject.error = "File '" + t.fileName + "' not in allowed directory";
return false;
}
}
return true;
}
@Override
protected boolean execute()
{
// Keyring
final File gpgDir;
if ( this.gpgPubKey != null ) {
Path pubkeyPath;
try {
pubkeyPath = Files.createTempDirectory( "bwlp-",
PosixFilePermissions.asFileAttribute( PosixFilePermissions.fromString( "rwx------" ) ) );
} catch ( Exception e ) {
statusObject.error = "Cannot create temp dir for gpg pubkey";
return false;
}
gpgDir = pubkeyPath.toFile();
File pubkey = new File( gpgDir, "import-file.asc" );
try {
FileUtils.write( pubkey, this.gpgPubKey, StandardCharsets.UTF_8 );
} catch ( IOException e ) {
statusObject.error = "Cannot write gpg pubkey to tempfile";
return false;
}
int ret = Exec.sync( 3, "gpg", "--batch", "--homedir", gpgDir.getAbsolutePath(), "--import", pubkey.getAbsolutePath() );
if ( ret != 0 ) {
statusObject.error = "gpg key import failed with exit code " + ret;
return false;
}
} else {
gpgDir = null;
}
// Create temp dir
Path td = null;
Exception ex = null;
for(int x = 0; x < 10; ++x) {
String rnd = Double.toString( Math.random() );
try {
td = Files.createDirectories( Paths.get( this.baseDir + "-" + rnd + ".tmp" ) );
break;
} catch (Exception e) {
ex = e;
}
}
if ( td == null ) {
statusObject.error = "Cannot create temporary directory: " + ex;
return false;
}
final Path tmpDir = td;
ExecutorService tp = Executors.newFixedThreadPool( files.length > 3 ? 3 : files.length,
new PrioThreadFactory( "DL" ) );
final AtomicBoolean retval = new AtomicBoolean( true );
for ( final Task t : this.files ) {
final FileStatus status = new FileStatus();
status.id = t.id;
statusObject.files.add( status );
tp.submit( new Runnable() {
public void run()
{
URLConnection connection = null;
InputStream in = null;
FileOutputStream fout = null;
try {
File tmpFile = new File( tmpDir.toFile(), t.fileName );
connection = new URL( t.url ).openConnection();
in = connection.getInputStream();
fout = new FileOutputStream( tmpFile );
status.size = connection.getContentLengthLong();
if ( status.size <= 0 ) { // If size is unknown, fake progress...
status.progress = 10;
}
final byte data[] = new byte[ 90000 ];
int count;
while ( ( count = in.read( data, 0, data.length ) ) != -1 ) {
fout.write( data, 0, count );
status.complete += count;
if ( status.size > 0 )
status.progress = (int) ( 100l * status.complete / status.size );
else if ( status.progress < 95 && System.currentTimeMillis() % 50 == 0 )
status.progress++;
}
// If we have a gpg sig, validate
if ( !Util.isEmpty( t.gpg ) ) {
if ( !checkSig( t, status, tmpFile, gpgDir ) ) {
retval.set( false );
return;
}
status.progress = 100;
}
} catch ( Exception e ) {
status.error = "Download error: " + e.toString();
retval.set( false );
} finally {
Util.multiClose( fout );
flushStream( in );
if ( connection instanceof HttpURLConnection ) {
InputStream es = ( (HttpURLConnection)connection ).getErrorStream();
flushStream( es );
Util.multiClose( es );
}
}
}
} );
}
tp.shutdown();
try {
tp.awaitTermination( 5, TimeUnit.MINUTES );
} catch ( InterruptedException e ) {
Thread.currentThread().interrupt();
}
FileUtils.deleteQuietly( gpgDir );
if ( retval.get() ) {
// Move dir to proper location
File dest = new File( this.baseDir );
try {
FileUtils.forceDelete( dest );
} catch ( FileNotFoundException e ) {
// Ignore
} catch ( Exception e ) {
statusObject.error = "Cannot delete target dir " + this.baseDir + ": " + e.toString();
retval.set( false );
}
if ( retval.get() && !tmpDir.toFile().renameTo( dest ) ) {
statusObject.error = "Cannot rename temporary download directory " + tmpDir.toString() + " to " + dest.toString();
retval.set( false );
}
}
if ( !retval.get() ) {
FileUtils.deleteQuietly( tmpDir.toFile() );
}
return retval.get();
}
private boolean checkSig( Task t, FileStatus status, File fileToCheck, File gpgDir )
{
File gpgTempFile = null;
try {
try {
gpgTempFile = File.createTempFile( "bwlp-", ".gpg", null );
Util.writeStringToFile( gpgTempFile, t.gpg );
} catch ( Exception e ) {
status.error = "Could not create temporary file for gpg signature";
return false;
}
ArrayList<String> args = new ArrayList<>();
args.add( "gpg" );
if ( gpgDir != null ) {
args.add( "--homedir" );
args.add( gpgDir.getAbsolutePath() );
}
args.add( "--batch" );
args.add( "--verify" );
args.add( gpgTempFile.getAbsolutePath() );
args.add( fileToCheck.getAbsolutePath() );
if ( 0 != Exec.sync( 10, args.toArray( new String[args.size()] ) ) ) {
status.error = "GPG signature of downloaded file not valid!\n\n" + t.gpg;
return false;
}
return true;
} finally {
if ( gpgTempFile != null && gpgTempFile.exists() )
gpgTempFile.delete();
}
}
private void flushStream( InputStream is )
{
if ( is == null )
return;
byte buffer[] = new byte[ 10000 ];
try {
while ( is.read( buffer ) != -1 );
} catch ( IOException e ) {
}
}
/**
* Output - contains additional status data of this task
*/
@SuppressWarnings( "unused" )
private static class Output
{
protected String error = null;
protected List<FileStatus> files = new CopyOnWriteArrayList<>();
}
@SuppressWarnings( "unused" )
private static class FileStatus
{
protected String error = null;
protected long size = -1;
protected long complete = 0;
protected int progress = 0;
protected String id;
}
private static class Task
{
@Expose
public String id;
@Expose
public String url;
@Expose
public String fileName;
@Expose
public String gpg;
}
}
