package org.openslx.taskmanager.tasks; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveException; import org.apache.commons.compress.archivers.ArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.openslx.satserver.util.Archive; import org.openslx.satserver.util.Util; import org.openslx.taskmanager.api.AbstractTask; import com.google.gson.annotations.Expose; public class RecompressArchive extends AbstractTask { @Expose private Map inputFiles; @Expose private String outputFile; @Expose private boolean forceRoot; /* * Own vars/constants not being deserialized */ protected static final String[] ALLOWED_DIRS = { "/tmp/", "/opt/openslx/configs/" }; private Output status = new Output(); /* * Code */ @Override protected boolean execute() { String fn = new File( this.outputFile ).getName(); String tmpFile = "/tmp/bwlp-" + System.nanoTime() + "-" + fn; if ( execute2( tmpFile ) ) { try { FileUtils.deleteQuietly( new File( this.outputFile ) ); Files.move( Paths.get( tmpFile ), Paths.get( this.outputFile ) ); return true; } catch ( Exception e ) { status.error = e.toString(); status.errorCode = Output.ErrorCode.WRITE_FAILED; } } FileUtils.deleteQuietly( new File( tmpFile ) ); return false; } private boolean execute2( String archiveFile ) { // Open output file archive TarArchiveOutputStream outArchive = null; try { try { FileUtils.forceMkdir( new File( new File( this.outputFile ).getParent() ) ); outArchive = Archive.createTarArchive( archiveFile ); } catch ( IOException e2 ) { status.error = e2.toString(); status.errorCode = Output.ErrorCode.WRITE_FAILED; return false; } // Remember all files added, so we can issue warnings for duplicate entries Map> entries = new HashMap<>(); // Open input file archives, one by one for ( final Entry it : this.inputFiles.entrySet() ) { final A a = new A( it.getKey(), it.getValue() ); ArchiveInputStream inArchive = null; try { try { inArchive = Archive.getArchiveInputStream( a.inputFile ); } catch ( FileNotFoundException e1 ) { status.error = "(" + a.inputFile + ") " + e1.getMessage(); status.errorCode = Output.ErrorCode.NOT_FOUND; return false; } catch ( IllegalArgumentException e1 ) { status.error = "(" + a.inputFile + ") " + e1.getMessage(); status.errorCode = Output.ErrorCode.UNKNOWN_ERROR; return false; } catch ( ArchiveException e1 ) { status.error = "(" + a.inputFile + ") " + e1.getMessage(); status.errorCode = Output.ErrorCode.UNKNOWN_FORMAT; return false; } // It's open ArchiveEntry inEntry; // Iterate over every entry while ( ( inEntry = inArchive.getNextEntry() ) != null ) { if ( !inArchive.canReadEntryData( inEntry ) ) continue; // Skip unreadable entries if ( inEntry.getName().equals( "/" ) || inEntry.getName().equals( "./" ) ) continue; // Construct TarArchiveEntry - we want unix stuff like uid/gid, links, file/dir mode, so try to get from source archive TarArchiveEntry outEntry = Archive.createTarArchiveEntry( inEntry, 0, 0, 0755, 0644, this.forceRoot ); if ( outEntry == null ) { status.addWarning( "Ignoring invalid entry " + inEntry.getName() + " in " + a.inputFile ); continue; } // Ask lib if the entry can be written if ( !outArchive.canWriteEntryData( outEntry ) ) { status.addWarning( "Commons-compress says entry '" + outEntry.getName() + "' cannot be written." ); continue; } if ( !outEntry.isDirectory() ) { // Dupcheck List existing = entries.get( outEntry.getName() ); if ( existing == null ) { entries.put( outEntry.getName(), existing = new ArrayList() ); existing.add( a ); } else { existing.add( a ); continue; // Just record, but don't bloat archive } } // Actually add entry now try { outArchive.putArchiveEntry( outEntry ); } catch (Exception e) { status.error = "(" + a.inputFile + ") Could not putArchiveEntry('" + outEntry.getName() + "'): " + e.getMessage(); status.errorCode = Output.ErrorCode.TAR_ERROR; return false; } if ( !Util.streamCopy( inArchive, outArchive, outEntry.getSize() ) ) { status.error = "(" + a.inputFile + ") Could not copy entry '" + inEntry.getName() + "' to destination archive."; status.errorCode = Output.ErrorCode.WRITE_FAILED; return false; } outArchive.closeArchiveEntry(); } // End entry } catch ( IOException e ) { return false; } finally { Util.multiClose( inArchive ); } } // End of loop over input archives // Check for dups boolean hasAnyDup = false; for ( Entry> fit : entries.entrySet() ) { if ( fit.getValue().size() < 2 ) continue; boolean hasDup = false; for ( A lit : fit.getValue() ) { if ( lit.dupCheck ) { hasDup = true; break; } } if ( !hasDup ) // No file requested to be checked for duplicates continue; hasAnyDup = true; status.addWarning( "Duplicate entry: " + fit.getKey() ); for ( A lit : fit.getValue() ) { status.addWarning( " Source: " + lit.inputFile ); } } if ( hasAnyDup ) { status.addWarning( "You have duplicate files in your source archives. It's undefined which one ends up in the final archive." ); } return true; } finally { Util.multiClose( outArchive ); } } @Override protected boolean initTask() { this.setStatusObject( this.status ); // Input files if ( this.inputFiles == null ) { status.error = "Input: No archives given"; status.errorCode = Output.ErrorCode.UNKNOWN_ERROR; return false; } for ( String inputFile : this.inputFiles.keySet() ) { if ( inputFile == null || inputFile.length() == 0 || inputFile.charAt( 0 ) != '/' ) { status.error = "Input: Need absolute path (got: " + inputFile + ")"; status.errorCode = Output.ErrorCode.INVALID_DIRECTORY; return false; } inputFile = FilenameUtils.normalize( inputFile ); if ( inputFile == null || !Util.startsWith( inputFile, ALLOWED_DIRS ) ) { status.error = "Input: File not in allowed directory"; status.errorCode = Output.ErrorCode.INVALID_DIRECTORY; return false; } } // Output file if ( this.outputFile == null || this.outputFile.length() == 0 || this.outputFile.charAt( 0 ) != '/' ) { status.error = "Output: Need absolute path."; status.errorCode = Output.ErrorCode.INVALID_DIRECTORY; return false; } this.outputFile = FilenameUtils.normalize( this.outputFile ); if ( !Util.startsWith( this.outputFile, ALLOWED_DIRS ) ) { status.error = "Output: File not in allowed directory"; status.errorCode = Output.ErrorCode.INVALID_DIRECTORY; return false; } return true; } @SuppressWarnings( "unused" ) private static class Output { protected String error = null; protected enum ErrorCode { NOT_FOUND, UNKNOWN_FORMAT, UNKNOWN_ERROR, INVALID_DIRECTORY, WRITE_FAILED, TAR_ERROR }; protected ErrorCode errorCode = null; protected String warnings = null; public static class Entry { protected String name = null; protected boolean isdir = false; protected long size = -1; } public void addWarning( String txt ) { if (warnings == null) { warnings = txt; } else { warnings += "\n" + txt; } } } private static class A { public final String inputFile; public final Boolean dupCheck; public A( String file, Boolean dupCheck ) { this.inputFile = file; this.dupCheck = dupCheck; } } }