FlarePublic/CrossJeeves
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added ability to copy artifacts, CrossJeeves is now working

Browse Source
This commit is contained in:
Flare Microsystems
2024-11-14 19:44:01 -08:00
parent db4d12c47a
commit f67892eac7
9 changed files with 331 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
src/com/flaremicro/crossjeeves/ScriptProcessor.java
View File

@@ -1,10 +1,14 @@
package com.flaremicro.crossjeeves;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
@@ -25,7 +29,10 @@ import com.flaremicro.crossjeeves.net.NetworkHandler;
import com.flaremicro.crossjeeves.net.StreamForwarder;
import com.flaremicro.crossjeeves.net.packet.Packet;
import com.flaremicro.crossjeeves.net.packet.Packet3Clone;
import com.flaremicro.crossjeeves.net.packet.Packet4FileData;
import com.flaremicro.crossjeeves.net.packet.Packet5Artifact;
import com.flaremicro.crossjeeves.net.packet.Packet7LogEntry;
import com.flaremicro.util.FileUtils;
import com.flaremicro.util.Util;
import com.flaremicro.util.ZipUtils;
@@ -43,7 +50,7 @@ public class ScriptProcessor {
public ScriptProcessor(NetworkHandler netHandler, File workspace, Properties properties) {
this.netHandler = netHandler;
this.properties = properties;
this.workspace = workspace;
this.workspace = workspace;
}
public String requireAttribute(Element e, String attribute) throws ScriptProcessingException {
@@ -70,7 +77,7 @@ public class ScriptProcessor {
public void processScript(String script) throws ScriptProcessingException {
workingDirectory = workspace;
if(terminated)
if (terminated)
throw new ScriptProcessingException("Processor has been terminated");
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
try
@@ -89,14 +96,14 @@ public class ScriptProcessor {
}
catch (Exception ex)
{
throw new ScriptProcessingException("Script threw an exception during processing" ,ex);
throw new ScriptProcessingException("Script threw an exception during processing", ex);
}
}
private void processScriptNodes(NodeList nodeList) throws ScriptProcessingException {
for (int i = 0; i < nodeList.getLength(); i++)
{
if(terminated)
if (terminated)
throw new ScriptProcessingException("Processor has been terminated");
Node node = nodeList.item(i);
Element e = getElement(node);
@@ -189,7 +196,7 @@ public class ScriptProcessor {
File file = netHandler.waitForFile(id);
if (file == null)
throw new ScriptProcessingException("File failed to transfer");
if(!ZipUtils.unzipDirectory(file, workspace))
if (!ZipUtils.unzipDirectory(file, workspace))
throw new ScriptProcessingException("File failed to decompress");
}
@@ -229,7 +236,6 @@ public class ScriptProcessor {
processBuilder.directory(workspace);
Map<String, String> environment = processBuilder.environment();
for (Entry<String, String> set : currentEnvironment.entrySet())
{
environment.put(set.getKey(), set.getValue());
@@ -296,9 +302,94 @@ public class ScriptProcessor {
throw new ScriptProcessingException("Process returned exit code " + returnCode);
}
private void artifactProcessor(Element e) {
// TODO Auto-generated method stub
private void artifactProcessor(Element e) throws ScriptProcessingException {
NodeList nodeList = e.getChildNodes();
for (int i = 0; i < nodeList.getLength(); i++)
{
Element var = getElement(nodeList.item(i));
if (var == null)
continue;
String name = var.getTagName().trim();
if (name.equalsIgnoreCase("workspace"))
{
System.out.println("Collecting workspace artifacts...");
List<String> filePaths = new ArrayList<String>();
NodeList workspaceNodes = var.getChildNodes();
for (int j = 0; j < workspaceNodes.getLength(); j++)
{
workspaceNodes.item(j);
Element fileElement = getElement(workspaceNodes.item(j));
if (fileElement == null)
continue;
String fileCommand = fileElement.getTagName().trim();
if (fileCommand.equalsIgnoreCase("file"))
{
filePaths.add(fileElement.getTextContent());
}
else if (fileCommand.equalsIgnoreCase("files"))
{
String resolver = this.getAttribute(fileElement, "resolver", "wildcard").trim();
try
{
if (resolver.equalsIgnoreCase("wildcard"))
{
FileUtils.resolvePathsWildcard(workspace, fileElement.getTextContent(), filePaths);
}
else if (resolver.equalsIgnoreCase("regex"))
{
FileUtils.resolvePathsRegex(workspace, fileElement.getTextContent(), filePaths);
}
else
{
throw new ScriptProcessingException("Invalid file resolver: " + resolver);
}
}
catch (Exception ex)
{
throw new ScriptProcessingException("Failed to process files directive " + fileElement.getTextContent() + " with resolver " + resolver, ex);
}
}
}
if (filePaths.size() <= 0)
{
System.out.println("No workspace artifacts!");
continue;
}
BufferedInputStream fileStream = null;
try
{
System.out.println("Sending artifacts...");
long fileId = random.nextLong();
File zipFile = File.createTempFile("workspace-" + fileId, ".zip");
if (!ZipUtils.zipList(workspace, filePaths, zipFile))
throw new ScriptProcessingException("Failed to compress workspace artifacts");
Packet packet = new Packet5Artifact(fileId, zipFile.length(), ".");
netHandler.enqueue(packet);
fileStream = new BufferedInputStream(new FileInputStream(zipFile));
int read;
byte[] buffer = new byte[4096];
while ((read = fileStream.read(buffer)) > -1)
{
if (read == 0)
continue;
Packet4FileData dataPacket = new Packet4FileData(fileId, (short) read, buffer);
netHandler.enqueue(dataPacket);
}
packet = new Packet4FileData(fileId, (short) 0, new byte[] {});
netHandler.enqueue(packet);
netHandler.waitForPacketSend(packet);
System.out.println("Sent!");
}
catch (IOException ex)
{
throw new ScriptProcessingException("Failed to upload workspace artifacts", ex);
}
finally
{
Util.cleanClose(fileStream);
}
}
}
}
public void processAsync(final String script) {
@@ -328,9 +419,9 @@ public class ScriptProcessor {
public void terminate() {
terminated = true;
if(runningProcess != null)
if (runningProcess != null)
runningProcess.destroy();
if(this.executionThread != null)
if (this.executionThread != null)
{
this.executionThread.interrupt();
}

69
src/com/flaremicro/crossjeeves/net/ClientHandler.java
View File

@@ -8,6 +8,8 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.Socket;
import java.util.ArrayList;
import java.util.List;
import com.flaremicro.crossjeeves.net.packet.Packet;
import com.flaremicro.crossjeeves.net.packet.Packet0Identify;
@@ -25,6 +27,8 @@ import com.flaremicro.util.ZipUtils;
public class ClientHandler extends NetworkHandler {
private String script;
private List<Thread> runningThreads = new ArrayList<Thread>();
public ClientHandler(Socket socket, String script) throws IOException {
super(socket);
this.script = script;
@@ -42,6 +46,18 @@ public class ClientHandler extends NetworkHandler {
disconnect(INVALID_PACKET_RECIEVED.id, "Recieved invalid packet " + packet.getId() + " (Unknown)");
}
@Override
public void doDisconnect(int err) {
super.doDisconnect(err);
synchronized (runningThreads)
{
for (Thread t : runningThreads)
{
t.interrupt();
}
}
}
@Override
public void handlePacket(Packet0Identify packet) {
if (packet.getProtocolVersion() != Packet.PROTOCOL_VERSION)
@@ -57,7 +73,7 @@ public class ClientHandler extends NetworkHandler {
@Override
public void handlePacket(Packet1Status packet) {
if((packet.getFlags() & Packet1Status.BUSY) != 0)
if ((packet.getFlags() & Packet1Status.BUSY) != 0)
{
disconnect(OUTDATED_AGENT.id, "Agent is too busy");
}
@@ -120,8 +136,50 @@ public class ClientHandler extends NetworkHandler {
}
@Override
public void handlePacket(Packet5Artifact packet) {
public void handlePacket(final Packet5Artifact packet) {
try
{
System.out.println("Getting artifacts!");
final String workspace = System.getenv("WORKSPACE");
if (workspace == null || workspace.trim().length() <= 0)
{
disconnect(INVALID_SYSTEM_STATE.id, "Workspace is not defined");
}
this.beginFile(packet.getFileId(), packet.getFileSize());
Thread thread = new Thread(new Runnable() {
public void run() {
File file = waitForFile(packet.getFileId());
System.out.println("Got artifacts!");
if (file == null)
{
disconnect(FILE_DOWNLOAD_FAILURE.id, "Failed to download transferred file");
}
if (!ZipUtils.unzipDirectory(file, new File(workspace)))
{
disconnect(FILE_DOWNLOAD_FAILURE.id, "Failed to extract transferred file");
}
else
{
System.out.println("Extracted artifacts!");
}
synchronized (runningThreads)
{
runningThreads.remove(this);
}
}
});
synchronized (runningThreads)
{
runningThreads.add(thread);
}
thread.start();
}
catch (IOException e)
{
disconnect(FILE_DOWNLOAD_FAILURE.id, "Failed to create file for transfer");
}
}
@Override
@@ -152,10 +210,9 @@ public class ClientHandler extends NetworkHandler {
@Override
public void handlePacket(Packet7LogEntry packet) {
if(packet.getStdOutput() == Packet7LogEntry.STD_ERR)
System.err.println(packet.getLogEntry());
else
System.out.println(packet.getLogEntry());
if (packet.getStdOutput() == Packet7LogEntry.STD_ERR)
System.err.println("[AGENT] " + packet.getLogEntry());
else System.out.println("[AGENT] " + packet.getLogEntry());
}
}

25
src/com/flaremicro/crossjeeves/net/NetworkHandler.java
View File

@@ -46,6 +46,22 @@ public abstract class NetworkHandler {
this.out = new DataOutputStream(socket.getOutputStream());
}
public void waitForPacketSend(Packet packet) {
synchronized (packet)
{
if (!outbox.contains(packet))
return;
try
{
packet.wait();
}
catch (InterruptedException e)
{
e.printStackTrace();
}
}
}
public File waitForFile(long fileId) {
try
{
@@ -103,6 +119,11 @@ public abstract class NetworkHandler {
Util.cleanClose(in);
Util.cleanClose(out);
Util.cleanClose(socket);
for (FileTransferInfo transferInfo : downloadQueue.values())
{
Util.cleanClose(transferInfo.outputStream);
transferInfo.file.delete();
}
}
public int getExitCode() {
@@ -161,6 +182,10 @@ public abstract class NetworkHandler {
try
{
packet.sendPacket(out);
synchronized (packet)
{
packet.notifyAll();
}
}
catch (IOException e)
{

2
src/com/flaremicro/crossjeeves/net/ServerHandler.java
View File

@@ -20,6 +20,7 @@ import com.flaremicro.crossjeeves.net.packet.Packet5Artifact;
import com.flaremicro.crossjeeves.net.packet.Packet4FileData;
import com.flaremicro.crossjeeves.net.packet.Packet6Disconnect;
import com.flaremicro.crossjeeves.net.packet.Packet7LogEntry;
import com.flaremicro.util.FileUtils;
public class ServerHandler extends NetworkHandler {
private CrossJeevesHost host;
@@ -52,6 +53,7 @@ public class ServerHandler extends NetworkHandler {
scriptProcessor.terminate();
host.removeConnection(this);
super.doDisconnect(exitCode);
FileUtils.deleteDirectory(workspace);
}
@Override

2
src/com/flaremicro/crossjeeves/net/StreamForwarder.java
View File

@@ -44,7 +44,7 @@ public class StreamForwarder {
}
public void start() {
if (!isRunning && parent != null)
if (!isRunning && parent == null)
{
isRunning = true;
parent = new Thread(this);

12
src/com/flaremicro/crossjeeves/net/packet/Packet5Artifact.java
View File

@@ -8,24 +8,28 @@ import com.flaremicro.crossjeeves.net.NetworkHandler;
public class Packet5Artifact extends Packet{
private long fileId;
private long fileSize;
private String relativeFile;
public Packet5Artifact(){
}
public Packet5Artifact(long fileId, String relativeFile){
public Packet5Artifact(long fileId, long fileSize, String relativeFile){
this.fileId = fileId;
this.fileSize = fileSize;
this.relativeFile = relativeFile;
}
public void recievePacket(DataInputStream in) throws IOException {
fileId = in.readLong();
fileSize = in.readLong();
relativeFile = in.readUTF();
}
public void sendPacket(DataOutputStream out) throws IOException {
super.sendPacket(out);
out.writeLong(fileId);
out.writeLong(fileSize);
out.writeUTF(relativeFile);
}
@@ -33,6 +37,12 @@ public class Packet5Artifact extends Packet{
networkHandler.handlePacket(this);
}
public long getFileSize()
{
return fileSize;
}
public long getFileId()
{
return fileId;

55
src/com/flaremicro/util/FileUtils.java Normal file
View File

@@ -0,0 +1,55 @@
package com.flaremicro.util;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.regex.Pattern;
public class FileUtils {
public static void resolvePathsWildcard(File baseDir, String wildcard, List<String> matchedPaths)
{
throw new RuntimeException("Resolving with wildcards is not yet supported");
}
public static void resolvePathsRegex(File baseDir, String regex, List<String> matchedPaths) throws IOException {
Pattern pattern = Pattern.compile(regex);
resolvePathsRecursiveRegex(baseDir, baseDir, pattern, matchedPaths);
}
private static void resolvePathsRecursiveRegex(File baseDir, File currDir, Pattern pattern, List<String> matchedPaths) throws IOException {
String baseDirPath = baseDir.getCanonicalPath();
for (File file : currDir.listFiles())
{
String filePath = file.getCanonicalPath();
//Likely a link, ignore
if (!filePath.startsWith(baseDirPath))
continue;
filePath = filePath.substring(baseDirPath.length());
if (file.isDirectory())
{
resolvePathsRecursiveRegex(baseDir, file, pattern, matchedPaths);
}
else
{
if (pattern.matcher(filePath).matches())
{
matchedPaths.add(filePath);
}
}
}
}
public static boolean deleteDirectory(File directory) {
File[] allContents = directory.listFiles();
if (allContents != null) {
for (File file : allContents) {
deleteDirectory(file);
}
}
directory.deleteOnExit();
return directory.delete();
}
}

62
src/com/flaremicro/util/ZipUtils.java
View File

@@ -6,6 +6,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Enumeration;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;
@@ -13,17 +14,68 @@ import java.util.zip.ZipOutputStream;
public class ZipUtils {
public static void zipDirectory(File sourceDir, File zipFile) throws IOException {
FileOutputStream fos = new FileOutputStream(zipFile);
ZipOutputStream zos = new ZipOutputStream(fos);
public static boolean zipList(File baseDir, List<String> filePaths, File zipFile) {
FileOutputStream fos = null;
ZipOutputStream zos = null;
FileInputStream fis = null;
try
{
zipFilesRecursively(sourceDir, sourceDir, zos);
fos = new FileOutputStream(zipFile);
zos = new ZipOutputStream(fos);
for (String filePath : filePaths)
{
File file = new File(baseDir, filePath);
if (file.isDirectory() || !file.exists())
continue;
fis = new FileInputStream(file);
String zipEntryName = filePath.replace("\\", "/");
ZipEntry zipEntry = new ZipEntry(zipEntryName);
zos.putNextEntry(zipEntry);
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = fis.read(buffer)) != -1)
{
zos.write(buffer, 0, bytesRead);
}
zos.closeEntry();
fis.close();
fis = null;
}
return true;
}
catch (IOException ex)
{
ex.printStackTrace();
}
finally
{
zos.close();
Util.cleanClose(fis);
Util.cleanClose(zos);
}
return false;
}
public static boolean zipDirectory(File sourceDir, File zipFile) {
FileOutputStream fos = null;
ZipOutputStream zos = null;
try
{
fos = new FileOutputStream(zipFile);
zos = new ZipOutputStream(fos);
zipFilesRecursively(sourceDir, sourceDir, zos);
return true;
}
catch (IOException ex)
{
ex.printStackTrace();
}
finally
{
Util.cleanClose(zos);
}
return false;
}
private static void zipFilesRecursively(File rootDir, File source, ZipOutputStream zos) throws IOException {

8
test/com/flaremicro/crossjeeves/net/packet/test/Packet5ArtifactTest.java
View File

@@ -22,13 +22,15 @@ public class Packet5ArtifactTest extends PacketTestBase {
@Test
public void testWrite() throws IOException
{
Packet5Artifact packet = new Packet5Artifact(1, "file.txt");
Packet5Artifact packet = new Packet5Artifact(1, 2, "file.txt");
assertEquals(packet.getFileId(), 1);
assertEquals(packet.getFileSize(), 2);
assertEquals("file.txt", packet.getRelativeFile());
packet.sendPacket(output());
assertEquals(5, input().readByte());
assertEquals(1, input().readLong());
assertEquals(2, input().readLong());
assertEquals("file.txt", input().readUTF());
}
@@ -38,18 +40,20 @@ public class Packet5ArtifactTest extends PacketTestBase {
Packet5Artifact packet = new Packet5Artifact();
output().writeLong(1);
output().writeLong(2);
output().writeUTF("file.txt");
packet.recievePacket(input());
assertEquals(1, packet.getFileId());
assertEquals(2, packet.getFileSize());
assertEquals("file.txt", packet.getRelativeFile());
}
@Test
public void testProcess() throws IOException
{
Packet5Artifact packet = new Packet5Artifact(1, "file.txt");
Packet5Artifact packet = new Packet5Artifact(1, 2, "file.txt");
packet.processPacket(handler);
Mockito.verify(handler, Mockito.times(1)).handlePacket(packet);
}