From 8fd25205e3bedaec91b6c90f1603cf1f11a8894d Mon Sep 17 00:00:00 2001 From: Svjatoslav Agejenko Date: Thu, 30 May 2024 00:44:24 +0300 Subject: [PATCH] Refactor code. Specify llama.cpp project directory instead of executable path. --- doc/index.org | 10 +++---- .../eu/svjatoslav/alyverkko_cli/AiTask.java | 29 ++++++++++--------- .../commands/MailCorrespondentCommand.java | 16 +++++----- .../commands/SelftestCommand.java | 4 +-- .../configuration/Configuration.java | 13 ++++----- .../configuration/ConfigurationModel.java | 1 + .../alyverkko_cli/model/ModelLibrary.java | 4 +-- 7 files changed, 38 insertions(+), 39 deletions(-) diff --git a/doc/index.org b/doc/index.org index f5d7633..32b1743 100644 --- a/doc/index.org +++ b/doc/index.org @@ -239,7 +239,7 @@ file. Below is an example of how the configuration file might look: mail_directory: "/home/user/AI/mail" models_directory: "/home/user/AI/models" default_temperature: 0.7 - llama_cpp_executable_path: "/home/user/AI/llama.cpp/main" + llama_cpp_dir_path: "/home/user/AI/llama.cpp/" batch_thread_count: 10 thread_count: 6 models: @@ -289,9 +289,9 @@ Here are available parameters: creative or random. - Default value: 0.7 -- llama_cpp_executable_path :: Specifies the file path to the - *llama.cpp* *main* executable. - - Example Value: /home/user/AI/llama.cpp/main +- llama_cpp_dir_path :: Specifies the filesystem path to the cloned + and compiled *llama.cpp* directory. + - Example Value: /home/user/AI/llama.cpp/ - This option is mandatory. - batch_thread_count :: Specifies the number of threads to use for @@ -616,8 +616,6 @@ Ideas to be possibly implemented in the future: in-progress computation. Unix process stop and continue signals could possibly be used. -- Add support for speculative decoding to speed up inference. - - Explain how to monitor system performance and resource usage during AI processing tasks. diff --git a/src/main/java/eu/svjatoslav/alyverkko_cli/AiTask.java b/src/main/java/eu/svjatoslav/alyverkko_cli/AiTask.java index 64baed0..832c1b8 100644 --- a/src/main/java/eu/svjatoslav/alyverkko_cli/AiTask.java +++ b/src/main/java/eu/svjatoslav/alyverkko_cli/AiTask.java @@ -1,7 +1,6 @@ package eu.svjatoslav.alyverkko_cli; import eu.svjatoslav.alyverkko_cli.commands.MailQuery; -import eu.svjatoslav.alyverkko_cli.model.Model; import java.io.*; import java.nio.file.Files; @@ -13,27 +12,23 @@ public class AiTask { public static final String AI_RESPONSE_MARKER = "ASSISTANT:"; private static final String LLAMA_CPP_META_INFO_MARKER = "llm_load_print_meta: "; - private final Model model; + MailQuery mailQuery; private final Float temperature; - private final String systemPrompt; - private final String userPrompt; File inputFile; /** * Creates a new AI task. */ public AiTask(MailQuery mailQuery) { - this.model = mailQuery.model; + this.mailQuery = mailQuery; this.temperature = configuration.getDefaultTemperature(); - this.systemPrompt = mailQuery.systemPrompt; - this.userPrompt = mailQuery.userPrompt; } private String buildAiQuery() { StringBuilder sb = new StringBuilder(); - sb.append("SYSTEM:\n").append(systemPrompt).append("\n"); + sb.append("SYSTEM:\n").append(mailQuery.systemPrompt).append("\n"); - String filteredUserPrompt = filterParticipantsInUserInput(userPrompt); + String filteredUserPrompt = filterParticipantsInUserInput(mailQuery.userPrompt); if (!filteredUserPrompt.startsWith("USER:")) sb.append("USER:\n"); sb.append(filteredUserPrompt).append("\n"); @@ -144,16 +139,21 @@ public class AiTask { int niceValue = 10; // Set the desired niceness level (10 is a common value for background tasks) + String executablePath = configuration.getLlamaCppDirPath().getAbsolutePath(); + if (!executablePath.endsWith("/")) executablePath += "/"; + + executablePath += "main"; + return join(" ", "nice", "-n", Integer.toString(niceValue), - configuration.getLlamaCppExecutablePath().getAbsolutePath(), - "--model " + model.filesystemPath, + executablePath, + "--model " + mailQuery.model.filesystemPath, "--threads " + configuration.getThreadCount(), "--threads-batch " + configuration.getBatchThreadCount(), "--mirostat 2", "--log-disable", "--temp " + temperature, - "--ctx-size " + model.contextSizeTokens, + "--ctx-size " + mailQuery.model.contextSizeTokens, "--batch-size 8", "-n -1", "--repeat_penalty 1.1", @@ -209,12 +209,13 @@ public class AiTask { } // remove text after end of text marker, if it exists - if (model.endOfTextMarker != null) { - int endOfTextMarkerIndex = result.indexOf(model.endOfTextMarker); + if (mailQuery.model.endOfTextMarker != null) { + int endOfTextMarkerIndex = result.indexOf(mailQuery.model.endOfTextMarker); if (endOfTextMarkerIndex != -1) { result = result.substring(0, endOfTextMarkerIndex); } } + return result + "\n"; } diff --git a/src/main/java/eu/svjatoslav/alyverkko_cli/commands/MailCorrespondentCommand.java b/src/main/java/eu/svjatoslav/alyverkko_cli/commands/MailCorrespondentCommand.java index 7ad54f0..3303f32 100644 --- a/src/main/java/eu/svjatoslav/alyverkko_cli/commands/MailCorrespondentCommand.java +++ b/src/main/java/eu/svjatoslav/alyverkko_cli/commands/MailCorrespondentCommand.java @@ -103,15 +103,17 @@ public class MailCorrespondentCommand implements Command { // Parse TOCOMPUTE line for inference settings String firstLine = inputFileContent.substring(0, firstNewLineIndex); Map settings = parseSettings(firstLine); - String systemPromptAlias = settings.getOrDefault("prompt", "default"); - String modelAlias = settings.getOrDefault("model", "default"); - mailQuery.systemPrompt = configuration.getPromptByAlias(systemPromptAlias); - Optional modelOptional = modelLibrary.findModelByAlias(modelAlias); - if (!modelOptional.isPresent()) { - throw new IllegalArgumentException("Model with alias '" + modelAlias + "' not found."); + mailQuery.systemPrompt = configuration.getPromptByAlias(settings.getOrDefault("prompt", "default")); + + { // resolve model + String modelAlias = settings.getOrDefault("model", "default"); + Optional modelOptional = modelLibrary.findModelByAlias(modelAlias); + if (!modelOptional.isPresent()) + throw new IllegalArgumentException("Model with alias '" + modelAlias + "' not found."); + mailQuery.model = modelOptional.get(); } - mailQuery.model = modelOptional.get(); + return mailQuery; } diff --git a/src/main/java/eu/svjatoslav/alyverkko_cli/commands/SelftestCommand.java b/src/main/java/eu/svjatoslav/alyverkko_cli/commands/SelftestCommand.java index 52843f1..836ef1e 100644 --- a/src/main/java/eu/svjatoslav/alyverkko_cli/commands/SelftestCommand.java +++ b/src/main/java/eu/svjatoslav/alyverkko_cli/commands/SelftestCommand.java @@ -35,8 +35,8 @@ public class SelftestCommand implements Command { } // Validate llama.cpp executable path - if (!Main.configuration.getLlamaCppExecutablePath().exists() || !Main.configuration.getLlamaCppExecutablePath().isFile()) { - System.err.println("llama.cpp executable not found at: " + Main.configuration.getLlamaCppExecutablePath()); + if (!Main.configuration.getLlamaCppDirPath().exists() || !Main.configuration.getLlamaCppDirPath().isDirectory()) { + System.err.println("llama.cpp project directory not found at: " + Main.configuration.getLlamaCppDirPath()); return; } diff --git a/src/main/java/eu/svjatoslav/alyverkko_cli/configuration/Configuration.java b/src/main/java/eu/svjatoslav/alyverkko_cli/configuration/Configuration.java index 46e0b9f..f432664 100644 --- a/src/main/java/eu/svjatoslav/alyverkko_cli/configuration/Configuration.java +++ b/src/main/java/eu/svjatoslav/alyverkko_cli/configuration/Configuration.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.io.*; import java.util.List; -import java.util.Map; public class Configuration { @@ -22,8 +21,8 @@ public class Configuration { @JsonProperty("default_temperature") private float defaultTemperature; - @JsonProperty("llama_cpp_executable_path") - private File llamaCppExecutablePath; + @JsonProperty("llama_cpp_dir_path") + private File llamaCppDirPath; @JsonProperty("batch_thread_count") private int batchThreadCount; @@ -99,12 +98,12 @@ public class Configuration { this.defaultTemperature = defaultTemperature; } - public File getLlamaCppExecutablePath() { - return llamaCppExecutablePath; + public File getLlamaCppDirPath() { + return llamaCppDirPath; } - public void setLlamaCppExecutablePath(File llamaCppExecutablePath) { - this.llamaCppExecutablePath = llamaCppExecutablePath; + public void setLlamaCppDirPath(File llamaCppDirPath) { + this.llamaCppDirPath = llamaCppDirPath; } public int getBatchThreadCount() { diff --git a/src/main/java/eu/svjatoslav/alyverkko_cli/configuration/ConfigurationModel.java b/src/main/java/eu/svjatoslav/alyverkko_cli/configuration/ConfigurationModel.java index f48a1f3..b3f5640 100644 --- a/src/main/java/eu/svjatoslav/alyverkko_cli/configuration/ConfigurationModel.java +++ b/src/main/java/eu/svjatoslav/alyverkko_cli/configuration/ConfigurationModel.java @@ -49,4 +49,5 @@ public class ConfigurationModel { public void setEndOfTextMarker(String endOfTextMarker) { this.endOfTextMarker = endOfTextMarker; } + } diff --git a/src/main/java/eu/svjatoslav/alyverkko_cli/model/ModelLibrary.java b/src/main/java/eu/svjatoslav/alyverkko_cli/model/ModelLibrary.java index b83f277..01ecaf3 100644 --- a/src/main/java/eu/svjatoslav/alyverkko_cli/model/ModelLibrary.java +++ b/src/main/java/eu/svjatoslav/alyverkko_cli/model/ModelLibrary.java @@ -29,9 +29,7 @@ public class ModelLibrary { models = new ArrayList<>(); - for (ConfigurationModel configModel : configModels) { - addModelFromConfig(configModel); - } + for (ConfigurationModel configModel : configModels) addModelFromConfig(configModel); if (models.isEmpty()) throw new RuntimeException("No models are defined!"); -- 2.20.1