diff --git a/README.md b/README.md index 00b4d7a..5fac4c7 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ The application is configured via `src/main/resources/application.properties`. | `server.port` | `11434` | The port on which the bridge runs. | | `bridge.upstream-base-url` | `http://localhost:5001` | The base URL of the upstream LLM server. | | `bridge.version` | `0.24.0` | The ollama version to report. | +| `bridge.reported-context-offset` | `0` | Tokens to subtract from the downstream context length before reporting it to clients. | ## API Endpoints @@ -68,11 +69,23 @@ This project is licensed under the Apache License 2.0. See the `LICENSE` file fo ./mvnw spring-boot:run ``` + You can also override any of the bridge settings with JVM system properties, for example: + ```bash + java -Dserver.port=11435 -Dbridge.upstream-base-url=http://localhost:5002 -Dbridge.version=0.24.1 -Dbridge.reported-context-offset=5000 -jar target/ollama-oai-bridge.jar + ``` + + If you use VS Code GitHub Copilot BYOK, configure the Ollama provider to point at the bridge endpoint and keep the upstream server behind it. + Or, after building, you can use the provided `start.sh` script. Copy the generated `.jar` file from the `target/` directory and `start.sh` into the same directory, then run: ```bash ./start.sh ``` + The `start.sh` launcher also accepts JVM `-D` options directly, so this works too: + ```bash + ./start.sh -Dserver.port=11435 -Dbridge.upstream-base-url=http://localhost:5002 -Dbridge.version=0.24.1 -Dbridge.reported-context-offset=5000 + ``` + ## Architecture - **`GatewayProxyConfiguration`**: Sets up the Spring Cloud Gateway routes to proxy requests. diff --git a/src/main/java/de/extio/ollama_oai_bridge/BridgeProperties.java b/src/main/java/de/extio/ollama_oai_bridge/BridgeProperties.java index 9bbae71..1876fdd 100644 --- a/src/main/java/de/extio/ollama_oai_bridge/BridgeProperties.java +++ b/src/main/java/de/extio/ollama_oai_bridge/BridgeProperties.java @@ -11,6 +11,8 @@ public class BridgeProperties { private String version = "0.24.0"; + private long reportedContextOffset = 0L; + public URI getUpstreamBaseUrl() { return this.upstreamBaseUrl; } @@ -27,4 +29,12 @@ public class BridgeProperties { this.version = version; } + public long getReportedContextOffset() { + return this.reportedContextOffset; + } + + public void setReportedContextOffset(final long reportedContextOffset) { + this.reportedContextOffset = reportedContextOffset; + } + } \ No newline at end of file diff --git a/src/main/java/de/extio/ollama_oai_bridge/OllamaDiscoveryService.java b/src/main/java/de/extio/ollama_oai_bridge/OllamaDiscoveryService.java index ce08aff..3951117 100644 --- a/src/main/java/de/extio/ollama_oai_bridge/OllamaDiscoveryService.java +++ b/src/main/java/de/extio/ollama_oai_bridge/OllamaDiscoveryService.java @@ -24,10 +24,13 @@ public class OllamaDiscoveryService { private final LlamaServerClient llamaServerClient; + private final BridgeProperties bridgeProperties; + private final ObjectMapper objectMapper; - public OllamaDiscoveryService(LlamaServerClient llamaServerClient, ObjectMapper objectMapper) { + public OllamaDiscoveryService(LlamaServerClient llamaServerClient, BridgeProperties bridgeProperties, ObjectMapper objectMapper) { this.llamaServerClient = llamaServerClient; + this.bridgeProperties = bridgeProperties; this.objectMapper = objectMapper; } @@ -63,12 +66,13 @@ public class OllamaDiscoveryService { ObjectNode response = this.objectMapper.createObjectNode(); response.put("license", ""); - response.put("modelfile", buildModelfile(model, snapshot.props(), snapshot.slots())); - response.put("parameters", buildParameters(snapshot.props())); + long reportedContextLength = reportedContextLength(model, snapshot.props(), snapshot.slots()); + response.put("modelfile", buildModelfile(model, snapshot.props(), reportedContextLength)); + response.put("parameters", buildParameters(snapshot.props(), reportedContextLength)); response.put("template", text(snapshot.props(), "chat_template")); response.put("system", ""); response.set("details", detailsNode(resolveDetails(model, snapshot.props()))); - response.set("model_info", buildModelInfo(model, snapshot.props(), snapshot.slots())); + response.set("model_info", buildModelInfo(model, snapshot.props(), snapshot.slots(), reportedContextLength)); response.set("capabilities", capabilitiesNode(resolveCapabilities(model, snapshot.props()))); response.put("modified_at", defaultString(model.modifiedAt())); return response; @@ -150,7 +154,7 @@ public class OllamaDiscoveryService { return List.copyOf(capabilities); } - private String buildModelfile(LlamaModelDescriptor model, JsonNode props, JsonNode slots) { + private String buildModelfile(LlamaModelDescriptor model, JsonNode props, long reportedContextLength) { StringBuilder modelfile = new StringBuilder("# Modelfile generated by ollama-oai-bridge\n"); modelfile.append("FROM ") .append(firstNonBlank(text(props, "model_path"), model.model(), model.name())) @@ -163,7 +167,7 @@ public class OllamaDiscoveryService { .append("\"\"\"\n"); } - appendNumericModelfileParameter(modelfile, "num_ctx", contextLength(model, props, slots)); + appendNumericModelfileParameter(modelfile, "num_ctx", reportedContextLength); appendTextModelfileParameter(modelfile, "temperature", text(props.path("default_generation_settings").path("params"), "temperature")); appendTextModelfileParameter(modelfile, "top_k", text(props.path("default_generation_settings").path("params"), "top_k")); appendTextModelfileParameter(modelfile, "top_p", text(props.path("default_generation_settings").path("params"), "top_p")); @@ -176,9 +180,9 @@ public class OllamaDiscoveryService { return modelfile.toString().trim(); } - private String buildParameters(JsonNode props) { + private String buildParameters(JsonNode props, long reportedContextLength) { StringBuilder parameters = new StringBuilder(); - appendParameterLine(parameters, "num_ctx", text(props.path("default_generation_settings"), "n_ctx")); + appendParameterLine(parameters, "num_ctx", String.valueOf(reportedContextLength)); appendParameterLine(parameters, "temperature", text(props.path("default_generation_settings").path("params"), "temperature")); appendParameterLine(parameters, "top_k", text(props.path("default_generation_settings").path("params"), "top_k")); appendParameterLine(parameters, "top_p", text(props.path("default_generation_settings").path("params"), "top_p")); @@ -191,7 +195,7 @@ public class OllamaDiscoveryService { return parameters.toString().trim(); } - private ObjectNode buildModelInfo(LlamaModelDescriptor model, JsonNode props, JsonNode slots) { + private ObjectNode buildModelInfo(LlamaModelDescriptor model, JsonNode props, JsonNode slots, long reportedContextLength) { LlamaModelDetails details = resolveDetails(model, props); ObjectNode modelInfo = this.objectMapper.createObjectNode(); @@ -201,7 +205,7 @@ public class OllamaDiscoveryService { putLongIfPositive(modelInfo, "general.parameter_count", longValue(model.meta(), "n_params")); putStringIfPresent(modelInfo, "general.size_label", details.parameterSize()); putStringIfPresent(modelInfo, "general.quantization", details.quantizationLevel()); - putStringIfPresent(modelInfo, architecture + ".context_length", String.valueOf(contextLength(model, props, slots))); + putStringIfPresent(modelInfo, architecture + ".context_length", String.valueOf(reportedContextLength)); putLongIfPositive(modelInfo, architecture + ".embedding_length", longValue(model.meta(), "n_embd")); putLongIfPositive(modelInfo, architecture + ".vocab_size", longValue(model.meta(), "n_vocab")); putLongIfPositive(modelInfo, architecture + ".trained_context_length", longValue(model.meta(), "n_ctx_train")); @@ -228,7 +232,13 @@ public class OllamaDiscoveryService { return busySlots; } - private long contextLength(LlamaModelDescriptor model, JsonNode props, JsonNode slots) { + private long reportedContextLength(LlamaModelDescriptor model, JsonNode props, JsonNode slots) { + long downstreamContextLength = downstreamContextLength(model, props, slots); + long offset = Math.max(0L, this.bridgeProperties.getReportedContextOffset()); + return Math.max(0L, downstreamContextLength - offset); + } + + private long downstreamContextLength(LlamaModelDescriptor model, JsonNode props, JsonNode slots) { long metaContext = longValue(model.meta(), "n_ctx"); if (metaContext > 0) { return metaContext; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f8962fc..c56eebc 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -3,3 +3,4 @@ server.port=11434 bridge.upstream-base-url=http://localhost:5001 bridge.version=0.24.0 +bridge.reported-context-offset=0 diff --git a/src/test/java/de/extio/ollama_oai_bridge/OllamaDiscoveryServiceContextOffsetTests.java b/src/test/java/de/extio/ollama_oai_bridge/OllamaDiscoveryServiceContextOffsetTests.java new file mode 100644 index 0000000..100922e --- /dev/null +++ b/src/test/java/de/extio/ollama_oai_bridge/OllamaDiscoveryServiceContextOffsetTests.java @@ -0,0 +1,46 @@ +package de.extio.ollama_oai_bridge; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; + +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.node.ObjectNode; + +class OllamaDiscoveryServiceContextOffsetTests { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Test + void reportedContextLengthIsReducedBeforeBeingReturnedToClients() throws Exception { + final LlamaServerClient llamaServerClient = mock(LlamaServerClient.class); + final BridgeProperties bridgeProperties = new BridgeProperties(); + bridgeProperties.setReportedContextOffset(5000); + final OllamaDiscoveryService service = new OllamaDiscoveryService(llamaServerClient, bridgeProperties, this.objectMapper); + + when(llamaServerClient.fetchSnapshot()).thenReturn(new LlamaServerClient.LlamaServerSnapshot( + this.objectMapper.readTree(""" + {"default_generation_settings":{"n_ctx":16128,"params":{"temperature":1.0}},"model_path":"/models/gemma.gguf","chat_template":"","eos_token":"","bos_token":"","model_alias":""} + """), + this.objectMapper.createArrayNode(), + java.util.List.of(new LlamaServerClient.LlamaModelDescriptor( + "gemma-4-E4B-it-Q6_K.gguf", + "gemma-4-E4B-it-Q6_K.gguf", + "", + 0L, + "", + new LlamaServerClient.LlamaModelDetails("", "", "gemma4", java.util.List.of("gemma4"), "", ""), + java.util.List.of(), + this.objectMapper.createObjectNode(), + "llamacpp")))); + + final ObjectNode response = service.show("gemma-4-E4B-it-Q6_K.gguf"); + + assertThat(response.path("model_info").path("gemma4.context_length").asLong()).isEqualTo(11_128L); + assertThat(response.path("parameters").asText()).contains("num_ctx 11128"); + assertThat(response.path("modelfile").asText()).contains("PARAMETER num_ctx 11128"); + } + +} \ No newline at end of file diff --git a/start.sh b/start.sh index 2ff34b6..c25a405 100755 --- a/start.sh +++ b/start.sh @@ -20,4 +20,12 @@ if [[ -n "${OLLAMA_OAI_BRIDGE_OPTS:-}" ]]; then read -r -a app_opts <<<"${OLLAMA_OAI_BRIDGE_OPTS}" fi -exec java "${java_opts[@]}" "${app_opts[@]}" -jar "$jar_file" "$@" \ No newline at end of file +for arg in "$@"; do + if [[ "$arg" == -D* ]]; then + java_opts+=("$arg") + else + app_opts+=("$arg") + fi +done + +exec java "${java_opts[@]}" -jar "$jar_file" "${app_opts[@]}" \ No newline at end of file