diff --git a/README.md b/README.md index f99c24b..95e59c9 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,7 @@ If you disable a module that another module depends on, you must provide the mis | `lmlib.client.proxy.port` | Integer | `0` | Proxy port. | | `lmlib.client.proxy.user` | String | | Proxy username. | | `lmlib.client.proxy.password` | String | | Proxy password. | +| `lmlib.client.proxy.auth-mode` | String | `http-client` | Proxy authentication mode: `http-client` uses the JDK `HttpClient` authenticator; `header` sends `Proxy-Authorization` on each request from the LLM clients. | | `lmlib.client.tls.verification.disabled` | Boolean | `false` | Disables TLS certificate verification. Not recommended in production. | | `lmlib.client.retry.max-attempts` | Integer | `5` | Maximum retry attempts for failed HTTP calls. | | `lmlib.client.retry.backoff-interval-min` | Long | `125` | Minimum retry backoff in milliseconds. | diff --git a/src/main/java/de/extio/lmlib/client/ProxyAuthorizationSupport.java b/src/main/java/de/extio/lmlib/client/ProxyAuthorizationSupport.java new file mode 100644 index 0000000..7b02239 --- /dev/null +++ b/src/main/java/de/extio/lmlib/client/ProxyAuthorizationSupport.java @@ -0,0 +1,48 @@ +package de.extio.lmlib.client; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import org.springframework.web.client.RestClient; + +public final class ProxyAuthorizationSupport { + + public static final String MODE_HTTP_CLIENT = "http-client"; + + public static final String MODE_HEADER = "header"; + + public static final String PROXY_AUTH_HEADER = "Proxy-Authorization"; + + private final boolean headerMode; + + private final String proxyUser; + + private final String proxyPassword; + + public ProxyAuthorizationSupport(final String authMode, final String proxyUser, final String proxyPassword) { + this.headerMode = MODE_HEADER.equalsIgnoreCase(authMode); + this.proxyUser = proxyUser; + this.proxyPassword = proxyPassword; + } + + public boolean isHeaderMode() { + return this.headerMode; + } + + public boolean isHttpClientMode() { + return !this.headerMode; + } + + @SuppressWarnings("unchecked") + public > S apply(final S requestSpec) { + if (!this.headerMode || this.proxyUser == null || this.proxyUser.isBlank()) { + return requestSpec; + } + return (S) requestSpec.header(PROXY_AUTH_HEADER, "Basic " + encodeCredentials(this.proxyUser, this.proxyPassword)); + } + + static String encodeCredentials(final String user, final String password) { + return Base64.getEncoder().encodeToString((user + ":" + (password != null ? password : "")).getBytes(StandardCharsets.UTF_8)); + } + +} diff --git a/src/main/java/de/extio/lmlib/client/RestClientConfiguration.java b/src/main/java/de/extio/lmlib/client/RestClientConfiguration.java index c0587ef..e30eea0 100644 --- a/src/main/java/de/extio/lmlib/client/RestClientConfiguration.java +++ b/src/main/java/de/extio/lmlib/client/RestClientConfiguration.java @@ -49,6 +49,7 @@ public class RestClientConfiguration { @Value("${lmlib.client.proxy.port:0}") final int proxyPort, @Value("${lmlib.client.proxy.user:}") final String proxyUser, @Value("${lmlib.client.proxy.password:}") final String proxyPassword, + @Value("${lmlib.client.proxy.auth-mode:http-client}") final String proxyAuthMode, @Value("${lmlib.client.tls.verification.disabled:false}") final boolean tlsVerificationDisabled, @Value("${lmlib.client.retry.max-attempts:5}") final int maxAttempts, @Value("${lmlib.client.retry.backoff-interval-min:125}") final long backoffIntervalMin, @@ -60,7 +61,7 @@ public class RestClientConfiguration { if (proxyEnabled && proxyHost != null && !proxyHost.isBlank()) { httpClientBuilder.proxy(ProxySelector.of(new InetSocketAddress(proxyHost, proxyPort))); - if (proxyUser != null && !proxyUser.isBlank()) { + if (proxyUser != null && !proxyUser.isBlank() && ProxyAuthorizationSupport.MODE_HTTP_CLIENT.equalsIgnoreCase(proxyAuthMode)) { httpClientBuilder.authenticator(new Authenticator() { @Override @@ -105,6 +106,15 @@ public class RestClientConfiguration { .defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) .requestInterceptor(createRetryInterceptor(maxAttempts, backoffIntervalMin, backoffIntervalMax)); } + + @Bean + ProxyAuthorizationSupport proxyAuthorizationSupport( + @Value("${lmlib.client.proxy.auth-mode:http-client}") final String proxyAuthMode, + @Value("${lmlib.client.proxy.user:}") final String proxyUser, + @Value("${lmlib.client.proxy.password:}") final String proxyPassword) { + LOGGER.debug("Proxy auth mode: {}", proxyAuthMode); + return new ProxyAuthorizationSupport(proxyAuthMode, proxyUser, proxyPassword); + } private ClientHttpRequestInterceptor createRetryInterceptor(final int maxAttempts, final long backoffIntervalMin, final long backoffIntervalMax) { return (request, body, execution) -> { diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java index 1ba9dd9..4b89f34 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/AbstractCompletionClient.java @@ -33,6 +33,7 @@ import de.extio.lmlib.client.Client; import de.extio.lmlib.client.Completion; import de.extio.lmlib.client.CompletionFinishReason; import de.extio.lmlib.client.Conversation; +import de.extio.lmlib.client.ProxyAuthorizationSupport; import de.extio.lmlib.client.ToolCallData; import de.extio.lmlib.client.oai.Model; import de.extio.lmlib.client.oai.ModelsResponse; @@ -62,6 +63,9 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean @Autowired(required = false) protected OpenAiProviderDialect openAiProviderDialect; + + @Autowired + protected ProxyAuthorizationSupport proxyAuthorizationSupport; @Value("${client.collectStatistics:false}") protected boolean collectStatistics; @@ -185,9 +189,11 @@ public abstract class AbstractCompletionClient implements Client, DisposableBean private List loadModelNames(final ModelProfile modelProfile) { try { final var restClient = this.restClientBuilder.baseUrl(modelProfile.url()).build(); - final var response = restClient + var requestSpec = restClient .method(HttpMethod.GET) - .uri(uriBuilder -> uriBuilder.path("/v1/models").build()) + .uri(uriBuilder -> uriBuilder.path("/v1/models").build()); + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); + final var response = requestSpec .retrieve() .body(ModelsResponse.class); return response == null || response.getData() == null ? List.of() : response.getData() diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java index 9b5ce5a..53b094f 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/chat/ChatCompletionClient.java @@ -80,6 +80,7 @@ public class ChatCompletionClient extends AbstractCompletionClient { if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); } + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); final var response = requestSpec .body(requestBody) diff --git a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java index 049456c..75d20d8 100755 --- a/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/completion/text/TextCompletionClient.java @@ -83,6 +83,7 @@ public class TextCompletionClient extends AbstractCompletionClient { if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); } + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); final var response = requestSpec .body(requestBody) .exchange((clientRequest, clientResponse) -> { diff --git a/src/main/java/de/extio/lmlib/client/oai/responses/ResponsesApiClient.java b/src/main/java/de/extio/lmlib/client/oai/responses/ResponsesApiClient.java index 9109513..2503d41 100644 --- a/src/main/java/de/extio/lmlib/client/oai/responses/ResponsesApiClient.java +++ b/src/main/java/de/extio/lmlib/client/oai/responses/ResponsesApiClient.java @@ -34,6 +34,7 @@ import de.extio.lmlib.client.CompletionFinishReason; import de.extio.lmlib.client.CompletionStatistics; import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.Conversation.Turn; +import de.extio.lmlib.client.ProxyAuthorizationSupport; import de.extio.lmlib.client.ToolCall; import de.extio.lmlib.client.ToolCallData; import de.extio.lmlib.client.ToolDefinition; @@ -74,6 +75,9 @@ public class ResponsesApiClient implements Client { @Autowired(required = false) private OpenAiResponsesApiDialect openAiResponsesApiDialect; + + @Autowired + private ProxyAuthorizationSupport proxyAuthorizationSupport; private final Map> resolvedModelNames = new ConcurrentHashMap<>(); @@ -185,6 +189,7 @@ public class ResponsesApiClient implements Client { if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); } + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); return requestSpec.body(requestBody).exchange((clientRequest, clientResponse) -> { if (clientResponse.getStatusCode().isError()) { @@ -787,6 +792,7 @@ public class ResponsesApiClient implements Client { if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); } + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); final var response = requestSpec.retrieve().body(String.class); if (response == null || response.isBlank()) { return List.of(); diff --git a/src/main/java/de/extio/lmlib/client/ollama/OllamaClient.java b/src/main/java/de/extio/lmlib/client/ollama/OllamaClient.java index 6962359..605a1a9 100644 --- a/src/main/java/de/extio/lmlib/client/ollama/OllamaClient.java +++ b/src/main/java/de/extio/lmlib/client/ollama/OllamaClient.java @@ -36,6 +36,7 @@ import de.extio.lmlib.client.CompletionFinishReason; import de.extio.lmlib.client.CompletionStatistics; import de.extio.lmlib.client.Conversation; import de.extio.lmlib.client.Conversation.Turn; +import de.extio.lmlib.client.ProxyAuthorizationSupport; import de.extio.lmlib.client.ToolCall; import de.extio.lmlib.client.ToolCallData; import de.extio.lmlib.client.ToolDefinition; @@ -80,6 +81,9 @@ public class OllamaClient implements Client { @Autowired(required = false) private OllamaDialect ollamaDialect; + + @Autowired + private ProxyAuthorizationSupport proxyAuthorizationSupport; @Override public Completion conversation(final ModelCategory modelCategory, final Conversation conversation, final ToolCallData toolCallData, final boolean skipCache) { @@ -124,6 +128,7 @@ public class OllamaClient implements Client { if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); } + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); final var response = requestSpec .body(requestBody) @@ -443,6 +448,7 @@ public class OllamaClient implements Client { if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); } + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); final var response = requestSpec.retrieve().body(OllamaTagsResponse.class); return response == null || response.getModels() == null ? List.of() : response.getModels().stream() @@ -482,6 +488,7 @@ public class OllamaClient implements Client { if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); } + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); final var requestBody = this.objectMapper.writeValueAsString(request); final var response = requestSpec.body(requestBody).retrieve().body(OllamaShowResponse.class); return response == null ? new OllamaShowResponse() : response; diff --git a/src/main/java/de/extio/lmlib/client/openrouter/OpenRouterModelDiscovery.java b/src/main/java/de/extio/lmlib/client/openrouter/OpenRouterModelDiscovery.java index 84fda5e..d3049f1 100644 --- a/src/main/java/de/extio/lmlib/client/openrouter/OpenRouterModelDiscovery.java +++ b/src/main/java/de/extio/lmlib/client/openrouter/OpenRouterModelDiscovery.java @@ -19,6 +19,7 @@ import org.springframework.web.client.RestClient; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestClientResponseException; +import de.extio.lmlib.client.ProxyAuthorizationSupport; import de.extio.lmlib.client.oai.OpenAiProviderDialect; import de.extio.lmlib.profile.ModelProfile; import de.extio.lmlib.profile.ModelProfile.ModelProvider; @@ -33,6 +34,9 @@ public class OpenRouterModelDiscovery { @Qualifier("lmLibRestClientBuilder") private RestClient.Builder restClientBuilder; + @Autowired + private ProxyAuthorizationSupport proxyAuthorizationSupport; + private final Map> modelListCache = new ConcurrentHashMap<>(); private final Map endpointCache = new ConcurrentHashMap<>(); @@ -109,6 +113,7 @@ public class OpenRouterModelDiscovery { if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); } + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); final var response = requestSpec.retrieve().body(OpenRouterModelsResponse.class); return response == null || response.getData() == null ? List.of() : List.copyOf(response.getData()); } @@ -135,6 +140,7 @@ public class OpenRouterModelDiscovery { if (modelProfile.apiKey() != null && !modelProfile.apiKey().isBlank()) { requestSpec = requestSpec.header("Authorization", "Bearer " + modelProfile.apiKey()); } + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); final var response = requestSpec.retrieve().body(OpenRouterModelEndpointsResponse.class); return response == null || response.getData() == null ? new OpenRouterModelEndpointsResponse.OpenRouterModelEndpoints() : response.getData(); } diff --git a/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java b/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java index 474f85b..9381f24 100755 --- a/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java +++ b/src/main/java/de/extio/lmlib/token/LlamaServerTokenizer.java @@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory; import org.springframework.http.HttpMethod; import org.springframework.web.client.RestClient; +import de.extio.lmlib.client.ProxyAuthorizationSupport; import de.extio.lmlib.profile.ModelProfile; final class LlamaServerTokenizer implements Tokenizer { @@ -17,9 +18,12 @@ final class LlamaServerTokenizer implements Tokenizer { private static final Logger LOGGER = LoggerFactory.getLogger(LlamaServerTokenizer.class); private final RestClient.Builder restClientBuilder; + + private final ProxyAuthorizationSupport proxyAuthorizationSupport; - public LlamaServerTokenizer(final RestClient.Builder restClientBuilder) { + public LlamaServerTokenizer(final RestClient.Builder restClientBuilder, final ProxyAuthorizationSupport proxyAuthorizationSupport) { this.restClientBuilder = restClientBuilder; + this.proxyAuthorizationSupport = proxyAuthorizationSupport; } @Override @@ -37,10 +41,12 @@ final class LlamaServerTokenizer implements Tokenizer { default -> modelProfile.url(); }; final var restClient = this.restClientBuilder.baseUrl(baseUrl).build(); - final var response = restClient + var requestSpec = restClient .method(HttpMethod.POST) .uri("/tokenize") - .body(request) + .body(request); + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); + final var response = requestSpec .retrieve() .onStatus(status -> { if (status.getStatusCode().isError()) { @@ -71,10 +77,12 @@ final class LlamaServerTokenizer implements Tokenizer { default -> modelProfile.url(); }; final var restClient = this.restClientBuilder.baseUrl(baseUrl).build(); - final var response = restClient + var requestSpec = restClient .method(HttpMethod.POST) .uri("/detokenize") - .body(request) + .body(request); + requestSpec = this.proxyAuthorizationSupport.apply(requestSpec); + final var response = requestSpec .retrieve() .onStatus(status -> { if (status.getStatusCode().isError()) { diff --git a/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java b/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java index 40943ee..dc0c9d6 100644 --- a/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java +++ b/src/main/java/de/extio/lmlib/token/TokenizerAutoConfiguration.java @@ -9,6 +9,8 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.web.client.RestClient; +import de.extio.lmlib.client.ProxyAuthorizationSupport; + @AutoConfiguration @ConditionalOnProperty(prefix = "lmlib.tokenizer", name = "enabled", havingValue = "true", matchIfMissing = true) public class TokenizerAutoConfiguration { @@ -16,8 +18,10 @@ public class TokenizerAutoConfiguration { @Bean @ConditionalOnBean(name = "lmLibRestClientBuilder") @ConditionalOnMissingBean(LlamaServerTokenizer.class) - Tokenizer llamaServerTokenizer(@Qualifier("lmLibRestClientBuilder") final RestClient.Builder restClientBuilder) { - return new LlamaServerTokenizer(restClientBuilder); + Tokenizer llamaServerTokenizer( + @Qualifier("lmLibRestClientBuilder") final RestClient.Builder restClientBuilder, + final ProxyAuthorizationSupport proxyAuthorizationSupport) { + return new LlamaServerTokenizer(restClientBuilder, proxyAuthorizationSupport); } @Bean diff --git a/src/test/java/de/extio/lmlib/client/ProxyAuthorizationSupportTest.java b/src/test/java/de/extio/lmlib/client/ProxyAuthorizationSupportTest.java new file mode 100644 index 0000000..0628985 --- /dev/null +++ b/src/test/java/de/extio/lmlib/client/ProxyAuthorizationSupportTest.java @@ -0,0 +1,30 @@ +package de.extio.lmlib.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class ProxyAuthorizationSupportTest { + + @Test + void httpClientModeIsDefault() { + final var support = new ProxyAuthorizationSupport(ProxyAuthorizationSupport.MODE_HTTP_CLIENT, "login", "password"); + assertTrue(support.isHttpClientMode()); + assertFalse(support.isHeaderMode()); + } + + @Test + void headerModeIsEnabled() { + final var support = new ProxyAuthorizationSupport(ProxyAuthorizationSupport.MODE_HEADER, "login", "password"); + assertTrue(support.isHeaderMode()); + assertFalse(support.isHttpClientMode()); + } + + @Test + void encodesCredentials() { + assertEquals("bG9naW46cGFzc3dvcmQ=", ProxyAuthorizationSupport.encodeCredentials("login", "password")); + } + +} diff --git a/src/test/java/de/extio/lmlib/token/TokenizerTest.java b/src/test/java/de/extio/lmlib/token/TokenizerTest.java index c1c8631..65ea793 100644 --- a/src/test/java/de/extio/lmlib/token/TokenizerTest.java +++ b/src/test/java/de/extio/lmlib/token/TokenizerTest.java @@ -13,6 +13,7 @@ import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; import org.springframework.test.context.TestPropertySource; import org.springframework.web.client.RestClient; +import de.extio.lmlib.client.ProxyAuthorizationSupport; import de.extio.lmlib.profile.ModelCategory; import de.extio.lmlib.profile.ModelProfileService; @@ -71,7 +72,7 @@ public class TokenizerTest { @Disabled("Requires llama.cpp model profile") @Test void llamaServer() throws Exception { - final var tokenizer = new LlamaServerTokenizer(this.restClientBuilder); + final var tokenizer = new LlamaServerTokenizer(this.restClientBuilder, new ProxyAuthorizationSupport(ProxyAuthorizationSupport.MODE_HTTP_CLIENT, "", "")); final var profile = this.modelProfileService.getModelProfile(ModelCategory.MEDIUM.getModelProfile()); assertEquals(21, tokenizer.count(TEXT, profile));