commit 4dac2ae7af63cfb49476767dc5500169e9cc9dc0 Author: extio-de Date: Fri May 15 18:54:44 2026 +0200 feat: Implement Ollama OAI Bridge with core functionality - Add BridgeProperties class for configuration management. - Create GatewayProxyConfiguration to define routing and proxying logic. - Implement LlamaServerClient for communication with the upstream Llama server. - Develop OllamaDiscoveryController to handle API requests and responses. - Introduce OllamaDiscoveryService for business logic and data processing. - Set up OllamaOaiBridgeApplication as the main entry point for the application. - Add RequestBodyCachingFilter to cache request bodies for processing. - Create application.properties for configuration settings. - Implement unit tests for the application to ensure functionality. - Add start.sh script for easy application startup. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7bb99d5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +target/ +.classpath +.project +.settings +.vscode/settings.json diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..1525ad9 --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip diff --git a/mvnw b/mvnw new file mode 100755 index 0000000..bd8896b --- /dev/null +++ b/mvnw @@ -0,0 +1,295 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.4 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +scriptDir="$(dirname "$0")" +scriptName="$(basename "$0")" + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +actualDistributionDir="" + +# First try the expected directory name (for regular distributions) +if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then + if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then + actualDistributionDir="$distributionUrlNameMain" + fi +fi + +# If not found, search for any directory with the Maven executable (for snapshots) +if [ -z "$actualDistributionDir" ]; then + # enable globbing to iterate over items + set +f + for dir in "$TMP_DOWNLOAD_DIR"/*; do + if [ -d "$dir" ]; then + if [ -f "$dir/bin/$MVN_CMD" ]; then + actualDistributionDir="$(basename "$dir")" + break + fi + fi + done + set -f +fi + +if [ -z "$actualDistributionDir" ]; then + verbose "Contents of $TMP_DOWNLOAD_DIR:" + verbose "$(ls -la "$TMP_DOWNLOAD_DIR")" + die "Could not find Maven distribution directory in extracted archive" +fi + +verbose "Found extracted Maven distribution directory: $actualDistributionDir" +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 0000000..92450f9 --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,189 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.4 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' + +$MAVEN_M2_PATH = "$HOME/.m2" +if ($env:MAVEN_USER_HOME) { + $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME" +} + +if (-not (Test-Path -Path $MAVEN_M2_PATH)) { + New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null +} + +$MAVEN_WRAPPER_DISTS = $null +if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) { + $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists" +} else { + $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists" +} + +$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain" +$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +$actualDistributionDir = "" + +# First try the expected directory name (for regular distributions) +$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain" +$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD" +if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) { + $actualDistributionDir = $distributionUrlNameMain +} + +# If not found, search for any directory with the Maven executable (for snapshots) +if (!$actualDistributionDir) { + Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object { + $testPath = Join-Path $_.FullName "bin/$MVN_CMD" + if (Test-Path -Path $testPath -PathType Leaf) { + $actualDistributionDir = $_.Name + } + } +} + +if (!$actualDistributionDir) { + Write-Error "Could not find Maven distribution directory in extracted archive" +} + +Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir" +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..712c067 --- /dev/null +++ b/pom.xml @@ -0,0 +1,83 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 4.0.6 + + + de.extio + ollama-oai-bridge + 0.0.1-SNAPSHOT + + + + + + + + + + + + + + + + + 25 + 2025.1.1 + + + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.cloud + spring-cloud-starter-gateway-server-webmvc + + + + org.springframework.boot + spring-boot-starter-actuator-test + test + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + org.springframework.cloud + spring-cloud-dependencies + ${spring-cloud.version} + pom + import + + + + + + ${project.artifactId} + + + org.springframework.boot + spring-boot-maven-plugin + + de.extio.ollama_oai_bridge.OllamaOaiBridgeApplication + + + + + + diff --git a/requests.txt b/requests.txt new file mode 100644 index 0000000..a0a9180 --- /dev/null +++ b/requests.txt @@ -0,0 +1,23 @@ +# ollama serve: + +GET /api/version +{"version":"0.24.0"} + +GET /api/tags +{"models":[{"name":"gemma-4-26B-A4B-it-UD-Q4_K_M:latest","model":"gemma-4-26B-A4B-it-UD-Q4_K_M:latest","modified_at":"2026-05-15T12:43:43.896860597+02:00","size":16868241115,"digest":"b16ef1d08ff9df45d8d95d352be6b0578785d9ab2d7e2a78a14151292b6093a4","details":{"parent_model":"","format":"gguf","family":"gemma4","families":["gemma4"],"parameter_size":"25.2B","quantization_level":"Q4_K_M"}},{"name":"gemma4-e4b-q4:latest","model":"gemma4-e4b-q4:latest","modified_at":"2026-05-09T17:04:49.482936443+02:00","size":4977169498,"digest":"50ddaa9b8045236a361e88c886904c9b6486fafd3434411fc897dc0e278f612a","details":{"parent_model":"","format":"gguf","family":"gemma4","families":["gemma4"],"parameter_size":"7.5B","quantization_level":"Q4_K_M"}}]} + +POST /api/show +request: {"model": "gemma-4-26B-A4B-it-UD-Q4_K_M:latest"} +response: {"modelfile":"# Modelfile generated by \"ollama show\"\n# To build a new Modelfile based on this, replace FROM with:\n# FROM gemma-4-26B-A4B-it-UD-Q4_K_M:latest\n\nFROM /home/sb/.ollama/models/blobs/sha256-b8707e57f676d8dd1b80f623b45200cc92e6966b0e95275e606f412095a49fde\nTEMPLATE {{ .Prompt }}\nRENDERER gemma4\nPARSER gemma4\nPARAMETER stop \u003cturn|\u003e\n","parameters":"stop \"\u003cturn|\u003e\"","template":"{{ .Prompt }}","details":{"parent_model":"/home/sb/.ollama/models/blobs/sha256-b8707e57f676d8dd1b80f623b45200cc92e6966b0e95275e606f412095a49fde","format":"gguf","family":"gemma4","families":["gemma4"],"parameter_size":"25.2B","quantization_level":"Q4_K_M"},"model_info":{"gemma4.attention.head_count":16,"gemma4.attention.head_count_kv":null,"gemma4.attention.key_length":512,"gemma4.attention.key_length_swa":256,"gemma4.attention.layer_norm_rms_epsilon":0.000001,"gemma4.attention.shared_kv_layers":0,"gemma4.attention.sliding_window":1024,"gemma4.attention.sliding_window_pattern":null,"gemma4.attention.value_length":512,"gemma4.attention.value_length_swa":256,"gemma4.block_count":30,"gemma4.context_length":262144,"gemma4.embedding_length":2816,"gemma4.embedding_length_per_layer_input":0,"gemma4.expert_count":128,"gemma4.expert_feed_forward_length":704,"gemma4.expert_used_count":8,"gemma4.feed_forward_length":2112,"gemma4.final_logit_softcapping":30,"gemma4.rope.dimension_count":512,"gemma4.rope.dimension_count_swa":256,"gemma4.rope.freq_base":1000000,"gemma4.rope.freq_base_swa":10000,"general.architecture":"gemma4","general.base_model.0.name":"Gemma 4 26B A4B It","general.base_model.0.organization":"Google","general.base_model.0.repo_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","general.base_model.count":1,"general.basename":"Gemma-4-26B-A4B-It","general.file_type":15,"general.finetune":"it","general.license":"apache-2.0","general.license.link":"https://ai.google.dev/gemma/docs/gemma_4_license","general.parameter_count":25233142046,"general.quantization_version":2,"general.quantized_by":"Unsloth","general.repo_url":"https://huggingface.co/unsloth","general.sampling.temp":1,"general.sampling.top_k":64,"general.sampling.top_p":0.95,"general.size_label":"26B-A4B","general.tags":null,"general.type":"model","quantize.imatrix.chunks_count":141,"quantize.imatrix.dataset":"unsloth_calibration_gemma-4-26B-A4B-it.txt","quantize.imatrix.entries_count":295,"quantize.imatrix.file":"gemma-4-26B-A4B-it-GGUF/imatrix_unsloth.gguf","tokenizer.ggml.add_bos_token":true,"tokenizer.ggml.add_space_prefix":false,"tokenizer.ggml.bos_token_id":2,"tokenizer.ggml.eos_token_id":106,"tokenizer.ggml.mask_token_id":4,"tokenizer.ggml.merges":null,"tokenizer.ggml.model":"gemma4","tokenizer.ggml.padding_token_id":0,"tokenizer.ggml.scores":null,"tokenizer.ggml.token_type":null,"tokenizer.ggml.tokens":null,"tokenizer.ggml.unknown_token_id":3},"tensors":[{"name":"output_norm.weight","type":"F32","shape":[2816]},{"name":"rope_freqs.weight","type":"F32","shape":[256]},{"name":"token_embd.weight","type":"Q8_0","shape":[2816,262144]},{"name":"blk.0.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.0.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.0.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.0.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.0.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.0.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.0.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.0.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.0.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.0.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.0.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.0.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.0.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.0.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.0.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.0.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.0.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.0.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.0.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.0.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.0.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.0.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.1.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.1.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.1.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.1.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.1.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.1.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.1.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.1.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.1.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.1.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.1.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.1.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.1.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.1.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.1.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.1.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.1.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.1.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.1.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.1.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.1.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.1.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.2.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.2.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.2.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.2.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.2.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.2.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.2.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.2.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.2.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.2.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.2.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.2.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.2.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.2.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.2.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.2.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.2.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.2.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.2.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.2.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.2.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.2.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.3.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.3.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.3.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.3.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.3.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.3.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.3.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.3.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.3.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.3.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.3.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.3.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.3.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.3.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.3.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.3.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.3.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.3.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.3.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.3.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.3.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.3.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.4.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.4.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.4.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.4.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.4.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.4.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.4.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.4.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.4.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.4.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.4.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.4.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.4.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.4.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.4.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.4.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.4.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.4.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.4.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.4.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.4.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.4.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.5.attn_k.weight","type":"Q8_0","shape":[2816,1024]},{"name":"blk.5.attn_k_norm.weight","type":"F32","shape":[512]},{"name":"blk.5.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.5.attn_output.weight","type":"Q8_0","shape":[8192,2816]},{"name":"blk.5.attn_q.weight","type":"Q8_0","shape":[2816,8192]},{"name":"blk.5.attn_q_norm.weight","type":"F32","shape":[512]},{"name":"blk.5.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.5.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.5.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.5.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.5.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.5.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.5.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.5.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.5.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.5.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.5.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.5.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.5.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.5.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.5.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.6.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.6.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.6.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.6.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.6.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.6.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.6.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.6.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.6.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.6.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.6.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.6.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.6.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.6.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.6.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.6.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.6.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.6.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.6.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.6.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.6.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.6.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.7.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.7.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.7.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.7.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.7.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.7.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.7.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.7.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.7.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.7.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.7.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.7.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.7.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.7.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.7.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.7.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.7.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.7.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.7.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.7.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.7.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.7.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.8.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.8.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.8.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.8.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.8.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.8.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.8.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.8.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.8.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.8.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.8.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.8.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.8.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.8.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.8.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.8.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.8.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.8.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.8.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.8.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.8.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.8.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.9.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.9.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.9.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.9.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.9.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.9.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.9.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.9.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.9.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.9.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.9.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.9.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.9.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.9.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.9.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.9.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.9.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.9.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.9.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.9.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.9.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.9.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.10.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.10.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.10.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.10.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.10.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.10.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.10.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.10.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.10.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.10.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.10.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.10.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.10.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.10.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.10.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.10.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.10.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.10.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.10.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.10.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.10.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.10.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.11.attn_k.weight","type":"Q8_0","shape":[2816,1024]},{"name":"blk.11.attn_k_norm.weight","type":"F32","shape":[512]},{"name":"blk.11.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.11.attn_output.weight","type":"Q8_0","shape":[8192,2816]},{"name":"blk.11.attn_q.weight","type":"Q8_0","shape":[2816,8192]},{"name":"blk.11.attn_q_norm.weight","type":"F32","shape":[512]},{"name":"blk.11.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.11.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.11.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.11.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.11.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.11.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.11.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.11.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.11.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.11.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.11.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.11.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.11.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.11.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.11.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.12.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.12.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.12.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.12.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.12.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.12.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.12.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.12.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.12.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.12.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.12.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.12.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.12.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.12.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.12.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.12.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.12.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.12.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.12.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.12.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.12.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.12.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.13.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.13.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.13.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.13.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.13.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.13.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.13.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.13.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.13.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.13.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.13.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.13.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.13.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.13.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.13.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.13.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.13.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.13.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.13.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.13.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.13.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.13.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.14.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.14.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.14.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.14.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.14.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.14.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.14.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.14.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.14.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.14.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.14.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.14.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.14.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.14.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.14.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.14.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.14.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.14.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.14.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.14.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.14.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.14.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.15.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.15.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.15.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.15.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.15.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.15.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.15.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.15.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.15.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.15.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.15.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.15.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.15.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.15.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.15.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.15.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.15.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.15.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.15.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.15.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.15.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.15.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.16.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.16.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.16.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.16.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.16.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.16.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.16.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.16.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.16.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.16.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.16.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.16.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.16.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.16.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.16.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.16.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.16.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.16.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.16.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.16.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.16.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.16.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.17.attn_k.weight","type":"Q8_0","shape":[2816,1024]},{"name":"blk.17.attn_k_norm.weight","type":"F32","shape":[512]},{"name":"blk.17.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.17.attn_output.weight","type":"Q8_0","shape":[8192,2816]},{"name":"blk.17.attn_q.weight","type":"Q8_0","shape":[2816,8192]},{"name":"blk.17.attn_q_norm.weight","type":"F32","shape":[512]},{"name":"blk.17.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.17.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.17.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.17.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.17.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.17.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.17.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.17.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.17.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.17.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.17.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.17.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.17.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.17.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.17.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.18.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.18.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.18.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.18.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.18.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.18.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.18.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.18.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.18.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.18.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.18.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.18.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.18.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.18.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.18.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.18.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.18.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.18.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.18.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.18.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.18.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.18.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.19.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.19.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.19.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.19.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.19.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.19.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.19.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.19.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.19.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.19.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.19.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.19.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.19.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.19.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.19.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.19.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.19.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.19.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.19.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.19.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.19.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.19.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.20.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.20.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.20.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.20.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.20.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.20.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.20.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.20.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.20.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.20.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.20.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.20.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.20.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.20.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.20.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.20.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.20.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.20.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.20.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.20.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.20.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.20.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.21.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.21.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.21.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.21.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.21.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.21.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.21.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.21.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.21.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.21.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.21.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.21.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.21.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.21.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.21.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.21.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.21.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.21.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.21.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.21.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.21.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.21.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.22.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.22.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.22.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.22.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.22.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.22.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.22.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.22.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.22.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.22.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.22.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.22.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.22.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.22.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.22.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.22.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.22.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.22.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.22.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.22.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.22.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.22.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.23.attn_k.weight","type":"Q8_0","shape":[2816,1024]},{"name":"blk.23.attn_k_norm.weight","type":"F32","shape":[512]},{"name":"blk.23.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.23.attn_output.weight","type":"Q8_0","shape":[8192,2816]},{"name":"blk.23.attn_q.weight","type":"Q8_0","shape":[2816,8192]},{"name":"blk.23.attn_q_norm.weight","type":"F32","shape":[512]},{"name":"blk.23.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.23.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.23.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.23.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.23.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.23.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.23.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.23.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.23.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.23.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.23.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.23.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.23.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.23.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.23.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.24.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.24.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.24.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.24.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.24.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.24.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.24.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.24.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.24.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.24.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.24.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.24.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.24.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.24.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.24.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.24.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.24.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.24.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.24.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.24.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.24.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.24.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.25.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.25.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.25.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.25.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.25.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.25.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.25.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.25.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.25.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.25.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.25.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.25.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.25.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.25.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.25.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.25.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.25.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.25.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.25.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.25.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.25.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.25.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.26.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.26.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.26.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.26.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.26.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.26.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.26.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.26.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.26.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.26.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.26.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.26.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.26.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.26.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.26.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.26.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.26.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.26.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.26.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.26.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.26.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.26.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.27.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.27.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.27.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.27.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.27.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.27.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.27.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.27.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.27.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.27.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.27.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.27.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.27.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.27.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.27.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.27.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.27.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.27.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.27.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.27.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.27.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.27.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.28.attn_k.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.28.attn_k_norm.weight","type":"F32","shape":[256]},{"name":"blk.28.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.28.attn_output.weight","type":"Q8_0","shape":[4096,2816]},{"name":"blk.28.attn_q.weight","type":"Q8_0","shape":[2816,4096]},{"name":"blk.28.attn_q_norm.weight","type":"F32","shape":[256]},{"name":"blk.28.attn_v.weight","type":"Q8_0","shape":[2816,2048]},{"name":"blk.28.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.28.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.28.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.28.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.28.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.28.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.28.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.28.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.28.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.28.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.28.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.28.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.28.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.28.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.28.pre_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.29.attn_k.weight","type":"Q8_0","shape":[2816,1024]},{"name":"blk.29.attn_k_norm.weight","type":"F32","shape":[512]},{"name":"blk.29.attn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.29.attn_output.weight","type":"Q8_0","shape":[8192,2816]},{"name":"blk.29.attn_q.weight","type":"Q8_0","shape":[2816,8192]},{"name":"blk.29.attn_q_norm.weight","type":"F32","shape":[512]},{"name":"blk.29.ffn_down.weight","type":"Q8_0","shape":[2112,2816]},{"name":"blk.29.ffn_down_exps.scale","type":"F32","shape":[128]},{"name":"blk.29.ffn_down_exps.weight","type":"Q5_1","shape":[704,2816,128]},{"name":"blk.29.ffn_gate.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.29.ffn_gate_inp.scale","type":"F32","shape":[2816]},{"name":"blk.29.ffn_gate_inp.weight","type":"F32","shape":[2816,128]},{"name":"blk.29.ffn_gate_up_exps.weight","type":"Q4_K","shape":[2816,1408,128]},{"name":"blk.29.ffn_norm.weight","type":"F32","shape":[2816]},{"name":"blk.29.ffn_up.weight","type":"Q8_0","shape":[2816,2112]},{"name":"blk.29.layer_output_scale.weight","type":"F32","shape":[1]},{"name":"blk.29.post_attention_norm.weight","type":"F32","shape":[2816]},{"name":"blk.29.post_ffw_norm.weight","type":"F32","shape":[2816]},{"name":"blk.29.post_ffw_norm_1.weight","type":"F32","shape":[2816]},{"name":"blk.29.post_ffw_norm_2.weight","type":"F32","shape":[2816]},{"name":"blk.29.pre_ffw_norm_2.weight","type":"F32","shape":[2816]}],"capabilities":["completion","tools","thinking"],"modified_at":"2026-05-15T12:43:43.896860597+02:00"} + +# llama-server + +GET /props +{"default_generation_settings":{"params":{"seed":4294967295,"temperature":1.0,"dynatemp_range":0.0,"dynatemp_exponent":1.0,"top_k":64,"top_p":0.949999988079071,"min_p":0.05000000074505806,"top_n_sigma":-1.0,"xtc_probability":0.0,"xtc_threshold":0.10000000149011612,"typical_p":1.0,"repeat_last_n":64,"repeat_penalty":1.0,"presence_penalty":0.0,"frequency_penalty":0.0,"dry_multiplier":0.0,"dry_base":1.75,"dry_allowed_length":2,"dry_penalty_last_n":-1,"mirostat":0,"mirostat_tau":5.0,"mirostat_eta":0.10000000149011612,"max_tokens":-1,"n_predict":-1,"n_keep":0,"n_discard":0,"ignore_eos":false,"stream":true,"n_probs":0,"min_keep":0,"chat_format":"Content-only","reasoning_format":"none","reasoning_in_content":false,"generation_prompt":"","samplers":["penalties","dry","top_n_sigma","top_k","typ_p","top_p","min_p","xtc","temperature"],"speculative.types":"none","timings_per_token":false,"post_sampling_probs":false,"backend_sampling":false,"lora":[]},"n_ctx":16128},"total_slots":4,"model_alias":"gemma-4-E4B-it-Q6_K.gguf","model_path":"/mnt/data/data/lm/models/gguf/s/gemma-4-E4B-it-Q6_K.gguf","modalities":{"vision":false,"audio":false},"media_marker":"<__media_h6cQgpf6gHGxot8QO2lPIcHvCadr4XSS__>","endpoint_slots":true,"endpoint_props":false,"endpoint_metrics":false,"webui":true,"webui_settings":{},"chat_template":"{%- macro format_parameters(properties, required) -%}\n {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}\n {%- set ns = namespace(found_first=false) -%}\n {%- for key, value in properties | dictsort -%}\n {%- set add_comma = false -%}\n {%- if key not in standard_keys -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {{ key }}:{\n {%- if value['description'] -%}\n description:<|\"|>{{ value['description'] }}<|\"|>\n {%- set add_comma = true -%}\n {%- endif -%}\n {%- if value['type'] | upper == 'STRING' -%}\n {%- if value['enum'] -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n enum:{{ format_argument(value['enum']) }}\n {%- endif -%}\n {%- elif value['type'] | upper == 'ARRAY' -%}\n {%- if value['items'] is mapping and value['items'] -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n items:{\n {%- set ns_items = namespace(found_first=false) -%}\n {%- for item_key, item_value in value['items'] | dictsort -%}\n {%- if item_value is not none -%}\n {%- if ns_items.found_first %},{% endif -%}\n {%- set ns_items.found_first = true -%}\n {%- if item_key == 'properties' -%}\n properties:{\n {%- if item_value is mapping -%}\n {{- format_parameters(item_value, value['items']['required'] | default([])) -}}\n {%- endif -%}\n }\n {%- elif item_key == 'required' -%}\n required:[\n {%- for req_item in item_value -%}\n <|\"|>{{- req_item -}}<|\"|>\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n {%- elif item_key == 'type' -%}\n {%- if item_value is string -%}\n type:{{ format_argument(item_value | upper) }}\n {%- else -%}\n type:{{ format_argument(item_value | map('upper') | list) }}\n {%- endif -%}\n {%- else -%}\n {{ item_key }}:{{ format_argument(item_value) }}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n }\n {%- endif -%}\n {%- endif -%}\n {%- if value['nullable'] %}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n nullable:true\n {%- endif -%}\n {%- if value['type'] | upper == 'OBJECT' -%}\n {%- if value['properties'] is defined and value['properties'] is mapping -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n properties:{\n {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n }\n {%- elif value is mapping -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n properties:{\n {{- format_parameters(value, value['required'] | default([])) -}}\n }\n {%- endif -%}\n {%- if value['required'] -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n required:[\n {%- for item in value['required'] | default([]) -%}\n <|\"|>{{- item -}}<|\"|>\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n {%- endif -%}\n {%- endif -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n type:<|\"|>{{ value['type'] | upper }}<|\"|>}\n {%- endif -%}\n {%- endfor -%}\n{%- endmacro -%}\n{%- macro format_function_declaration(tool_data) -%}\n declaration:{{- tool_data['function']['name'] -}}{description:<|\"|>{{- tool_data['function']['description'] -}}<|\"|>\n {%- set params = tool_data['function']['parameters'] -%}\n {%- if params -%}\n ,parameters:{\n {%- if params['properties'] -%}\n properties:{ {{- format_parameters(params['properties'], params['required']) -}} },\n {%- endif -%}\n {%- if params['required'] -%}\n required:[\n {%- for item in params['required'] -%}\n <|\"|>{{- item -}}<|\"|>\n {{- ',' if not loop.last -}}\n {%- endfor -%}\n ],\n {%- endif -%}\n {%- if params['type'] -%}\n type:<|\"|>{{- params['type'] | upper -}}<|\"|>}\n {%- endif -%}\n {%- endif -%}\n {%- if 'response' in tool_data['function'] -%}\n {%- set response_declaration = tool_data['function']['response'] -%}\n ,response:{\n {%- if response_declaration['description'] -%}\n description:<|\"|>{{- response_declaration['description'] -}}<|\"|>,\n {%- endif -%}\n {%- if response_declaration['type'] | upper == 'OBJECT' -%}\n type:<|\"|>{{- response_declaration['type'] | upper -}}<|\"|>}\n {%- endif -%}\n {%- endif -%}\n }\n{%- endmacro -%}\n{%- macro format_argument(argument, escape_keys=True) -%}\n {%- if argument is string -%}\n {{- '<|\"|>' + argument + '<|\"|>' -}}\n {%- elif argument is boolean -%}\n {{- 'true' if argument else 'false' -}}\n {%- elif argument is mapping -%}\n {{- '{' -}}\n {%- set ns = namespace(found_first=false) -%}\n {%- for key, value in argument | dictsort -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {%- if escape_keys -%}\n {{- '<|\"|>' + key + '<|\"|>' -}}\n {%- else -%}\n {{- key -}}\n {%- endif -%}\n :{{- format_argument(value, escape_keys=escape_keys) -}}\n {%- endfor -%}\n {{- '}' -}}\n {%- elif argument is sequence -%}\n {{- '[' -}}\n {%- for item in argument -%}\n {{- format_argument(item, escape_keys=escape_keys) -}}\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n {{- ']' -}}\n {%- else -%}\n {{- argument -}}\n {%- endif -%}\n{%- endmacro -%}\n{%- macro strip_thinking(text) -%}\n {%- set ns = namespace(result='') -%}\n {%- for part in text.split('') -%}\n {%- if '<|channel>' in part -%}\n {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}\n {%- else -%}\n {%- set ns.result = ns.result + part -%}\n {%- endif -%}\n {%- endfor -%}\n {{- ns.result | trim -}}\n{%- endmacro -%}\n\n{%- macro format_tool_response_block(tool_name, response) -%}\n {{- '<|tool_response>' -}}\n {%- if response is mapping -%}\n {{- 'response:' + tool_name + '{' -}}\n {%- for key, value in response | dictsort -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n {{- '}' -}}\n {%- else -%}\n {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}\n {%- endif -%}\n {{- '' -}}\n{%- endmacro -%}\n\n{%- set ns = namespace(prev_message_type=None) -%}\n{%- set loop_messages = messages -%}\n{{- bos_token -}}\n{#- Handle System/Tool Definitions Block -#}\n{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}\n {{- '<|turn>system\\n' -}}\n\n {#- Inject Thinking token at the very top of the FIRST system turn -#}\n {%- if enable_thinking is defined and enable_thinking -%}\n {{- '<|think|>\\n' -}}\n {%- set ns.prev_message_type = 'think' -%}\n {%- endif -%}\n\n {%- if messages[0]['role'] in ['system', 'developer'] -%}\n {{- messages[0]['content'] | trim -}}\n {%- set loop_messages = messages[1:] -%}\n {%- endif -%}\n\n {%- if tools -%}\n {%- for tool in tools %}\n {{- '<|tool>' -}}\n {{- format_function_declaration(tool) | trim -}}\n {{- '' -}}\n {%- endfor %}\n {%- set ns.prev_message_type = 'tool' -%}\n {%- endif -%}\n\n {{- '\\n' -}}\n{%- endif %}\n\n{#- Pre-scan: find last user message index for reasoning guard -#}\n{%- set ns_turn = namespace(last_user_idx=-1) -%}\n{%- for i in range(loop_messages | length) -%}\n {%- if loop_messages[i]['role'] == 'user' -%}\n {%- set ns_turn.last_user_idx = i -%}\n {%- endif -%}\n{%- endfor -%}\n\n{#- Loop through messages -#}\n{%- for message in loop_messages -%}\n {%- if message['role'] != 'tool' -%}\n {%- set ns.prev_message_type = None -%}\n {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n {#- Detect continuation: suppress duplicate <|turn>model when previous non-tool message was also assistant -#}\n {%- set prev_nt = namespace(role=None, found=false) -%}\n {%- if loop.index0 > 0 -%}\n {%- for j in range(loop.index0 - 1, -1, -1) -%}\n {%- if not prev_nt.found -%}\n {%- if loop_messages[j]['role'] != 'tool' -%}\n {%- set prev_nt.role = loop_messages[j]['role'] -%}\n {%- set prev_nt.found = true -%}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}\n {%- if not continue_same_model_turn -%}\n {{- '<|turn>' + role + '\\n' }}\n {%- endif -%}\n\n {#- Render reasoning/reasoning_content as thinking channel -#}\n {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}\n {{- '<|channel>thought\\n' + thinking_text + '\\n' -}}\n {%- endif -%}\n\n {%- if message['tool_calls'] -%}\n {%- for tool_call in message['tool_calls'] -%}\n {%- set function = tool_call['function'] -%}\n {{- '<|tool_call>call:' + function['name'] + '{' -}}\n {%- if function['arguments'] is mapping -%}\n {%- set ns_args = namespace(found_first=false) -%}\n {%- for key, value in function['arguments'] | dictsort -%}\n {%- if ns_args.found_first %},{% endif -%}\n {%- set ns_args.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- endfor -%}\n {%- elif function['arguments'] is string -%}\n {{- function['arguments'] -}}\n {%- endif -%}\n {{- '}' -}}\n {%- endfor -%}\n {%- set ns.prev_message_type = 'tool_call' -%}\n {%- endif -%}\n\n {%- set ns_tr_out = namespace(flag=false) -%}\n {%- if message.get('tool_responses') -%}\n {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n {%- for tool_response in message['tool_responses'] -%}\n {{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}\n {%- set ns_tr_out.flag = true -%}\n {%- set ns.prev_message_type = 'tool_response' -%}\n {%- endfor -%}\n {%- elif message.get('tool_calls') -%}\n {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#}\n {%- set ns_tool_scan = namespace(stopped=false) -%}\n {%- for k in range(loop.index0 + 1, loop_messages | length) -%}\n {%- if ns_tool_scan.stopped -%}\n {%- elif loop_messages[k]['role'] != 'tool' -%}\n {%- set ns_tool_scan.stopped = true -%}\n {%- else -%}\n {%- set follow = loop_messages[k] -%}\n {#- Resolve tool_call_id to function name -#}\n {%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}\n {%- for tc in message['tool_calls'] -%}\n {%- if tc.get('id') == follow.get('tool_call_id') -%}\n {%- set ns_tname.name = tc['function']['name'] -%}\n {%- endif -%}\n {%- endfor -%}\n {#- Handle content as string or content-parts array -#}\n {%- set tool_body = follow.get('content') -%}\n {%- if tool_body is string -%}\n {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n {%- elif tool_body is sequence and tool_body is not string -%}\n {%- set ns_txt = namespace(s='') -%}\n {%- for part in tool_body -%}\n {%- if part.get('type') == 'text' -%}\n {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}\n {%- endif -%}\n {%- endfor -%}\n {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n {%- else -%}\n {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n {%- endif -%}\n {%- set ns_tr_out.flag = true -%}\n {%- set ns.prev_message_type = 'tool_response' -%}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n\n {%- if message['content'] is string -%}\n {%- if role == 'model' -%}\n {{- strip_thinking(message['content']) -}}\n {%- else -%}\n {{- message['content'] | trim -}}\n {%- endif -%}\n {%- elif message['content'] is sequence -%}\n {%- for item in message['content'] -%}\n {%- if item['type'] == 'text' -%}\n {%- if role == 'model' -%}\n {{- strip_thinking(item['text']) -}}\n {%- else -%}\n {{- item['text'] | trim -}}\n {%- endif -%}\n {%- elif item['type'] == 'image' -%}\n {{- '<|image|>' -}}\n {%- set ns.prev_message_type = 'image' -%}\n {%- elif item['type'] == 'audio' -%}\n {{- '<|audio|>' -}}\n {%- set ns.prev_message_type = 'audio' -%}\n {%- elif item['type'] == 'video' -%}\n {{- '<|video|>' -}}\n {%- set ns.prev_message_type = 'video' -%}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n\n {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n {{- '<|tool_response>' -}}\n {%- elif not (ns_tr_out.flag and not message.get('content')) -%}\n {{- '\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n\n{%- if add_generation_prompt -%}\n {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n {{- '<|turn>model\\n' -}}\n {%- endif -%}\n{%- endif -%}","chat_template_caps":{"supports_object_arguments":true,"supports_parallel_tool_calls":true,"supports_preserve_reasoning":true,"supports_string_content":true,"supports_system_role":true,"supports_tool_calls":true,"supports_tools":true,"supports_typed_content":false},"bos_token":"","eos_token":"","build_info":"b9164-d5dc2e0a0","is_sleeping":false} + +GET /slots +[{"id":0,"n_ctx":16128,"speculative":false,"is_processing":false},{"id":1,"n_ctx":16128,"speculative":false,"is_processing":false},{"id":2,"n_ctx":16128,"speculative":false,"is_processing":false},{"id":3,"n_ctx":16128,"speculative":false,"is_processing":false}] + +GET /v1/models +{"models":[{"name":"gemma-4-E4B-it-Q6_K.gguf","model":"gemma-4-E4B-it-Q6_K.gguf","modified_at":"","size":"","digest":"","type":"model","description":"","tags":[""],"capabilities":["completion"],"parameters":"","details":{"parent_model":"","format":"gguf","family":"","families":[""],"parameter_size":"","quantization_level":""}}],"object":"list","data":[{"id":"gemma-4-E4B-it-Q6_K.gguf","aliases":[],"tags":[],"object":"model","created":1778857538,"owned_by":"llamacpp","meta":{"vocab_type":2,"n_vocab":262144,"n_ctx":16128,"n_ctx_train":131072,"n_embd":2560,"n_params":7518069290,"size":7059101864}}]} + diff --git a/src/main/java/de/extio/ollama_oai_bridge/BridgeProperties.java b/src/main/java/de/extio/ollama_oai_bridge/BridgeProperties.java new file mode 100644 index 0000000..9bbae71 --- /dev/null +++ b/src/main/java/de/extio/ollama_oai_bridge/BridgeProperties.java @@ -0,0 +1,30 @@ +package de.extio.ollama_oai_bridge; + +import java.net.URI; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "bridge") +public class BridgeProperties { + + private URI upstreamBaseUrl = URI.create("http://localhost:5001"); + + private String version = "0.24.0"; + + public URI getUpstreamBaseUrl() { + return this.upstreamBaseUrl; + } + + public void setUpstreamBaseUrl(final URI upstreamBaseUrl) { + this.upstreamBaseUrl = upstreamBaseUrl; + } + + public String getVersion() { + return this.version; + } + + public void setVersion(final String version) { + this.version = version; + } + +} \ No newline at end of file diff --git a/src/main/java/de/extio/ollama_oai_bridge/GatewayProxyConfiguration.java b/src/main/java/de/extio/ollama_oai_bridge/GatewayProxyConfiguration.java new file mode 100644 index 0000000..5fdcd67 --- /dev/null +++ b/src/main/java/de/extio/ollama_oai_bridge/GatewayProxyConfiguration.java @@ -0,0 +1,35 @@ +package de.extio.ollama_oai_bridge; + +import static org.springframework.web.servlet.function.RequestPredicates.GET; +import static org.springframework.web.servlet.function.RequestPredicates.POST; +import static org.springframework.web.servlet.function.RequestPredicates.all; +import static org.springframework.web.servlet.function.RequestPredicates.path; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.gateway.server.mvc.filter.BeforeFilterFunctions; +import org.springframework.cloud.gateway.server.mvc.handler.GatewayRouterFunctions; +import org.springframework.cloud.gateway.server.mvc.handler.HandlerFunctions; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.function.RequestPredicate; +import org.springframework.web.servlet.function.RouterFunction; +import org.springframework.web.servlet.function.ServerResponse; + +@Configuration +@EnableConfigurationProperties(BridgeProperties.class) +public class GatewayProxyConfiguration { + + @Bean + RouterFunction gatewayRoutes(final BridgeProperties bridgeProperties) { + final RequestPredicate locallyHandledRoutes = GET("/api/version") + .or(GET("/api/tags")) + .or(POST("/api/show")) + .or(path("/error")); + + return GatewayRouterFunctions.route("llama-server-proxy") + .before(BeforeFilterFunctions.uri(bridgeProperties.getUpstreamBaseUrl())) + .route(all().and(locallyHandledRoutes.negate()), HandlerFunctions.http()) + .build(); + } + +} \ No newline at end of file diff --git a/src/main/java/de/extio/ollama_oai_bridge/LlamaServerClient.java b/src/main/java/de/extio/ollama_oai_bridge/LlamaServerClient.java new file mode 100644 index 0000000..1dfd586 --- /dev/null +++ b/src/main/java/de/extio/ollama_oai_bridge/LlamaServerClient.java @@ -0,0 +1,251 @@ +package de.extio.ollama_oai_bridge; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.node.MissingNode; + +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClient; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestClientResponseException; +import org.springframework.web.server.ResponseStatusException; + +@Service +public class LlamaServerClient { + + private final RestClient restClient; + + public LlamaServerClient(RestClient.Builder restClientBuilder, BridgeProperties bridgeProperties) { + this.restClient = restClientBuilder.baseUrl(bridgeProperties.getUpstreamBaseUrl().toString()).build(); + } + + public LlamaServerSnapshot fetchSnapshot() { + JsonNode props = getJson("/props"); + JsonNode slots = getJson("/slots"); + JsonNode modelsResponse = getJson("/v1/models"); + + return new LlamaServerSnapshot(props, slots, normalizeModels(modelsResponse)); + } + + private JsonNode getJson(String path) { + try { + JsonNode response = this.restClient.get().uri(path).retrieve().body(JsonNode.class); + if (response == null) { + throw new ResponseStatusException(HttpStatus.BAD_GATEWAY, + "llama-server returned an empty response for " + path); + } + return response; + } + catch (RestClientResponseException exception) { + throw new ResponseStatusException(HttpStatus.BAD_GATEWAY, + "llama-server returned " + exception.getStatusCode().value() + " for " + path, exception); + } + catch (RestClientException exception) { + throw new ResponseStatusException(HttpStatus.BAD_GATEWAY, + "Failed to call llama-server endpoint " + path, exception); + } + } + + private List normalizeModels(JsonNode modelsResponse) { + List models = new ArrayList<>(); + + if (modelsResponse.path("models").isArray()) { + for (JsonNode modelNode : modelsResponse.path("models")) { + models.add(fromOllamaStyleModel(modelNode)); + } + return List.copyOf(models); + } + + if (modelsResponse.path("data").isArray()) { + for (JsonNode modelNode : modelsResponse.path("data")) { + models.add(fromOpenAiStyleModel(modelNode)); + } + } + + return List.copyOf(models); + } + + private LlamaModelDescriptor fromOllamaStyleModel(JsonNode modelNode) { + JsonNode detailsNode = modelNode.path("details"); + String name = text(modelNode, "name"); + return new LlamaModelDescriptor( + name, + defaultText(modelNode, "model", name), + text(modelNode, "modified_at"), + longValue(modelNode.path("size")), + text(modelNode, "digest"), + new LlamaModelDetails( + text(detailsNode, "parent_model"), + text(detailsNode, "format"), + text(detailsNode, "family"), + textList(detailsNode.path("families")), + text(detailsNode, "parameter_size"), + upperCase(text(detailsNode, "quantization_level"))), + textList(modelNode.path("capabilities")), + modelNode.path("meta"), + text(modelNode, "owned_by")); + } + + private LlamaModelDescriptor fromOpenAiStyleModel(JsonNode modelNode) { + JsonNode metaNode = modelNode.path("meta"); + String name = text(modelNode, "id"); + String family = inferFamily(name); + return new LlamaModelDescriptor( + name, + name, + "", + longValue(metaNode.path("size")), + "", + new LlamaModelDetails( + "", + inferFormat(name), + family, + family.isBlank() ? List.of() : List.of(family), + humanizeParameterCount(longValue(metaNode.path("n_params"))), + inferQuantization(name)), + List.of("completion"), + metaNode, + text(modelNode, "owned_by")); + } + + private String text(JsonNode node, String fieldName) { + JsonNode field = node.path(fieldName); + return field.isMissingNode() || field.isNull() ? "" : field.asText(""); + } + + private String defaultText(JsonNode node, String fieldName, String defaultValue) { + String value = text(node, fieldName); + return value.isBlank() ? defaultValue : value; + } + + private long longValue(JsonNode node) { + if (node == null || node.isMissingNode() || node.isNull()) { + return 0L; + } + + if (node.isIntegralNumber()) { + return node.longValue(); + } + + if (node.isTextual()) { + String value = node.asText().trim(); + if (value.isEmpty()) { + return 0L; + } + try { + return Long.parseLong(value); + } + catch (NumberFormatException ignored) { + return 0L; + } + } + + return node.asLong(0L); + } + + private List textList(JsonNode arrayNode) { + if (!arrayNode.isArray()) { + return List.of(); + } + + List values = new ArrayList<>(); + for (JsonNode node : arrayNode) { + String value = node.asText("").trim(); + if (!value.isEmpty()) { + values.add(value); + } + } + return List.copyOf(values); + } + + private String humanizeParameterCount(long parameterCount) { + if (parameterCount <= 0) { + return ""; + } + + double billions = parameterCount / 1_000_000_000d; + if (billions >= 1d) { + return String.format(Locale.ROOT, "%.1fB", billions); + } + + double millions = parameterCount / 1_000_000d; + return String.format(Locale.ROOT, "%.1fM", millions); + } + + private String inferFormat(String modelName) { + String lowerCaseName = modelName.toLowerCase(Locale.ROOT); + return lowerCaseName.endsWith(".gguf") ? "gguf" : ""; + } + + private String inferQuantization(String modelName) { + String upperCaseName = modelName.toUpperCase(Locale.ROOT); + int quantizationIndex = upperCaseName.indexOf("Q"); + if (quantizationIndex < 0) { + return ""; + } + + StringBuilder quantization = new StringBuilder(); + for (int index = quantizationIndex; index < upperCaseName.length(); index++) { + char currentCharacter = upperCaseName.charAt(index); + if (Character.isLetterOrDigit(currentCharacter) || currentCharacter == '_') { + quantization.append(currentCharacter); + continue; + } + break; + } + + return quantization.toString(); + } + + private String inferFamily(String modelName) { + String normalizedName = modelName.toLowerCase(Locale.ROOT) + .replace(":latest", "") + .replace(".gguf", ""); + + if (normalizedName.startsWith("gemma-")) { + return normalizedName.replaceFirst("^gemma-(\\d+).*$", "gemma$1"); + } + + if (normalizedName.startsWith("llama")) { + return "llama"; + } + + if (normalizedName.startsWith("mistral")) { + return "mistral"; + } + + if (normalizedName.startsWith("qwen")) { + return normalizedName.replaceFirst("^([a-z]+\\d*).*$", "$1"); + } + + return normalizedName.replaceFirst("^([a-z]+[0-9]*).*$", "$1"); + } + + private String upperCase(String value) { + return value.isBlank() ? value : value.toUpperCase(Locale.ROOT); + } + + record LlamaServerSnapshot(JsonNode props, JsonNode slots, List models) { + } + + record LlamaModelDescriptor(String name, String model, String modifiedAt, long size, String digest, + LlamaModelDetails details, List capabilities, JsonNode meta, String ownedBy) { + + LlamaModelDescriptor { + meta = meta == null ? MissingNode.getInstance() : meta.deepCopy(); + } + } + + record LlamaModelDetails(String parentModel, String format, String family, List families, + String parameterSize, String quantizationLevel) { + + LlamaModelDetails { + families = families == null ? List.of() : List.copyOf(families); + } + } + +} \ No newline at end of file diff --git a/src/main/java/de/extio/ollama_oai_bridge/OllamaDiscoveryController.java b/src/main/java/de/extio/ollama_oai_bridge/OllamaDiscoveryController.java new file mode 100644 index 0000000..cf1a98a --- /dev/null +++ b/src/main/java/de/extio/ollama_oai_bridge/OllamaDiscoveryController.java @@ -0,0 +1,41 @@ +package de.extio.ollama_oai_bridge; + +import tools.jackson.databind.node.ObjectNode; + +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class OllamaDiscoveryController { + + private final BridgeProperties bridgeProperties; + + private final OllamaDiscoveryService ollamaDiscoveryService; + + public OllamaDiscoveryController(BridgeProperties bridgeProperties, OllamaDiscoveryService ollamaDiscoveryService) { + this.bridgeProperties = bridgeProperties; + this.ollamaDiscoveryService = ollamaDiscoveryService; + } + + @GetMapping(path = "/api/version", produces = MediaType.APPLICATION_JSON_VALUE) + ObjectNode version() { + return this.ollamaDiscoveryService.version(this.bridgeProperties.getVersion()); + } + + @GetMapping(path = "/api/tags", produces = MediaType.APPLICATION_JSON_VALUE) + ObjectNode tags() { + return this.ollamaDiscoveryService.tags(); + } + + @PostMapping(path = "/api/show", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) + ObjectNode show(@RequestBody ShowModelRequest showModelRequest) { + return this.ollamaDiscoveryService.show(showModelRequest.model()); + } + + record ShowModelRequest(String model, Boolean verbose) { + } + +} \ No newline at end of file diff --git a/src/main/java/de/extio/ollama_oai_bridge/OllamaDiscoveryService.java b/src/main/java/de/extio/ollama_oai_bridge/OllamaDiscoveryService.java new file mode 100644 index 0000000..ce08aff --- /dev/null +++ b/src/main/java/de/extio/ollama_oai_bridge/OllamaDiscoveryService.java @@ -0,0 +1,393 @@ +package de.extio.ollama_oai_bridge; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.node.ArrayNode; +import tools.jackson.databind.node.ObjectNode; + +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; + +import de.extio.ollama_oai_bridge.LlamaServerClient.LlamaModelDescriptor; +import de.extio.ollama_oai_bridge.LlamaServerClient.LlamaModelDetails; +import de.extio.ollama_oai_bridge.LlamaServerClient.LlamaServerSnapshot; + +@Service +public class OllamaDiscoveryService { + + private final LlamaServerClient llamaServerClient; + + private final ObjectMapper objectMapper; + + public OllamaDiscoveryService(LlamaServerClient llamaServerClient, ObjectMapper objectMapper) { + this.llamaServerClient = llamaServerClient; + this.objectMapper = objectMapper; + } + + public ObjectNode version(String version) { + ObjectNode response = this.objectMapper.createObjectNode(); + response.put("version", version); + return response; + } + + public ObjectNode tags() { + LlamaServerSnapshot snapshot = this.llamaServerClient.fetchSnapshot(); + ObjectNode response = this.objectMapper.createObjectNode(); + ArrayNode models = response.putArray("models"); + + for (LlamaModelDescriptor model : snapshot.models()) { + models.add(buildTagModel(model, snapshot.props())); + } + + return response; + } + + public ObjectNode show(String modelName) { + if (modelName == null || modelName.isBlank()) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "model is required"); + } + + LlamaServerSnapshot snapshot = this.llamaServerClient.fetchSnapshot(); + LlamaModelDescriptor model = snapshot.models().stream() + .filter(candidate -> matchesModel(modelName, candidate)) + .findFirst() + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, + "Model not found: " + modelName)); + + ObjectNode response = this.objectMapper.createObjectNode(); + response.put("license", ""); + response.put("modelfile", buildModelfile(model, snapshot.props(), snapshot.slots())); + response.put("parameters", buildParameters(snapshot.props())); + response.put("template", text(snapshot.props(), "chat_template")); + response.put("system", ""); + response.set("details", detailsNode(resolveDetails(model, snapshot.props()))); + response.set("model_info", buildModelInfo(model, snapshot.props(), snapshot.slots())); + response.set("capabilities", capabilitiesNode(resolveCapabilities(model, snapshot.props()))); + response.put("modified_at", defaultString(model.modifiedAt())); + return response; + } + + private boolean matchesModel(String requestedModel, LlamaModelDescriptor candidate) { + return requestedModel.equals(candidate.name()) || requestedModel.equals(candidate.model()); + } + + private ObjectNode buildTagModel(LlamaModelDescriptor model, JsonNode props) { + LlamaModelDetails details = resolveDetails(model, props); + ObjectNode response = this.objectMapper.createObjectNode(); + response.put("name", model.name()); + response.put("model", model.model()); + response.put("modified_at", defaultString(model.modifiedAt())); + response.put("size", model.size()); + response.put("digest", defaultString(model.digest())); + response.set("details", detailsNode(details)); + return response; + } + + private ObjectNode detailsNode(LlamaModelDetails details) { + ObjectNode detailsNode = this.objectMapper.createObjectNode(); + detailsNode.put("parent_model", defaultString(details.parentModel())); + detailsNode.put("format", defaultString(details.format())); + detailsNode.put("family", defaultString(details.family())); + + ArrayNode familiesNode = detailsNode.putArray("families"); + for (String family : details.families()) { + familiesNode.add(family); + } + + detailsNode.put("parameter_size", defaultString(details.parameterSize())); + detailsNode.put("quantization_level", defaultString(details.quantizationLevel())); + return detailsNode; + } + + private ArrayNode capabilitiesNode(List capabilities) { + ArrayNode capabilitiesNode = this.objectMapper.createArrayNode(); + for (String capability : capabilities) { + capabilitiesNode.add(capability); + } + return capabilitiesNode; + } + + private LlamaModelDetails resolveDetails(LlamaModelDescriptor model, JsonNode props) { + String family = firstNonBlank(model.details().family(), inferFamily(text(props, "model_alias")), inferFamily(model.name())); + List families = model.details().families().isEmpty() && !family.isBlank() + ? List.of(family) + : model.details().families(); + + return new LlamaModelDetails( + model.details().parentModel(), + firstNonBlank(model.details().format(), inferFormat(text(props, "model_path")), inferFormat(model.name())), + family, + families, + firstNonBlank(model.details().parameterSize(), humanizeParameterCount(longValue(model.meta(), "n_params"))), + firstNonBlank(model.details().quantizationLevel(), inferQuantization(text(props, "model_alias")), inferQuantization(model.name()))); + } + + private List resolveCapabilities(LlamaModelDescriptor model, JsonNode props) { + LinkedHashSet capabilities = new LinkedHashSet<>(model.capabilities()); + if (capabilities.isEmpty()) { + capabilities.add("completion"); + } + + if (props.path("modalities").path("vision").asBoolean(false)) { + capabilities.add("vision"); + } + + String chatTemplate = text(props, "chat_template").toLowerCase(Locale.ROOT); + if (chatTemplate.contains("tool") || chatTemplate.contains("function")) { + capabilities.add("tools"); + } + if (chatTemplate.contains("reasoning") || chatTemplate.contains("thinking")) { + capabilities.add("thinking"); + } + + return List.copyOf(capabilities); + } + + private String buildModelfile(LlamaModelDescriptor model, JsonNode props, JsonNode slots) { + StringBuilder modelfile = new StringBuilder("# Modelfile generated by ollama-oai-bridge\n"); + modelfile.append("FROM ") + .append(firstNonBlank(text(props, "model_path"), model.model(), model.name())) + .append('\n'); + + String template = text(props, "chat_template"); + if (!template.isBlank()) { + modelfile.append("TEMPLATE \"\"\"") + .append(template) + .append("\"\"\"\n"); + } + + appendNumericModelfileParameter(modelfile, "num_ctx", contextLength(model, props, slots)); + appendTextModelfileParameter(modelfile, "temperature", text(props.path("default_generation_settings").path("params"), "temperature")); + appendTextModelfileParameter(modelfile, "top_k", text(props.path("default_generation_settings").path("params"), "top_k")); + appendTextModelfileParameter(modelfile, "top_p", text(props.path("default_generation_settings").path("params"), "top_p")); + appendTextModelfileParameter(modelfile, "min_p", text(props.path("default_generation_settings").path("params"), "min_p")); + appendTextModelfileParameter(modelfile, "repeat_last_n", text(props.path("default_generation_settings").path("params"), "repeat_last_n")); + appendTextModelfileParameter(modelfile, "repeat_penalty", text(props.path("default_generation_settings").path("params"), "repeat_penalty")); + appendTextModelfileParameter(modelfile, "presence_penalty", text(props.path("default_generation_settings").path("params"), "presence_penalty")); + appendTextModelfileParameter(modelfile, "frequency_penalty", text(props.path("default_generation_settings").path("params"), "frequency_penalty")); + appendQuotedModelfileParameter(modelfile, "stop", text(props, "eos_token")); + return modelfile.toString().trim(); + } + + private String buildParameters(JsonNode props) { + StringBuilder parameters = new StringBuilder(); + appendParameterLine(parameters, "num_ctx", text(props.path("default_generation_settings"), "n_ctx")); + appendParameterLine(parameters, "temperature", text(props.path("default_generation_settings").path("params"), "temperature")); + appendParameterLine(parameters, "top_k", text(props.path("default_generation_settings").path("params"), "top_k")); + appendParameterLine(parameters, "top_p", text(props.path("default_generation_settings").path("params"), "top_p")); + appendParameterLine(parameters, "min_p", text(props.path("default_generation_settings").path("params"), "min_p")); + appendParameterLine(parameters, "repeat_last_n", text(props.path("default_generation_settings").path("params"), "repeat_last_n")); + appendParameterLine(parameters, "repeat_penalty", text(props.path("default_generation_settings").path("params"), "repeat_penalty")); + appendParameterLine(parameters, "presence_penalty", text(props.path("default_generation_settings").path("params"), "presence_penalty")); + appendParameterLine(parameters, "frequency_penalty", text(props.path("default_generation_settings").path("params"), "frequency_penalty")); + appendParameterLine(parameters, "stop", quoted(text(props, "eos_token"))); + return parameters.toString().trim(); + } + + private ObjectNode buildModelInfo(LlamaModelDescriptor model, JsonNode props, JsonNode slots) { + LlamaModelDetails details = resolveDetails(model, props); + ObjectNode modelInfo = this.objectMapper.createObjectNode(); + + String architecture = firstNonBlank(details.family(), "unknown"); + modelInfo.put("general.architecture", architecture); + modelInfo.put("general.basename", stripExtension(stripTag(model.name()))); + putLongIfPositive(modelInfo, "general.parameter_count", longValue(model.meta(), "n_params")); + putStringIfPresent(modelInfo, "general.size_label", details.parameterSize()); + putStringIfPresent(modelInfo, "general.quantization", details.quantizationLevel()); + putStringIfPresent(modelInfo, architecture + ".context_length", String.valueOf(contextLength(model, props, slots))); + putLongIfPositive(modelInfo, architecture + ".embedding_length", longValue(model.meta(), "n_embd")); + putLongIfPositive(modelInfo, architecture + ".vocab_size", longValue(model.meta(), "n_vocab")); + putLongIfPositive(modelInfo, architecture + ".trained_context_length", longValue(model.meta(), "n_ctx_train")); + putStringIfPresent(modelInfo, "tokenizer.ggml.bos_token", text(props, "bos_token")); + putStringIfPresent(modelInfo, "tokenizer.ggml.eos_token", text(props, "eos_token")); + putStringIfPresent(modelInfo, "bridge.build_info", text(props, "build_info")); + modelInfo.put("bridge.total_slots", slots.isArray() ? slots.size() : 0); + modelInfo.put("bridge.busy_slots", busySlots(slots)); + putStringIfPresent(modelInfo, "bridge.owned_by", model.ownedBy()); + return modelInfo; + } + + private int busySlots(JsonNode slots) { + if (!slots.isArray()) { + return 0; + } + + int busySlots = 0; + for (JsonNode slot : slots) { + if (slot.path("is_processing").asBoolean(false)) { + busySlots++; + } + } + return busySlots; + } + + private long contextLength(LlamaModelDescriptor model, JsonNode props, JsonNode slots) { + long metaContext = longValue(model.meta(), "n_ctx"); + if (metaContext > 0) { + return metaContext; + } + + long configuredContext = longValue(props.path("default_generation_settings"), "n_ctx"); + if (configuredContext > 0) { + return configuredContext; + } + + if (slots.isArray() && !slots.isEmpty()) { + return longValue(slots.get(0), "n_ctx"); + } + + return 0L; + } + + private void appendNumericModelfileParameter(StringBuilder modelfile, String key, long value) { + if (value > 0) { + modelfile.append("PARAMETER ").append(key).append(' ').append(value).append('\n'); + } + } + + private void appendTextModelfileParameter(StringBuilder modelfile, String key, String value) { + if (!value.isBlank()) { + modelfile.append("PARAMETER ").append(key).append(' ').append(value).append('\n'); + } + } + + private void appendQuotedModelfileParameter(StringBuilder modelfile, String key, String value) { + if (!value.isBlank()) { + modelfile.append("PARAMETER ").append(key).append(' ').append(quoted(value)).append('\n'); + } + } + + private void appendParameterLine(StringBuilder parameters, String key, String value) { + if (!value.isBlank()) { + parameters.append(key).append(' ').append(value).append('\n'); + } + } + + private void putLongIfPositive(ObjectNode node, String key, long value) { + if (value > 0) { + node.put(key, value); + } + } + + private void putStringIfPresent(ObjectNode node, String key, String value) { + if (!value.isBlank()) { + node.put(key, value); + } + } + + private String text(JsonNode node, String fieldName) { + JsonNode field = node.path(fieldName); + return field.isMissingNode() || field.isNull() ? "" : field.asText(""); + } + + private long longValue(JsonNode node, String fieldName) { + JsonNode field = node.path(fieldName); + if (field.isMissingNode() || field.isNull()) { + return 0L; + } + if (field.isIntegralNumber()) { + return field.longValue(); + } + if (field.isTextual()) { + try { + return Long.parseLong(field.asText().trim()); + } + catch (NumberFormatException ignored) { + return 0L; + } + } + return field.asLong(0L); + } + + private String humanizeParameterCount(long parameterCount) { + if (parameterCount <= 0) { + return ""; + } + double billions = parameterCount / 1_000_000_000d; + if (billions >= 1d) { + return String.format(Locale.ROOT, "%.1fB", billions); + } + double millions = parameterCount / 1_000_000d; + return String.format(Locale.ROOT, "%.1fM", millions); + } + + private String inferFormat(String value) { + String lowerCaseValue = value.toLowerCase(Locale.ROOT); + return lowerCaseValue.endsWith(".gguf") ? "gguf" : ""; + } + + private String inferQuantization(String value) { + String upperCaseValue = value.toUpperCase(Locale.ROOT); + int quantizationStart = upperCaseValue.indexOf('Q'); + if (quantizationStart < 0) { + return ""; + } + + StringBuilder quantization = new StringBuilder(); + for (int index = quantizationStart; index < upperCaseValue.length(); index++) { + char character = upperCaseValue.charAt(index); + if (Character.isLetterOrDigit(character) || character == '_') { + quantization.append(character); + continue; + } + break; + } + return quantization.toString(); + } + + private String inferFamily(String value) { + String normalizedValue = stripExtension(stripTag(value.toLowerCase(Locale.ROOT))); + if (normalizedValue.startsWith("gemma-")) { + return normalizedValue.replaceFirst("^gemma-(\\d+).*$", "gemma$1"); + } + if (normalizedValue.startsWith("llama")) { + return "llama"; + } + if (normalizedValue.startsWith("mistral")) { + return "mistral"; + } + if (normalizedValue.startsWith("qwen")) { + return normalizedValue.replaceFirst("^([a-z]+\\d*).*$", "$1"); + } + return normalizedValue.replaceFirst("^([a-z]+[0-9]*).*$", "$1"); + } + + private String stripTag(String value) { + int tagSeparator = value.indexOf(':'); + return tagSeparator >= 0 ? value.substring(0, tagSeparator) : value; + } + + private String stripExtension(String value) { + return value.endsWith(".gguf") ? value.substring(0, value.length() - 5) : value; + } + + private String quoted(String value) { + return '"' + value.replace("\\", "\\\\").replace("\"", "\\\"") + '"'; + } + + private String firstNonBlank(String... values) { + if (values == null) { + return ""; + } + + for (String value : values) { + if (value != null && !value.trim().isEmpty()) { + return value.trim(); + } + } + + return ""; + } + + private String defaultString(String value) { + return value == null ? "" : value; + } + +} \ No newline at end of file diff --git a/src/main/java/de/extio/ollama_oai_bridge/OllamaOaiBridgeApplication.java b/src/main/java/de/extio/ollama_oai_bridge/OllamaOaiBridgeApplication.java new file mode 100644 index 0000000..fb7f15a --- /dev/null +++ b/src/main/java/de/extio/ollama_oai_bridge/OllamaOaiBridgeApplication.java @@ -0,0 +1,13 @@ +package de.extio.ollama_oai_bridge; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class OllamaOaiBridgeApplication { + + public static void main(String[] args) { + SpringApplication.run(OllamaOaiBridgeApplication.class, args); + } + +} diff --git a/src/main/java/de/extio/ollama_oai_bridge/RequestBodyCachingFilter.java b/src/main/java/de/extio/ollama_oai_bridge/RequestBodyCachingFilter.java new file mode 100644 index 0000000..a033d12 --- /dev/null +++ b/src/main/java/de/extio/ollama_oai_bridge/RequestBodyCachingFilter.java @@ -0,0 +1,28 @@ +package de.extio.ollama_oai_bridge; + +import java.io.IOException; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; +import org.springframework.web.util.ContentCachingRequestWrapper; + +@Component +@Order(Ordered.HIGHEST_PRECEDENCE) +public class RequestBodyCachingFilter extends OncePerRequestFilter { + + private static final int CACHE_LIMIT = 1024 * 1024; + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) + throws ServletException, IOException { + filterChain.doFilter(new ContentCachingRequestWrapper(request, CACHE_LIMIT), response); + } + +} \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..f8962fc --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,5 @@ +spring.application.name=ollama-oai-bridge +server.port=11434 + +bridge.upstream-base-url=http://localhost:5001 +bridge.version=0.24.0 diff --git a/src/test/java/de/extio/ollama_oai_bridge/OllamaOaiBridgeApplicationTests.java b/src/test/java/de/extio/ollama_oai_bridge/OllamaOaiBridgeApplicationTests.java new file mode 100644 index 0000000..b96e85a --- /dev/null +++ b/src/test/java/de/extio/ollama_oai_bridge/OllamaOaiBridgeApplicationTests.java @@ -0,0 +1,226 @@ +package de.extio.ollama_oai_bridge; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class OllamaOaiBridgeApplicationTests { + + private static final StubLlamaServer LLAMA_SERVER = StubLlamaServer.start(); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final HttpClient httpClient = HttpClient.newHttpClient(); + + @LocalServerPort + private int port; + + @DynamicPropertySource + static void registerProperties(DynamicPropertyRegistry registry) { + registry.add("bridge.upstream-base-url", () -> "http://localhost:" + LLAMA_SERVER.port()); + } + + @BeforeEach + void resetStubServer() { + LLAMA_SERVER.reset(); + } + + @AfterAll + static void stopStubServer() { + LLAMA_SERVER.stop(); + } + + @Test + void contextLoads() { + } + + @Test + void versionIsServedLocally() throws Exception { + JsonNode response = getJson("/api/version"); + + assertThat(response.path("version").asText()).isEqualTo("0.24.0"); + assertThat(LLAMA_SERVER.requests()).isEmpty(); + } + + @Test + void tagsMapsLlamaServerModels() throws Exception { + stubCommonUpstreamResponses(DATA_MODELS_RESPONSE); + + JsonNode response = getJson("/api/tags"); + + assertThat(response.path("models")).hasSize(1); + JsonNode model = response.path("models").get(0); + assertThat(model.path("name").asText()).isEqualTo("gemma-4-E4B-it-Q6_K.gguf"); + assertThat(model.path("size").asLong()).isEqualTo(7_059_101_864L); + assertThat(model.path("details").path("format").asText()).isEqualTo("gguf"); + assertThat(model.path("details").path("family").asText()).isEqualTo("gemma4"); + assertThat(model.path("details").path("parameter_size").asText()).isEqualTo("7.5B"); + assertThat(model.path("details").path("quantization_level").asText()).isEqualTo("Q6_K"); + assertThat(LLAMA_SERVER.requests()).containsExactly("GET /props", "GET /slots", "GET /v1/models"); + } + + @Test + void showMapsToStrictOllamaLikeResponse() throws Exception { + stubCommonUpstreamResponses(OLLAMA_STYLE_MODELS_RESPONSE); + + JsonNode response = postJson("/api/show", """ + {"model":"gemma-4-26B-A4B-it-UD-Q4_K_M:latest"} + """); + + assertThat(response.path("modelfile").asText()).contains("FROM /mnt/data/data/lm/models/gguf/s/gemma-4-E4B-it-Q6_K.gguf"); + assertThat(response.path("template").asText()).contains("tool_call"); + assertThat(response.path("details").path("quantization_level").asText()).isEqualTo("Q4_K_M"); + assertThat(response.path("capabilities")).extracting(JsonNode::asText) + .contains("completion", "tools", "thinking"); + assertThat(response.path("model_info").path("bridge.total_slots").asInt()).isEqualTo(4); + assertThat(response.path("model_info").path("bridge.busy_slots").asInt()).isEqualTo(1); + } + + @Test + void nonOverriddenRoutesAreProxied() throws Exception { + LLAMA_SERVER.stubJson("GET", "/props", PROPS_RESPONSE); + + HttpResponse response = send(HttpRequest.newBuilder(uri("/props")).GET().build()); + + assertThat(response.statusCode()).isEqualTo(200); + assertThat(response.body()).isEqualTo(PROPS_RESPONSE); + assertThat(LLAMA_SERVER.requests()).containsExactly("GET /props"); + } + + private void stubCommonUpstreamResponses(String modelsResponse) { + LLAMA_SERVER.stubJson("GET", "/props", PROPS_RESPONSE); + LLAMA_SERVER.stubJson("GET", "/slots", SLOTS_RESPONSE); + LLAMA_SERVER.stubJson("GET", "/v1/models", modelsResponse); + } + + private JsonNode getJson(String path) throws Exception { + HttpResponse response = send(HttpRequest.newBuilder(uri(path)).GET().build()); + assertThat(response.statusCode()).isEqualTo(200); + return OBJECT_MAPPER.readTree(response.body()); + } + + private JsonNode postJson(String path, String body) throws Exception { + HttpRequest request = HttpRequest.newBuilder(uri(path)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(body)) + .build(); + + HttpResponse response = send(request); + assertThat(response.statusCode()).isEqualTo(200); + return OBJECT_MAPPER.readTree(response.body()); + } + + private HttpResponse send(HttpRequest request) throws IOException, InterruptedException { + return this.httpClient.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + } + + private URI uri(String path) { + return URI.create("http://localhost:" + this.port + path); + } + + private static final String PROPS_RESPONSE = """ + {"default_generation_settings":{"params":{"temperature":1.0,"top_k":64,"top_p":0.95,"min_p":0.05,"repeat_last_n":64,"repeat_penalty":1.0,"presence_penalty":0.0,"frequency_penalty":0.0},"n_ctx":16128},"total_slots":4,"model_alias":"gemma-4-E4B-it-Q6_K.gguf","model_path":"/mnt/data/data/lm/models/gguf/s/gemma-4-E4B-it-Q6_K.gguf","modalities":{"vision":false,"audio":false},"bos_token":"","eos_token":"","build_info":"b9164-d5dc2e0a0","chat_template":"template with tool_call and reasoning support"} + """; + + private static final String SLOTS_RESPONSE = """ + [{"id":0,"n_ctx":16128,"speculative":false,"is_processing":false},{"id":1,"n_ctx":16128,"speculative":false,"is_processing":true},{"id":2,"n_ctx":16128,"speculative":false,"is_processing":false},{"id":3,"n_ctx":16128,"speculative":false,"is_processing":false}] + """; + + private static final String DATA_MODELS_RESPONSE = """ + {"object":"list","data":[{"id":"gemma-4-E4B-it-Q6_K.gguf","aliases":[],"tags":[],"object":"model","created":1778857538,"owned_by":"llamacpp","meta":{"vocab_type":2,"n_vocab":262144,"n_ctx":16128,"n_ctx_train":131072,"n_embd":2560,"n_params":7518069290,"size":7059101864}}]} + """; + + private static final String OLLAMA_STYLE_MODELS_RESPONSE = """ + {"models":[{"name":"gemma-4-26B-A4B-it-UD-Q4_K_M:latest","model":"gemma-4-26B-A4B-it-UD-Q4_K_M:latest","modified_at":"2026-05-15T12:43:43.896860597+02:00","size":16868241115,"digest":"b16ef1d08ff9df45d8d95d352be6b0578785d9ab2d7e2a78a14151292b6093a4","capabilities":["completion"],"details":{"parent_model":"","format":"gguf","family":"gemma4","families":["gemma4"],"parameter_size":"25.2B","quantization_level":"Q4_K_M"}}]} + """; + + private static final class StubLlamaServer { + + private final HttpServer httpServer; + + private final Map responses = new ConcurrentHashMap<>(); + + private final List requests = new CopyOnWriteArrayList<>(); + + private StubLlamaServer(HttpServer httpServer) { + this.httpServer = httpServer; + this.httpServer.createContext("/", this::handle); + this.httpServer.start(); + } + + static StubLlamaServer start() { + try { + return new StubLlamaServer(HttpServer.create(new InetSocketAddress(0), 0)); + } + catch (IOException exception) { + throw new IllegalStateException("Failed to start stub llama-server", exception); + } + } + + int port() { + return this.httpServer.getAddress().getPort(); + } + + void reset() { + this.responses.clear(); + this.requests.clear(); + } + + void stop() { + this.httpServer.stop(0); + } + + void stubJson(String method, String path, String body) { + this.responses.put(key(method, path), new StubResponse(200, body)); + } + + List requests() { + return List.copyOf(this.requests); + } + + private void handle(HttpExchange exchange) throws IOException { + String key = key(exchange.getRequestMethod(), exchange.getRequestURI().getPath()); + this.requests.add(key); + + StubResponse response = this.responses.getOrDefault(key, new StubResponse(404, "{}")); + byte[] body = response.body().getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(response.status(), body.length); + exchange.getResponseBody().write(body); + exchange.close(); + } + + private static String key(String method, String path) { + return method + " " + path; + } + + record StubResponse(int status, String body) { + } + + } + +} diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..2ff34b6 --- /dev/null +++ b/start.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +jar_file="$script_dir/ollama-oai-bridge.jar" + +if [[ ! -f "$jar_file" ]]; then + echo "Jar not found: $jar_file" >&2 + exit 1 +fi + +java_opts=() +if [[ -n "${JAVA_OPTS:-}" ]]; then + read -r -a java_opts <<<"${JAVA_OPTS}" +fi + +app_opts=() +if [[ -n "${OLLAMA_OAI_BRIDGE_OPTS:-}" ]]; then + read -r -a app_opts <<<"${OLLAMA_OAI_BRIDGE_OPTS}" +fi + +exec java "${java_opts[@]}" "${app_opts[@]}" -jar "$jar_file" "$@" \ No newline at end of file