exo/game-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(storage): Implemented StorageService

Browse Source 
- Implemented methods for storing and loading objects and streams in StorageServiceImpl.
- Added support for saving and loading streams using InputStream and OutputStream.
- Enhanced ObjectSerialization to support serialization directly to OutputStream.
- Updated StorageIndex to support recursive listing of items.
- Added comprehensive unit tests for new functionalities in StorageServiceImpl and ObjectSerialization.
- Improved error handling and logging for storage operations.
- Updated AutoConfiguration to include StorageServiceAutoConfiguration.
This commit is contained in:
extio-de
2025-12-22 15:36:53 +01:00
parent 65d3775a04
commit b5cd3fa1fe
10 changed files with 1026 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/main/java/de/extio/game_engine/storage/StorageIndex.java
View File

@@ -18,8 +18,9 @@ import java.nio.file.PathMatcher;
public class StorageIndex {
private final Node root;
private final Map<UUID, IndexedItem> byId;
public StorageIndex() {
this.root = new Node();
this.byId = new HashMap<>();
@@ -33,6 +34,14 @@ public class StorageIndex {
return this.byId.isEmpty();
}
public List<StorageItemDescriptor> listAll() {
List<StorageItemDescriptor> result = new ArrayList<>(this.byId.size());
for (IndexedItem item : this.byId.values()) {
result.add(item.descriptor());
}
return result;
}
public void clear() {
this.root.clear();
this.byId.clear();
@@ -64,16 +73,29 @@ public class StorageIndex {
return Optional.ofNullable(node.itemsByNormalizedName.get(normalizedName)).map(IndexedItem::descriptor);
}
public List<StorageItemDescriptor> list(final List<String> path) {
public List<StorageItemDescriptor> list(final List<String> path, final boolean recursive) {
final List<String> normalizedPath = this.normalizePath(path);
final Node node = this.getNode(normalizedPath);
if (node == null) {
return List.of();
}
if (node.itemsByNormalizedName.isEmpty()) {
return List.of();
if (!recursive) {
if (node.itemsByNormalizedName.isEmpty()) {
return List.of();
}
return node.itemsByNormalizedName.values().stream().map(IndexedItem::descriptor).toList();
}
return node.itemsByNormalizedName.values().stream().map(IndexedItem::descriptor).toList();
final List<StorageItemDescriptor> items = new ArrayList<>();
final Deque<Node> work = new ArrayDeque<>();
work.addLast(node);
while (!work.isEmpty()) {
final Node n = work.removeFirst();
for (final IndexedItem item : n.itemsByNormalizedName.values()) {
items.add(item.descriptor());
}
work.addAll(n.children.values());
}
return List.copyOf(items);
}
public List<StorageItemDescriptor> findByFilenamePattern(final List<String> basePath, final String pattern, final boolean recursive) {
@@ -256,7 +278,9 @@ public class StorageIndex {
}
private static final class Node {
private final Map<String, Node> children;
private final Map<String, IndexedItem> itemsByNormalizedName;
private Node() {

11
src/main/java/de/extio/game_engine/storage/StorageOutputStream.java Normal file
View File

@@ -0,0 +1,11 @@
package de.extio.game_engine.storage;
import java.io.OutputStream;
import java.util.UUID;
/**
* Wrapper that provides access to an OutputStream for storing data while
* also returning the storage UUID associated with the data.
*/
public record StorageOutputStream(UUID id, OutputStream outputStream) {
}

42
src/main/java/de/extio/game_engine/storage/StorageService.java Normal file
View File

@@ -0,0 +1,42 @@
package de.extio.game_engine.storage;
import java.io.InputStream;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
public interface StorageService {
boolean deleteById(UUID id);
boolean deleteByPath(List<String> path, String name);
List<StorageItemDescriptor> listAll();
List<StorageItemDescriptor> listPath(List<String> path, boolean recursive);
<T> Optional<T> loadById(Class<T> clazz, UUID id);
<T> Optional<T> loadByPath(Class<T> clazz, List<String> path, String name);
Optional<InputStream> loadStreamById(UUID id);
Optional<InputStream> loadStreamByPath(List<String> path, String name);
boolean moveById(UUID id, List<String> newPath, String newName);
boolean moveByPath(List<String> oldPath, String oldName, List<String> newPath, String newName);
Optional<StorageItemDescriptor> searchById(UUID id);
Optional<StorageItemDescriptor> searchByPath(List<String> path, String name);
List<StorageItemDescriptor> searchByPattern(List<String> basePath, String pattern, boolean recursive);
UUID store(String name, List<String> path, Object obj);
UUID storeStream(String name, List<String> path, InputStream inputStream);
StorageOutputStream storeStream(String name, List<String> path);
}

15
src/main/java/de/extio/game_engine/storage/StorageServiceAutoConfiguration.java Normal file
View File

@@ -0,0 +1,15 @@
package de.extio.game_engine.storage;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
@AutoConfiguration
@ConditionalOnProperty(name = "game-engine.storage.enabled", havingValue = "true", matchIfMissing = true)
public class StorageServiceAutoConfiguration {
@Bean
StorageService storageService() {
return new StorageServiceImpl();
}
}

303
src/main/java/de/extio/game_engine/storage/StorageServiceImpl.java
View File

@@ -1,17 +1,25 @@
package de.extio.game_engine.storage;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import de.extio.game_engine.util.FastRandomUUID;
import de.extio.game_engine.util.ObjectSerialization;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class StorageServiceImpl {
public class StorageServiceImpl implements StorageService {
private static final Logger LOGGER = LoggerFactory.getLogger(StorageServiceImpl.class);
@@ -83,7 +91,7 @@ public class StorageServiceImpl {
final StorageItemDescriptor[] descriptors;
try (final var in = Files.newInputStream(INDEX_LOCATION)) {
descriptors = ObjectSerialization.deserialize(StorageItemDescriptor[].class, in, false, false, null, null, null);
descriptors = ObjectSerialization.deserialize(StorageItemDescriptor[].class, in, true, false, null, null, null);
}
if (descriptors == null || descriptors.length == 0) {
return;
@@ -103,4 +111,295 @@ public class StorageServiceImpl {
}
}
private void saveIndex() {
try {
final StorageItemDescriptor[] descriptors = this.index.listAll().toArray(StorageItemDescriptor[]::new);
try (var outputStream = Files.newOutputStream(INDEX_LOCATION)) {
ObjectSerialization.serialize(descriptors, outputStream, true, false, false, null, null);
}
LOGGER.debug("Saved {} items to index", descriptors.length);
}
catch (final Exception e) {
LOGGER.error("Could not save index to " + INDEX_LOCATION.toString(), e);
}
}
private Path getFilePath(final UUID id) {
return DATA_LOCATION.resolve(id.toString());
}
private void addToIndexAndSave(final UUID id, final String name, final List<String> path) {
final StorageItemDescriptor descriptor = new StorageItemDescriptor(id, name, path == null ? List.of() : path);
this.index.add(descriptor);
this.saveIndex();
}
@Override
public UUID store(final String name, final List<String> path, final Object obj) {
Objects.requireNonNull(name, "name");
Objects.requireNonNull(obj, "obj");
final Optional<StorageItemDescriptor> existing = this.index.find(path, name);
final UUID id = existing.map(StorageItemDescriptor::id).orElse(FastRandomUUID.create());
final Path filePath = getFilePath(id);
try {
try (var outputStream = Files.newOutputStream(filePath)) {
ObjectSerialization.serialize(obj, outputStream, true, false, false, null, null);
}
addToIndexAndSave(id, name, path);
LOGGER.debug("Stored object with id {} at path {}/{}", id, path, name);
return id;
}
catch (final Exception e) {
LOGGER.error("Could not store object at " + filePath.toString(), e);
throw new RuntimeException("Failed to store object", e);
}
}
@Override
public UUID storeStream(final String name, final List<String> path, final InputStream inputStream) {
Objects.requireNonNull(name, "name");
Objects.requireNonNull(inputStream, "inputStream");
final Optional<StorageItemDescriptor> existing = this.index.find(path, name);
final UUID id = existing.map(StorageItemDescriptor::id).orElse(FastRandomUUID.create());
final Path filePath = getFilePath(id);
try {
Files.copy(inputStream, filePath, StandardCopyOption.REPLACE_EXISTING);
addToIndexAndSave(id, name, path);
LOGGER.debug("Stored stream with id {} at path {}/{}", id, path, name);
return id;
}
catch (final Exception e) {
LOGGER.error("Could not store stream at " + filePath.toString(), e);
throw new RuntimeException("Failed to store stream", e);
}
}
@Override
public boolean deleteById(final UUID id) {
Objects.requireNonNull(id, "id");
final boolean removed = this.index.remove(id);
if (!removed) {
return false;
}
final Path filePath = DATA_LOCATION.resolve(id.toString());
try {
Files.deleteIfExists(filePath);
this.saveIndex();
LOGGER.debug("Deleted object with id {}", id);
return true;
}
catch (final IOException e) {
LOGGER.error("Could not delete file " + filePath.toString(), e);
return false;
}
}
@Override
public boolean deleteByPath(final List<String> path, final String name) {
Objects.requireNonNull(name, "name");
final Optional<StorageItemDescriptor> descriptor = this.index.find(path, name);
if (descriptor.isEmpty()) {
return false;
}
return this.deleteById(descriptor.get().id());
}
@Override
public boolean moveById(final UUID id, final List<String> newPath, final String newName) {
Objects.requireNonNull(id, "id");
Objects.requireNonNull(newName, "newName");
final Optional<StorageItemDescriptor> existing = this.index.getById(id);
if (existing.isEmpty()) {
return false;
}
final StorageItemDescriptor newDescriptor = new StorageItemDescriptor(id, newName, newPath == null ? List.of() : newPath);
this.index.add(newDescriptor);
this.saveIndex();
LOGGER.debug("Moved object with id {} to path {}/{}", id, newPath, newName);
return true;
}
@Override
public boolean moveByPath(final List<String> oldPath, final String oldName, final List<String> newPath, final String newName) {
Objects.requireNonNull(oldName, "oldName");
Objects.requireNonNull(newName, "newName");
final Optional<StorageItemDescriptor> descriptor = this.index.find(oldPath, oldName);
if (descriptor.isEmpty()) {
return false;
}
return this.moveById(descriptor.get().id(), newPath, newName);
}
@Override
public <T> Optional<T> loadById(final Class<T> clazz, final UUID id) {
Objects.requireNonNull(clazz, "clazz");
Objects.requireNonNull(id, "id");
if (!this.index.getById(id).isPresent()) {
return Optional.empty();
}
final Path filePath = getFilePath(id);
if (!Files.exists(filePath)) {
LOGGER.warn("Index references non-existent file for id {}", id);
return Optional.empty();
}
try {
try (final var in = Files.newInputStream(filePath)) {
final T obj = ObjectSerialization.deserialize(clazz, in, true, false, null, null, null);
return Optional.ofNullable(obj);
}
}
catch (final Exception e) {
LOGGER.error("Could not load object from " + filePath.toString(), e);
return Optional.empty();
}
}
@Override
public <T> Optional<T> loadByPath(final Class<T> clazz, final List<String> path, final String name) {
Objects.requireNonNull(clazz, "clazz");
Objects.requireNonNull(name, "name");
final Optional<StorageItemDescriptor> descriptor = this.index.find(path, name);
if (descriptor.isEmpty()) {
return Optional.empty();
}
return this.loadById(clazz, descriptor.get().id());
}
@Override
public Optional<InputStream> loadStreamById(final UUID id) {
Objects.requireNonNull(id, "id");
if (!this.index.getById(id).isPresent()) {
return Optional.empty();
}
final Path filePath = getFilePath(id);
if (!Files.exists(filePath)) {
LOGGER.warn("Index references non-existent file for id {}", id);
return Optional.empty();
}
try {
return Optional.of(Files.newInputStream(filePath));
}
catch (final Exception e) {
LOGGER.error("Could not load stream from " + filePath.toString(), e);
return Optional.empty();
}
}
@Override
public Optional<InputStream> loadStreamByPath(final List<String> path, final String name) {
Objects.requireNonNull(name, "name");
final Optional<StorageItemDescriptor> descriptor = this.index.find(path, name);
if (descriptor.isEmpty()) {
return Optional.empty();
}
return this.loadStreamById(descriptor.get().id());
}
@Override
public StorageOutputStream storeStream(final String name, final List<String> path) {
Objects.requireNonNull(name, "name");
final Optional<StorageItemDescriptor> existing = this.index.find(path, name);
final UUID id = existing.map(StorageItemDescriptor::id).orElse(FastRandomUUID.create());
final Path filePath = getFilePath(id);
try {
final OutputStream outputStream = Files.newOutputStream(filePath);
return new StorageOutputStream(id, new OutputStream() {
private boolean closed = false;
@Override
public void write(int b) throws IOException {
outputStream.write(b);
}
@Override
public void write(byte[] b) throws IOException {
outputStream.write(b);
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
outputStream.write(b, off, len);
}
@Override
public void flush() throws IOException {
outputStream.flush();
}
@Override
public void close() throws IOException {
if (closed) {
return;
}
closed = true;
outputStream.close();
addToIndexAndSave(id, name, path);
LOGGER.debug("Stored stream with id {} at path {}/{}", id, path, name);
}
});
}
catch (final Exception e) {
LOGGER.error("Could not create output stream at " + filePath.toString(), e);
throw new RuntimeException("Failed to create output stream", e);
}
}
@Override
public Optional<StorageItemDescriptor> searchById(final UUID id) {
Objects.requireNonNull(id, "id");
return this.index.getById(id);
}
@Override
public Optional<StorageItemDescriptor> searchByPath(final List<String> path, final String name) {
Objects.requireNonNull(name, "name");
return this.index.find(path, name);
}
@Override
public List<StorageItemDescriptor> searchByPattern(final List<String> basePath, final String pattern, final boolean recursive) {
Objects.requireNonNull(pattern, "pattern");
return this.index.findByFilenamePattern(basePath, pattern, recursive);
}
@Override
public List<StorageItemDescriptor> listPath(final List<String> path, final boolean recursive) {
return this.index.list(path, recursive);
}
@Override
public List<StorageItemDescriptor> listAll() {
return this.index.listAll();
}
}

12
src/main/java/de/extio/game_engine/util/ObjectSerialization.java
View File

@@ -31,11 +31,16 @@ public final class ObjectSerialization {
public static byte[] serialize(final Object o, final boolean compress, final boolean base64, final boolean digest, final byte[] digestSalt, final Consumer<byte[]> digestConsumer) {
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(1 << 16);
serialize(o, byteArrayOutputStream, compress, base64, digest, digestSalt, digestConsumer);
return byteArrayOutputStream.toByteArray();
}
public static void serialize(final Object o, final OutputStream outputStream, final boolean compress, final boolean base64, final boolean digest, final byte[] digestSalt, final Consumer<byte[]> digestConsumer) {
OutputStream base64OutputStream = null;
ZstdOutputStream zstdOutputStream = null;
DigestOutputStream digestOutputStream = null;
try {
OutputStream stream = byteArrayOutputStream;
OutputStream stream = outputStream;
if (base64) {
stream = base64OutputStream = Base64.getEncoder().wrap(stream);
}
@@ -77,13 +82,10 @@ public final class ObjectSerialization {
}
}
}
return byteArrayOutputStream.toByteArray();
}
public static <T extends Object> T deserialize(final Class<T> clazz, final byte[] data, final boolean decompress, final boolean base64, final byte[] checkDigest, final byte[] digestSalt, final Consumer<Boolean> digestCheckConsumer) {
final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(data);
return deserialize(clazz, byteArrayInputStream, decompress, base64, checkDigest, digestSalt, digestCheckConsumer);
return deserialize(clazz, new ByteArrayInputStream(data), decompress, base64, checkDigest, digestSalt, digestCheckConsumer);
}
public static <T extends Object> T deserialize(final Class<T> clazz, final InputStream inputStream, final boolean decompress, final boolean base64, final byte[] checkDigest, final byte[] digestSalt, final Consumer<Boolean> digestCheckConsumer) {