--- /dev/null
+/.idea/
+/.settings/
+/target/
+/doc/apidocs/
+/*.iml
+/test.byar
+*.html
+/.classpath
+/.project
\ No newline at end of file
--- /dev/null
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+ CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+ LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+ ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+ INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+ REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+ PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+ THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+ HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+ i. the right to reproduce, adapt, distribute, perform, display,
+ communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+ likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+ subject to the limitations in paragraph 4(a), below;
+ v. rights protecting the extraction, dissemination, use and reuse of data
+ in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+ European Parliament and of the Council of 11 March 1996 on the legal
+ protection of databases, and under any national implementation
+ thereof, including any amended or successor version of such
+ directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+ world based on applicable law or treaty, and any national
+ implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+ surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+ warranties of any kind concerning the Work, express, implied,
+ statutory or otherwise, including without limitation warranties of
+ title, merchantability, fitness for a particular purpose, non
+ infringement, or the absence of latent or other defects, accuracy, or
+ the present or absence of errors, whether or not discoverable, all to
+ the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+ that may apply to the Work or any use thereof, including without
+ limitation any person's Copyright and Related Rights in the Work.
+ Further, Affirmer disclaims responsibility for obtaining any necessary
+ consents, permissions or other rights required for any use of the
+ Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+ party to this document and has no duty or obligation with respect to
+ this CC0 or use of the Work.
--- /dev/null
+#!/bin/bash
+
+#
+# This is a helper bash script that starts IntelliJ with the current project.
+# Script is written is such a way that you can simply click on it in file
+# navigator to run it.
+#
+#
+# Script assumes:
+#
+# + GNU operating system
+# + IntelliJ is installed and commandline launcher "idea" is enabled.
+#
+
+cd "${0%/*}"
+cd ..
+
+setsid idea . &>/dev/null
--- /dev/null
+#!/bin/bash
+cd "${0%/*}"; if [ "$1" != "T" ]; then gnome-terminal -e "'$0' T"; exit; fi;
+
+cd ..
+
+export_org_to_html() {
+ local org_file=$1
+ local dir=$(dirname "$org_file")
+ local base=$(basename "$org_file" .org)
+ (
+ cd "$dir" || return 1
+ local html_file="${base}.html"
+ if [ -f "$html_file" ]; then
+ rm -f "$html_file"
+ fi
+ echo "Exporting: $org_file → $dir/$html_file"
+ emacs --batch -l ~/.emacs --visit="${base}.org" --funcall=org-html-export-to-html --kill
+ if [ $? -eq 0 ]; then
+ echo "✓ Successfully exported $org_file"
+ else
+ echo "✗ Failed to export $org_file"
+ return 1
+ fi
+ )
+}
+
+export_org_files_to_html() {
+ echo "🔍 Searching for .org files in doc/ ..."
+ echo "======================================="
+
+ mapfile -t ORG_FILES < <(find doc -type f -name "*.org" | sort)
+
+ if [ ${#ORG_FILES[@]} -eq 0 ]; then
+ echo "❌ No .org files found!"
+ return 1
+ fi
+
+ echo "Found ${#ORG_FILES[@]} .org file(s):"
+ printf '%s\n' "${ORG_FILES[@]}"
+ echo "======================================="
+
+ SUCCESS_COUNT=0
+ FAILED_COUNT=0
+
+ for org_file in "${ORG_FILES[@]}"; do
+ export_org_to_html "$org_file"
+ if [ $? -eq 0 ]; then
+ ((SUCCESS_COUNT++))
+ else
+ ((FAILED_COUNT++))
+ fi
+ done
+
+ echo "======================================="
+ echo "📊 SUMMARY:"
+ echo " ✓ Successful: $SUCCESS_COUNT"
+ echo " ✗ Failed: $FAILED_COUNT"
+ echo " Total: $((SUCCESS_COUNT + FAILED_COUNT))"
+ echo ""
+}
+
+mvn clean package
+
+export_org_files_to_html
+
+echo "📤 Uploading to server..."
+rsync -avz --delete -e 'ssh -p 10006' doc/ \
+ n0@www3.svjatoslav.eu:/mnt/big/projects/aukio-data/
+
+if [ $? -eq 0 ]; then
+ echo "✓ Upload completed successfully!"
+else
+ echo "✗ Upload failed!"
+fi
+
+echo ""
+echo "Press ENTER to close this window."
+read
\ No newline at end of file
--- /dev/null
+#+SETUPFILE: ~/.emacs.d/org-styles/html/darksun.theme
+#+TITLE: Aukio Data - Data storage and computing engine
+#+LANGUAGE: en
+#+LATEX_HEADER: \usepackage[margin=1.0in]{geometry}
+#+LATEX_HEADER: \usepackage{parskip}
+#+LATEX_HEADER: \usepackage[none]{hyphenat}
+
+#+OPTIONS: H:20 num:20
+#+OPTIONS: author:nil
+
+* General
+:PROPERTIES:
+:CUSTOM_ID: general
+:END:
+
+- This program is free software: released under Creative Commons Zero
+ (CC0) license
+
+- Program author:
+ - Svjatoslav Agejenko
+ - Homepage: https://svjatoslav.eu
+ - Email: mailto://svjatoslav@svjatoslav.eu
+
+- [[https://www.svjatoslav.eu/projects/][Other software projects hosted at svjatoslav.eu]]
+
+** Source code
+:PROPERTIES:
+:CUSTOM_ID: source-code
+:END:
+- [[https://www2.svjatoslav.eu/gitweb/?p=aukio-data.git;a=snapshot;h=HEAD;sf=tgz][Download latest snapshot in TAR GZ format]]
+
+- [[https://www2.svjatoslav.eu/gitweb/?p=aukio-data.git;a=summary][Browse Git repository online]]
+
+- Clone Git repository using command:
+ : git clone https://www2.svjatoslav.eu/git/aukio-data.git
+
+- See [[https://www3.svjatoslav.eu/projects/aukio-data/apidocs/][JavaDoc]].
+
+* Vision / goal
+ :PROPERTIES:
+ :CUSTOM_ID: vision-goal
+ :ID: f6764282-a6f6-44e6-8716-b428074dd093
+ :END:
+
+Provide hackable, versioned, optimized, distributed, geometrical,
+arbitrary dimensional ([[id:96116550-a6a1-4700-bef7-865d0deee7ea][hypercube based]]) data storage and computation
+engine ([[id:d2375acc-af14-4f18-8ad0-7949501178c5][as inspired by the brain]]) for general purpose visual computing
+environment called [[http://www2.svjatoslav.eu/gitbrowse/aukio/doc/index.html][Aukio]].
+
+
+* Ideas / inspiration
+:PROPERTIES:
+:CUSTOM_ID: ideas
+:END:
+
++ [[https://en.wikipedia.org/wiki/OLAP_cube][OLAP cube]].
+
+** Brain
+ :PROPERTIES:
+ :CUSTOM_ID: brain
+ :ID: d2375acc-af14-4f18-8ad0-7949501178c5
+ :END:
+
++ Brain appears to be natural geometrical/parallel data storage and
+ computational engine:
+ + https://www.quantamagazine.org/the-brain-maps-out-ideas-and-memories-like-spaces-20190114/
+ + [[https://www.simonsfoundation.org/2021/04/07/geometrical-thinking-offers-a-window-into-computation/][Geometrical Thinking Offers a Window Into Computation]]
+
++ Even more awesome is that brain appears to operate and is wired as
+ arbitrary/variable dimensional structure:
+ https://singularityhub.com/2017/06/21/is-there-a-multidimensional-mathematical-world-hidden-in-the-brains-computation/
+
++ On top of this, this multidimensional space that brain represents
+ has dynamic/variable resolution/density:
+ + https://www.quantamagazine.org/goals-and-rewards-redraw-the-brains-map-of-the-world-20190328
+
++ Such properties allow parallel Geometrical computation and
+ beautifully fits CM-1 Connection Machine architecture (for extra
+ hardware accelerated solution).
+
+** CM-1 Connection Machine
+:PROPERTIES:
+:CUSTOM_ID: cm-1-connection-machine
+:ID: 01aa65c1-3d44-44a8-9b90-58454bc6be80
+:END:
+
+https://en.wikipedia.org/wiki/Connection_Machine
+
+Massively parallel (thousands of CPUs) connected via
+machine's internal 12-dimensional hypercube network allows to
+efficiently simulate arbitrary dimensional hypercube and network
+topology between computational units. So that when we are
+solving/simulating for example 5 dimensional problem, we can arrange
+computational units into virtual 5D network. See:
+http://www.mission-base.com/tamiko/theory/cm_txts/di-ch2.html
+
+we can pre-distribute data across computation units and perform
+parallel [[id:171fe375-c737-41e6-b429-a414f6abc5d8][geometrical computation]].
+
+
+** Integrate with lossless diamond square algorithm based multidimensional data compressor
+:PROPERTIES:
+:CUSTOM_ID: integrate-lossless-diamond-square-compressor
+:END:
+
++ Demonstrate using 3D UI for debugging algorithm internals.
+
++ Separate RGB/YUV channels into separate encoders/decoders
+
++ Make demo lossless image and sound CODECS.
+
+** Distributed computation and data storage
+ :PROPERTIES:
+ :CUSTOM_ID: distributed-computation-and-data-storage
+ :ID: 5d287158-53ea-44a2-a754-dd862366066a
+ :END:
+
+Lots of problems can be translated to geometry (use any shapes and as
+many dimensions as you need). Solution(s) to such problems could be
+then found via geometrical search/comparison/lookup results. As a
+bonus, such geometrical *data storage* AND *computation* can be
+naturally made in *parallel* and *distributed*.
+
+** The "Datacube" is a Fundamental Data Structure
+
+The world is not just Sets (SQL), Hierarchies (XML/JSON), or Graphs
+(SPARQL). *Arrays* are the fourth pillar. Treating them as "just
+another file format" or "a blob in a table" is a fundamental error
+that leads to performance collapse.
+
+* Current status
+:PROPERTIES:
+:CUSTOM_ID: current-status
+:END:
+
+- More or less defined [[id:f6764282-a6f6-44e6-8716-b428074dd093][Vision / goal]].
+
+- Collected some [[id:0fa6354b-18c9-4120-bbf5-c7239aebecab][inspiring]] [[id:171fe375-c737-41e6-b429-a414f6abc5d8][ideas]].
+
+- Implemented very simple persistent key-value map.
+ - Long term goal is to use it as a backing storage engine and
+ implement more advanced features on top of this via layered
+ architecture.
--- /dev/null
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>eu.svjatoslav</groupId>
+ <artifactId>aukio-data</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <name>Aukio data</name>
+ <description>Aukio data</description>
+
+ <properties>
+ <java.version>1.8</java.version>
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+
+ <organization>
+ <name>svjatoslav.eu</name>
+ <url>http://svjatoslav.eu</url>
+ </organization>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>eu.svjatoslav</groupId>
+ <artifactId>javainspect</artifactId>
+ <version>1.7</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.8.1</version>
+ <configuration>
+ <optimize>true</optimize>
+ <encoding>UTF-8</encoding>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>2.2.1</version>
+ <executions>
+ <execution>
+ <id>attach-sources</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>3.1.1</version>
+ <executions>
+ <execution>
+ <id>attach-javadocs</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <javadocExecutable>${java.home}/bin/javadoc</javadocExecutable>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.4.3</version>
+ <configuration>
+ <encoding>UTF-8</encoding>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ <extensions>
+ <extension>
+ <groupId>org.apache.maven.wagon</groupId>
+ <artifactId>wagon-ssh-external</artifactId>
+ <version>2.6</version>
+ </extension>
+ </extensions>
+ </build>
+
+
+ <distributionManagement>
+ <snapshotRepository>
+ <id>svjatoslav.eu</id>
+ <name>svjatoslav.eu</name>
+ <url>scpexe://svjatoslav.eu:10006/srv/maven</url>
+ </snapshotRepository>
+ <repository>
+ <id>svjatoslav.eu</id>
+ <name>svjatoslav.eu</name>
+ <url>scpexe://svjatoslav.eu:10006/srv/maven</url>
+ </repository>
+ </distributionManagement>
+
+ <repositories>
+ <repository>
+ <id>svjatoslav.eu</id>
+ <name>Svjatoslav repository</name>
+ <url>https://www3.svjatoslav.eu/maven/</url>
+ </repository>
+ </repositories>
+
+ <scm>
+ <connection>scm:git:ssh://n0@svjatoslav.eu/home/git/repositories/aukio-data.git</connection>
+ <developerConnection>scm:git:ssh://n0@svjatoslav.eu/home/git/repositories/aukio-data.git
+ </developerConnection>
+ </scm>
+
+
+</project>
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.model;
+
+public class ArrayStorage {
+
+ // load 16bit integer from byte array
+ public static int load16bit(final byte[] array, final int addr) {
+ // return (int)array[addr] + (int)array[addr+1]*256;
+ return ((array[addr + 1] & 0xFF) << 8) | (array[addr] & 0xFF);
+ }
+
+ // load 16bit integer from byte array
+ public static int load32bit(final byte[] array, final int addr) {
+ return ((array[addr + 3] & 0xFF) << 24)
+ | ((array[addr + 2] & 0xFF) << 16)
+ | ((array[addr + 1] & 0xFF) << 8) | (array[addr] & 0xFF);
+ }
+
+ // store 16bit integer into byte array
+ public static void store16bit(final byte[] array, final int addr,
+ final int value) {
+ final int byteHigh = value / 256;
+ final int byteLow = value - (byteHigh * 256);
+ array[addr] = (byte) byteLow;
+ array[addr + 1] = (byte) byteHigh;
+ }
+
+}
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.model;
+
+public class EntryClass {
+
+ /**
+ * <pre>
+ * Datastore entries are instances of particular class.
+ *
+ * Class declares:
+ * data fields
+ *
+ * methods
+ * method is a tree like (code branches) data structure
+ * of commands and user comments.
+ *
+ * Class in actually a declaration of an multidimensional array.
+ * With every field acting as a dimension.
+ *
+ *
+ * Field properties:
+ * * mandatory.
+ *
+ * * data type:
+ * Instance <of Type>
+ *
+ * enumeration
+ *
+ * float
+ * integer
+ * string
+ * byte[]
+ *
+ * Collection <of Type>
+ *
+ * </pre>
+ */
+
+}
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.model;
+
+public interface InformationUnit {
+
+ /*
+ * Mobile. Can be shared and can move accross cores.
+ */
+
+ /*
+ * 0 integer
+ *
+ * 1 text
+ *
+ * 2 collection Can have any number of dimensions. Infinite in every
+ * dimension. Every dimension behaves similarly to a column in a table.
+ *
+ * Could be used as a stack FILO Could be used as a pipe FIFO
+ *
+ * List would correspond to 1 dimensional collection. Dict would correspond
+ * to 2 dimensional collection. Entries could be easilly appended, inserted
+ * and removed.
+ *
+ * All dimensions are indexed. It is possible to quickly retrieve any
+ * element(s) by querying against any dimensions.
+ *
+ * Queries can be geometrical too.
+ *
+ * Every dimension could be inclusive or exclusive. Inclusive dimension
+ * allows multiple same values in the same location simultaneously.
+ * Exclusive dimension allows only one value at the location.
+ *
+ * Attempt to store new value at the occupied location would overwrite the
+ * previous.
+ *
+ * Exclusive dimensions could be grouped. This will guarantee unique
+ * combination within group.
+ *
+ * Every dimension could have undefined or fixed type.
+ *
+ * Classes are multidimensional collections. Instances of classes are
+ * collection entries.
+ *
+ * 3 function
+ *
+ * list of input parameters list of output parameters
+ *
+ * 4 byte array
+ *
+ * 5 boolean
+ *
+ * 6 float
+ *
+ * 7 class
+ *
+ * 8 class instance
+ */
+ int getType();
+
+}
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.model;
+
+public class IntegerUnit implements InformationUnit {
+
+ public final int value;
+
+ public IntegerUnit(final int value) {
+ this.value = value;
+ }
+
+ @Override
+ public int getType() {
+ return 0;
+ }
+
+}
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.store;
+
+import java.io.IOException;
+
+/**
+ * Data store acts as a numerical ID to corresponding record map.
+ * <p>
+ * Record is basically an array of bytes of arbitrary length, identifiable by ID
+ * that is assigned to record during record creation.
+ * <p>
+ * Records can be updated with alternative content and length. Data store takes
+ * care of data fragmentation.
+ */
+public interface DataStore {
+
+ /**
+ * Close datastore.
+ */
+ void close() throws IOException;
+
+ /**
+ * Create new record and set its initial contents.
+ */
+ int createRecord(byte[] value) throws IOException;
+
+ /**
+ * Delete record identified by given ID. DataStore will mark given ID as
+ * unused, and could reuse this ID later for another newly created record.
+ */
+ void deleteRecord(int id) throws IOException;
+
+ /**
+ * Read entire record into byte array.
+ */
+ byte[] readRecord(int id) throws IOException;
+
+ /**
+ * Update record with new value.
+ */
+ void updateRecord(int id, byte[] value) throws IOException;
+
+}
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.store.file;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class EntryAllocationTable {
+
+ private final FileDataStore fileDataStore;
+
+ public EntryAllocationTable(final FileDataStore fileDataStore) {
+ this.fileDataStore = fileDataStore;
+ }
+
+ public void enlarge(final int newSize) throws IOException {
+ final int oldSize = fileDataStore.metaData.getEntriesTableSize();
+
+ for (int i = oldSize; i < newSize; i++) {
+ final EntryRecord entryRecord = new EntryRecord(i, 0, 0);
+ entryRecord.save(fileDataStore);
+ }
+
+ fileDataStore.metaData.setEntriesTableSize(newSize);
+ }
+
+ public long getEntryRecordLocation(final int entryId) {
+ return (entryId * EntryRecord.ENTRY_RECORD_LENGTH)
+ + MetaData.FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START;
+ }
+
+ public int getNewUnusedEntryId() throws IOException {
+ while (true) {
+
+ final int newEntryId = fileDataStore.metaData.getNewEntryId();
+
+ final EntryRecord entryRecord = new EntryRecord(fileDataStore,
+ newEntryId);
+
+ if (!entryRecord.isUsed())
+ return newEntryId;
+ }
+ }
+
+ public void initializeNewFile() throws IOException {
+ for (int i = 0; i < fileDataStore.metaData.getEntriesTableSize(); i++) {
+ final EntryRecord entryRecord = new EntryRecord(i, 0, 0);
+ entryRecord.save(fileDataStore);
+ }
+ }
+
+ /**
+ * Sorted list of @link {@link EntryRecord}'s.
+ */
+ public List<EntryRecord> loadAllEntryRecords() throws IOException {
+ final List<EntryRecord> entryRecords = new ArrayList<>();
+
+ for (int i = 0; i < fileDataStore.metaData.getEntriesTableSize(); i++) {
+ final EntryRecord entryRecord = new EntryRecord(fileDataStore, i);
+ if (entryRecord.isUsed())
+ entryRecords.add(entryRecord);
+ }
+
+ Collections.sort(entryRecords);
+
+ return entryRecords;
+ }
+
+}
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.store.file;
+
+import java.io.IOException;
+
+class EntryRecord implements Comparable<EntryRecord> {
+ public static final int ENTRY_RECORD_LENGTH = 12;
+ final int id;
+ long location = 0;
+ int length = 0;
+
+ public EntryRecord(final FileDataStore dataStore, final int id)
+ throws IOException {
+
+ this.id = id;
+
+ final long entryRecordLocation = dataStore.entryAllocationTable
+ .getEntryRecordLocation(id);
+
+ location = dataStore.readLong(entryRecordLocation);
+
+ if (isUsed())
+ length = dataStore.readInt(entryRecordLocation + 8);
+ }
+
+ public EntryRecord(final int id, final long location, final int length) {
+ this.id = id;
+ this.location = location;
+ this.length = length;
+ }
+
+ public void clear() {
+ location = 0;
+ length = 0;
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (o == null) return false;
+ return o instanceof EntryRecord && compareTo((EntryRecord) o) == 0;
+ }
+
+ @Override
+ public int compareTo(final EntryRecord o) {
+ if (location < o.location)
+ return -1;
+ if (location > o.location)
+ return 1;
+
+ if (id < o.id)
+ return -1;
+ if (id > o.id)
+ return 1;
+
+ return 0;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = id;
+ result = 31 * result + (int) (location ^ (location >>> 32));
+ return result;
+ }
+
+ public boolean isUsed() {
+ return location != 0;
+
+ }
+
+ public void save(final FileDataStore dataStore) throws IOException {
+
+ final long entryRecordLocation = dataStore.entryAllocationTable
+ .getEntryRecordLocation(id);
+
+ dataStore.writeLong(entryRecordLocation, location);
+ dataStore.writeInt(entryRecordLocation + 8, length);
+ }
+}
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.store.file;
+
+import eu.svjatoslav.aukio.data.store.DataStore;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.List;
+
+/**
+ * DataStore backed by single filesystem file.
+ */
+public class FileDataStore implements DataStore {
+
+ final MetaData metaData = new MetaData(this);
+ final EntryAllocationTable entryAllocationTable = new EntryAllocationTable(this);
+ RandomAccessFile randomAccessFile;
+
+ public FileDataStore(final File backingFile) throws IOException {
+ if (backingFile.exists())
+ initializeFromExistingFile(backingFile);
+ else
+ initializeNewFile(backingFile);
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ metaData.writeFileHeader();
+ randomAccessFile.close();
+ }
+
+ @Override
+ public synchronized int createRecord(final byte[] value) throws IOException {
+
+ if (metaData.entriesTableNeedsIncreasing())
+ increaseEntriesTable();
+
+ final int newEntryId = entryAllocationTable.getNewUnusedEntryId();
+
+ final long currentLocation = metaData
+ .allocateStorageSpace(value.length);
+
+ final EntryRecord record = new EntryRecord(newEntryId, currentLocation,
+ value.length);
+
+ metaData.increaseUsedEntriesCount();
+
+ record.save(this);
+
+ randomAccessFile.seek(currentLocation);
+ randomAccessFile.write(value);
+
+ // update header to increase crash resilience
+ metaData.writeFileHeader();
+ return newEntryId;
+ }
+
+ @Override
+ public synchronized void deleteRecord(final int id) throws IOException {
+ final EntryRecord entryRecord = new EntryRecord(this, id);
+
+ if (!entryRecord.isUsed())
+ throw new RuntimeException("Record already does not exist!");
+
+ entryRecord.clear();
+ entryRecord.save(this);
+ metaData.decreaseUsedEntriesCount();
+
+ // update header to increase crash resilience
+ metaData.writeFileHeader();
+ }
+
+ public long getDefragmentationStartAddress(final int allowedFragmentationPercent,
+ final List<EntryRecord> allEntryRecords) {
+
+ final VacuumContext context = new VacuumContext(this,
+ metaData.allocateStorageSpace(0), allowedFragmentationPercent);
+
+ for (int i = allEntryRecords.size() - 1; i >= 0; i--) {
+ final EntryRecord entryRecord = allEntryRecords.get(i);
+ context.analyzeEntry(entryRecord);
+ }
+ context.analyzeStartOfDataArea();
+
+ return context.defragmentationStartAddress;
+ }
+
+ public void increaseEntriesTable() throws IOException {
+ final List<EntryRecord> allEntryRecords = entryAllocationTable
+ .loadAllEntryRecords();
+
+ final int newEntriesTableSize = metaData.getEntriesTableSize() * 2;
+
+ final long dataEvacuationTreshold = metaData
+ .getEntriesStorageAreaStart(newEntriesTableSize);
+
+ metaData.ensureMinimumCurrentLocation(dataEvacuationTreshold);
+
+ for (final EntryRecord record : allEntryRecords)
+ if (record.location < dataEvacuationTreshold) {
+
+ final long newEntryLocation = metaData
+ .allocateStorageSpace(record.length);
+
+ // read record content
+ final byte[] entryContent = new byte[record.length];
+
+ randomAccessFile.seek(record.location);
+ randomAccessFile.readFully(entryContent);
+
+ // write record content to new location
+ randomAccessFile.seek(newEntryLocation);
+ randomAccessFile.write(entryContent);
+
+ // update record header
+ record.location = newEntryLocation;
+ record.save(this);
+ }
+
+ entryAllocationTable.enlarge(newEntriesTableSize);
+ System.out.println("Entries table increased.");
+
+ // update header to increase crash resilience
+ metaData.writeFileHeader();
+ }
+
+ public void initializeFromExistingFile(final File backingFile)
+ throws IOException {
+ try {
+ randomAccessFile = new RandomAccessFile(backingFile, "rw");
+
+ metaData.readFileHeader();
+
+ } catch (final FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void initializeNewFile(final File backingFile) throws IOException {
+ try {
+ randomAccessFile = new RandomAccessFile(backingFile, "rw");
+ } catch (final FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+
+ metaData.initializeNewFile();
+
+ entryAllocationTable.initializeNewFile();
+ }
+
+ public int readInt(final long position) throws IOException {
+ randomAccessFile.seek(position);
+ return randomAccessFile.readInt();
+ }
+
+ public long readLong(final long position) throws IOException {
+ randomAccessFile.seek(position);
+ return randomAccessFile.readLong();
+ }
+
+ @Override
+ public synchronized byte[] readRecord(final int id) throws IOException {
+
+ final EntryRecord entryRecord = new EntryRecord(this, id);
+
+ if (!entryRecord.isUsed())
+ throw new RuntimeException("Entity record by id: " + id
+ + " does not exist.");
+
+ final byte[] result = new byte[entryRecord.length];
+
+ randomAccessFile.seek(entryRecord.location);
+ randomAccessFile.readFully(result);
+
+ return result;
+ }
+
+ public long relocateEntry(final EntryRecord record, final long newLocation)
+ throws IOException {
+
+ if (record.location != newLocation) {
+ System.out.println("Relocating record " + record.id + " from: "
+ + record.location + " to: " + newLocation);
+
+ final byte[] result = new byte[record.length];
+
+ randomAccessFile.seek(record.location);
+ randomAccessFile.readFully(result);
+
+ randomAccessFile.seek(newLocation);
+ randomAccessFile.write(result);
+
+ record.location = newLocation;
+ record.save(this);
+ }
+
+ return newLocation + record.length;
+ }
+
+ @Override
+ public synchronized void updateRecord(final int id, final byte[] value)
+ throws IOException {
+
+ final EntryRecord entryRecord = new EntryRecord(this, id);
+
+ if (!entryRecord.isUsed())
+ throw new RuntimeException("Entry record is empty!");
+
+ final int newDataSize = value.length;
+
+ if (entryRecord.length >= newDataSize) {
+ // update record in place
+ if (entryRecord.length != newDataSize) {
+ entryRecord.length = newDataSize;
+ entryRecord.save(this);
+ }
+ } else {
+ // save record to the new location
+ entryRecord.location = metaData.allocateStorageSpace(newDataSize);
+ entryRecord.length = newDataSize;
+ entryRecord.save(this);
+ }
+
+ randomAccessFile.seek(entryRecord.location);
+ randomAccessFile.write(value);
+
+ // update header to increase crash resilience
+ metaData.writeFileHeader();
+ }
+
+ /**
+ * Defragment empty space.
+ *
+ * @param allowedFragmentationPercent allowed maximum percentage of free space, relative to total
+ * used space.
+ * @throws IOException
+ */
+ public void vacuum(final int allowedFragmentationPercent) throws IOException {
+
+ final List<EntryRecord> allEntryRecords = entryAllocationTable
+ .loadAllEntryRecords();
+
+ final long defragmentationStart = getDefragmentationStartAddress(
+ allowedFragmentationPercent, allEntryRecords);
+
+ long nextFreeSpace = defragmentationStart;
+
+ for (final EntryRecord record : allEntryRecords) {
+ if (record.location < defragmentationStart)
+ continue;
+
+ nextFreeSpace = relocateEntry(record, nextFreeSpace);
+
+ }
+
+ metaData.setCurrentLocation(nextFreeSpace);
+
+ // update header to increase crash resilience
+ metaData.writeFileHeader();
+ }
+
+ public void writeInt(final long position, final int value)
+ throws IOException {
+ randomAccessFile.seek(position);
+ randomAccessFile.writeInt(value);
+ }
+
+ public void writeLong(final long position, final long value)
+ throws IOException {
+ randomAccessFile.seek(position);
+ randomAccessFile.writeLong(value);
+ }
+}
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.store.file;
+
+import java.io.IOException;
+
+public class MetaData {
+
+ public static final int FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START = 4 * 6;
+ private static final int FORMAT_VERSION = 1;
+ private static final int FILE_LOCATION_VERSION = 4 * 0;
+ private static final int FILE_LOCATION_ENTRIES_TABLE_SIZE = 4 * 1;
+ private static final int FILE_LOCATION_USED_ENTRIES_COUNT = 4 * 2;
+ private static final int FILE_LOCATION_CURRENT_ENTRY = 4 * 3;
+ private static final int FILE_LOCATION_CURRENT_LOCATION = 4 * 4; // LONG!!
+ private final FileDataStore dataStore;
+ private int entriesTableSize;
+ private int currentEntry;
+ private int usedEntriesCount;
+ private long currentLocation;
+
+ public MetaData(final FileDataStore dataStore) {
+ this.dataStore = dataStore;
+ }
+
+ /**
+ * @return address of the start of the allocated space
+ */
+ public synchronized long allocateStorageSpace(final int amountOfBytes) {
+ final long locationStart = currentLocation;
+ currentLocation += amountOfBytes;
+ return locationStart;
+ }
+
+ public void decreaseUsedEntriesCount() {
+ usedEntriesCount--;
+ }
+
+ public synchronized void ensureMinimumCurrentLocation(
+ final long minimumLocation) {
+
+ if (currentLocation < minimumLocation)
+ currentLocation = minimumLocation;
+ }
+
+ public boolean entriesTableNeedsIncreasing() {
+ final int reserveSize = (entriesTableSize / 10) + 1;
+
+ return usedEntriesCount >= (entriesTableSize - reserveSize);
+
+ }
+
+ public long getEntriesStorageAreaStart() {
+ return FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START
+ + (EntryRecord.ENTRY_RECORD_LENGTH * entriesTableSize);
+ }
+
+ public long getEntriesStorageAreaStart(
+ final int hypotheticalEntriesTableSize) {
+ return FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START
+ + (EntryRecord.ENTRY_RECORD_LENGTH * hypotheticalEntriesTableSize);
+ }
+
+ /**
+ * @return amount of entries in the entries table
+ */
+ public int getEntriesTableSize() {
+ return entriesTableSize;
+ }
+
+ public void setEntriesTableSize(final int newSize) {
+ entriesTableSize = newSize;
+ }
+
+ public int getNewEntryId() {
+ currentEntry++;
+
+ if (currentEntry >= getEntriesTableSize())
+ currentEntry = 0;
+
+ return currentEntry;
+ }
+
+ public void increaseUsedEntriesCount() {
+ usedEntriesCount++;
+ }
+
+ public void initializeNewFile() throws IOException {
+ entriesTableSize = 16;
+ usedEntriesCount = 0;
+ currentEntry = entriesTableSize - 1;
+ currentLocation = getEntriesStorageAreaStart();
+
+ writeFileHeader();
+ }
+
+ public void readFileHeader() throws IOException {
+
+ final int fileVersion = dataStore.readInt(FILE_LOCATION_VERSION);
+ if (fileVersion != FORMAT_VERSION)
+ throw new RuntimeException("File version is " + fileVersion
+ + " but version " + FORMAT_VERSION + " is required.");
+
+ entriesTableSize = dataStore.readInt(FILE_LOCATION_ENTRIES_TABLE_SIZE);
+
+ usedEntriesCount = dataStore.readInt(FILE_LOCATION_USED_ENTRIES_COUNT);
+
+ currentEntry = dataStore.readInt(FILE_LOCATION_CURRENT_ENTRY);
+
+ currentLocation = dataStore.readLong(FILE_LOCATION_CURRENT_LOCATION);
+ }
+
+ public void setCurrentLocation(final long currentLocation) {
+ this.currentLocation = currentLocation;
+ }
+
+ public void writeFileHeader() throws IOException {
+ dataStore.writeInt(FILE_LOCATION_VERSION, FORMAT_VERSION);
+ dataStore.writeInt(FILE_LOCATION_ENTRIES_TABLE_SIZE,
+ getEntriesTableSize());
+ dataStore.writeInt(FILE_LOCATION_USED_ENTRIES_COUNT, usedEntriesCount);
+ dataStore.writeInt(FILE_LOCATION_CURRENT_ENTRY, currentEntry);
+ dataStore.writeLong(FILE_LOCATION_CURRENT_LOCATION, currentLocation);
+ }
+
+}
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.store.file;
+
+public class VacuumContext {
+ private final FileDataStore vacuumContext;
+ private final int targetFragmentationPercent;
+ long analyzedBytesCount = 0;
+ long freeBytesCount = 0;
+ long lastRecordStartAddress;
+ long defragmentationStartAddress;
+
+ public VacuumContext(final FileDataStore fileDataStore,
+ final long currentFreeAddress, final int targetFragmentationPercent) {
+
+ vacuumContext = fileDataStore;
+ lastRecordStartAddress = currentFreeAddress;
+ defragmentationStartAddress = currentFreeAddress;
+ this.targetFragmentationPercent = targetFragmentationPercent;
+ }
+
+ public void analyzeEntry(final EntryRecord record) {
+ final long bytesSinceLastEntry = lastRecordStartAddress
+ - record.location;
+
+ final long freeSpace = bytesSinceLastEntry - record.length;
+ freeBytesCount += freeSpace;
+ analyzedBytesCount += freeSpace;
+
+ checkDefragmentationPointer(record.location + record.length);
+
+ analyzedBytesCount += record.length;
+ lastRecordStartAddress = record.location;
+ }
+
+ public void analyzeStartOfDataArea() {
+ freeBytesCount += lastRecordStartAddress
+ - vacuumContext.metaData.getEntriesStorageAreaStart();
+
+ checkDefragmentationPointer(vacuumContext.metaData
+ .getEntriesStorageAreaStart());
+ }
+
+ public void checkDefragmentationPointer(
+ final long defragmentationStartAddress) {
+
+ if (analyzedBytesCount == 0) {
+ this.defragmentationStartAddress = defragmentationStartAddress;
+ return;
+ }
+
+ final int fragmentationPercentage = (int) ((100L * freeBytesCount) / analyzedBytesCount);
+
+ if (fragmentationPercentage >= targetFragmentationPercent)
+ this.defragmentationStartAddress = defragmentationStartAddress;
+ }
+}
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.store.file;
+
+/**
+ * <pre>
+ * File based storage consists of three segments in the following order:
+ *
+ * METADATA
+ * Small and fixed length.
+ * Contains few variables.
+ *
+ * ENTRY ALLOCATION TABLE
+ * Size of this segment dynamically expands as more
+ * entries are created and more of unique ID's are needed.
+ *
+ * This segment consists of many small fixed length records,
+ * one per entry.
+ *
+ * ESTRIES STORAGE AREA
+ * This area holds actual content of stored entries.
+ *
+ * </pre>
+ */
--- /dev/null
+/*
+ * Aukio Data. Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.data.store;
+
+/**
+ * Underlying storage layer.
+ * An fast ID to byte array map.
+ * Main interface {@link eu.svjatoslav.aukio.data.store.DataStore}.
+ */
--- /dev/null
+/*
+ * Aukio - System for data storage, computation, exploration and interaction.
+ * Author: Svjatoslav Agejenko.
+ * This project is released under Creative Commons Zero (CC0) license.
+ *
+*/
+
+package eu.svjatoslav.aukio.datastore;
+
+import eu.svjatoslav.aukio.data.store.DataStore;
+import eu.svjatoslav.aukio.data.store.file.FileDataStore;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+
+public class DataStoreImplTest {
+
+ public static final String UTF_8 = "UTF-8";
+ File dataSourceFile;
+
+ DataStore dataStore;
+
+ @Before
+ public void setUp() throws Exception {
+ dataSourceFile = new File("test.byar");
+ if (dataSourceFile.exists())
+ if (!dataSourceFile.delete())
+ throw new RuntimeException("Cannot delete file: " + dataSourceFile.getCanonicalPath());
+
+ dataStore = new FileDataStore(dataSourceFile);
+ }
+
+ @Test
+ public void testBasicStorageAndRetrieval() throws IOException {
+
+ final int record1 = dataStore.createRecord("See on test".getBytes(UTF_8));
+ final int record2 = dataStore.createRecord("another test".getBytes(UTF_8));
+
+ Assert.assertEquals(0, record1);
+ Assert.assertEquals(1, record2);
+
+ Assert.assertEquals("another test", new String(dataStore.readRecord(record2),
+ UTF_8));
+ Assert.assertEquals("See on test", new String(dataStore.readRecord(record1),
+ UTF_8));
+
+ dataStore.updateRecord(record1, "!!!".getBytes(UTF_8));
+
+ Assert.assertEquals("!!!", new String(dataStore.readRecord(record1), UTF_8));
+
+ dataStore.updateRecord(record1, dataStore.readRecord(record2));
+
+ Assert.assertEquals("another test", new String(dataStore.readRecord(record1),
+ UTF_8));
+
+ dataStore.close();
+ }
+
+ @Test
+ public void testDefragmentation() throws IOException {
+ final int record1 = dataStore.createRecord("1111".getBytes(UTF_8));
+ final int record2 = dataStore.createRecord("22".getBytes(UTF_8));
+ final int record3 = dataStore.createRecord("3333333".getBytes(UTF_8));
+ final int record4 = dataStore.createRecord("4".getBytes(UTF_8));
+
+ ((FileDataStore) dataStore).vacuum(0);
+
+ final int record5 = dataStore.createRecord("555".getBytes());
+
+ dataStore.deleteRecord(record1);
+
+ ((FileDataStore) dataStore).vacuum(0);
+
+ final int record6 = dataStore.createRecord("66".getBytes());
+
+ dataStore.close();
+ }
+
+ @Test
+ public void testEntriesTableIncreasing() throws IOException {
+ // dataStore.createRecord(String.valueOf("lalalala").getBytes());
+
+ for (int j = 0; j < 10; j++) {
+ final ArrayList<Integer> ids = new ArrayList<>();
+
+ for (int i = 0; i < 100; i++) {
+ final int id = dataStore.createRecord(String.valueOf(
+ "____________" + i).getBytes());
+
+ ids.add(id);
+ }
+
+ for (final int id : ids)
+ dataStore.deleteRecord(id);
+
+ ((FileDataStore) dataStore).vacuum(0);
+ }
+ }
+
+}
projects / aukio-data.git / commitdiff