From: Svjatoslav Agejenko Date: Sat, 19 Sep 2026 18:50:12 +0000 (+0300) Subject: initial commit X-Git-Url: http://www2.svjatoslav.eu/gitweb/?a=commitdiff_plain;h=c43eba5d629334af11dcb0730a0e20225bd56d7e;p=aukio-data.git initial commit --- c43eba5d629334af11dcb0730a0e20225bd56d7e diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0842dc9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +/.idea/ +/.settings/ +/target/ +/doc/apidocs/ +/*.iml +/test.byar +*.html +/.classpath +/.project \ No newline at end of file diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..0e259d4 --- /dev/null +++ b/COPYING @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/Tools/Open with IntelliJ IDEA b/Tools/Open with IntelliJ IDEA new file mode 100755 index 0000000..de9bae5 --- /dev/null +++ b/Tools/Open with IntelliJ IDEA @@ -0,0 +1,18 @@ +#!/bin/bash + +# +# This is a helper bash script that starts IntelliJ with the current project. +# Script is written is such a way that you can simply click on it in file +# navigator to run it. +# +# +# Script assumes: +# +# + GNU operating system +# + IntelliJ is installed and commandline launcher "idea" is enabled. +# + +cd "${0%/*}" +cd .. + +setsid idea . &>/dev/null diff --git a/Tools/Update web site b/Tools/Update web site new file mode 100755 index 0000000..5c45e7d --- /dev/null +++ b/Tools/Update web site @@ -0,0 +1,78 @@ +#!/bin/bash +cd "${0%/*}"; if [ "$1" != "T" ]; then gnome-terminal -e "'$0' T"; exit; fi; + +cd .. + +export_org_to_html() { + local org_file=$1 + local dir=$(dirname "$org_file") + local base=$(basename "$org_file" .org) + ( + cd "$dir" || return 1 + local html_file="${base}.html" + if [ -f "$html_file" ]; then + rm -f "$html_file" + fi + echo "Exporting: $org_file → $dir/$html_file" + emacs --batch -l ~/.emacs --visit="${base}.org" --funcall=org-html-export-to-html --kill + if [ $? -eq 0 ]; then + echo "✓ Successfully exported $org_file" + else + echo "✗ Failed to export $org_file" + return 1 + fi + ) +} + +export_org_files_to_html() { + echo "🔍 Searching for .org files in doc/ ..." + echo "=======================================" + + mapfile -t ORG_FILES < <(find doc -type f -name "*.org" | sort) + + if [ ${#ORG_FILES[@]} -eq 0 ]; then + echo "❌ No .org files found!" + return 1 + fi + + echo "Found ${#ORG_FILES[@]} .org file(s):" + printf '%s\n' "${ORG_FILES[@]}" + echo "=======================================" + + SUCCESS_COUNT=0 + FAILED_COUNT=0 + + for org_file in "${ORG_FILES[@]}"; do + export_org_to_html "$org_file" + if [ $? -eq 0 ]; then + ((SUCCESS_COUNT++)) + else + ((FAILED_COUNT++)) + fi + done + + echo "=======================================" + echo "📊 SUMMARY:" + echo " ✓ Successful: $SUCCESS_COUNT" + echo " ✗ Failed: $FAILED_COUNT" + echo " Total: $((SUCCESS_COUNT + FAILED_COUNT))" + echo "" +} + +mvn clean package + +export_org_files_to_html + +echo "📤 Uploading to server..." +rsync -avz --delete -e 'ssh -p 10006' doc/ \ + n0@www3.svjatoslav.eu:/mnt/big/projects/aukio-data/ + +if [ $? -eq 0 ]; then + echo "✓ Upload completed successfully!" +else + echo "✗ Upload failed!" +fi + +echo "" +echo "Press ENTER to close this window." +read \ No newline at end of file diff --git a/doc/index.org b/doc/index.org new file mode 100644 index 0000000..ac0230d --- /dev/null +++ b/doc/index.org @@ -0,0 +1,143 @@ +#+SETUPFILE: ~/.emacs.d/org-styles/html/darksun.theme +#+TITLE: Aukio Data - Data storage and computing engine +#+LANGUAGE: en +#+LATEX_HEADER: \usepackage[margin=1.0in]{geometry} +#+LATEX_HEADER: \usepackage{parskip} +#+LATEX_HEADER: \usepackage[none]{hyphenat} + +#+OPTIONS: H:20 num:20 +#+OPTIONS: author:nil + +* General +:PROPERTIES: +:CUSTOM_ID: general +:END: + +- This program is free software: released under Creative Commons Zero + (CC0) license + +- Program author: + - Svjatoslav Agejenko + - Homepage: https://svjatoslav.eu + - Email: mailto://svjatoslav@svjatoslav.eu + +- [[https://www.svjatoslav.eu/projects/][Other software projects hosted at svjatoslav.eu]] + +** Source code +:PROPERTIES: +:CUSTOM_ID: source-code +:END: +- [[https://www2.svjatoslav.eu/gitweb/?p=aukio-data.git;a=snapshot;h=HEAD;sf=tgz][Download latest snapshot in TAR GZ format]] + +- [[https://www2.svjatoslav.eu/gitweb/?p=aukio-data.git;a=summary][Browse Git repository online]] + +- Clone Git repository using command: + : git clone https://www2.svjatoslav.eu/git/aukio-data.git + +- See [[https://www3.svjatoslav.eu/projects/aukio-data/apidocs/][JavaDoc]]. + +* Vision / goal + :PROPERTIES: + :CUSTOM_ID: vision-goal + :ID: f6764282-a6f6-44e6-8716-b428074dd093 + :END: + +Provide hackable, versioned, optimized, distributed, geometrical, +arbitrary dimensional ([[id:96116550-a6a1-4700-bef7-865d0deee7ea][hypercube based]]) data storage and computation +engine ([[id:d2375acc-af14-4f18-8ad0-7949501178c5][as inspired by the brain]]) for general purpose visual computing +environment called [[http://www2.svjatoslav.eu/gitbrowse/aukio/doc/index.html][Aukio]]. + + +* Ideas / inspiration +:PROPERTIES: +:CUSTOM_ID: ideas +:END: + ++ [[https://en.wikipedia.org/wiki/OLAP_cube][OLAP cube]]. + +** Brain + :PROPERTIES: + :CUSTOM_ID: brain + :ID: d2375acc-af14-4f18-8ad0-7949501178c5 + :END: + ++ Brain appears to be natural geometrical/parallel data storage and + computational engine: + + https://www.quantamagazine.org/the-brain-maps-out-ideas-and-memories-like-spaces-20190114/ + + [[https://www.simonsfoundation.org/2021/04/07/geometrical-thinking-offers-a-window-into-computation/][Geometrical Thinking Offers a Window Into Computation]] + ++ Even more awesome is that brain appears to operate and is wired as + arbitrary/variable dimensional structure: + https://singularityhub.com/2017/06/21/is-there-a-multidimensional-mathematical-world-hidden-in-the-brains-computation/ + ++ On top of this, this multidimensional space that brain represents + has dynamic/variable resolution/density: + + https://www.quantamagazine.org/goals-and-rewards-redraw-the-brains-map-of-the-world-20190328 + ++ Such properties allow parallel Geometrical computation and + beautifully fits CM-1 Connection Machine architecture (for extra + hardware accelerated solution). + +** CM-1 Connection Machine +:PROPERTIES: +:CUSTOM_ID: cm-1-connection-machine +:ID: 01aa65c1-3d44-44a8-9b90-58454bc6be80 +:END: + +https://en.wikipedia.org/wiki/Connection_Machine + +Massively parallel (thousands of CPUs) connected via +machine's internal 12-dimensional hypercube network allows to +efficiently simulate arbitrary dimensional hypercube and network +topology between computational units. So that when we are +solving/simulating for example 5 dimensional problem, we can arrange +computational units into virtual 5D network. See: +http://www.mission-base.com/tamiko/theory/cm_txts/di-ch2.html + +we can pre-distribute data across computation units and perform +parallel [[id:171fe375-c737-41e6-b429-a414f6abc5d8][geometrical computation]]. + + +** Integrate with lossless diamond square algorithm based multidimensional data compressor +:PROPERTIES: +:CUSTOM_ID: integrate-lossless-diamond-square-compressor +:END: + ++ Demonstrate using 3D UI for debugging algorithm internals. + ++ Separate RGB/YUV channels into separate encoders/decoders + ++ Make demo lossless image and sound CODECS. + +** Distributed computation and data storage + :PROPERTIES: + :CUSTOM_ID: distributed-computation-and-data-storage + :ID: 5d287158-53ea-44a2-a754-dd862366066a + :END: + +Lots of problems can be translated to geometry (use any shapes and as +many dimensions as you need). Solution(s) to such problems could be +then found via geometrical search/comparison/lookup results. As a +bonus, such geometrical *data storage* AND *computation* can be +naturally made in *parallel* and *distributed*. + +** The "Datacube" is a Fundamental Data Structure + +The world is not just Sets (SQL), Hierarchies (XML/JSON), or Graphs +(SPARQL). *Arrays* are the fourth pillar. Treating them as "just +another file format" or "a blob in a table" is a fundamental error +that leads to performance collapse. + +* Current status +:PROPERTIES: +:CUSTOM_ID: current-status +:END: + +- More or less defined [[id:f6764282-a6f6-44e6-8716-b428074dd093][Vision / goal]]. + +- Collected some [[id:0fa6354b-18c9-4120-bbf5-c7239aebecab][inspiring]] [[id:171fe375-c737-41e6-b429-a414f6abc5d8][ideas]]. + +- Implemented very simple persistent key-value map. + - Long term goal is to use it as a backing storage engine and + implement more advanced features on top of this via layered + architecture. diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..60f5e6c --- /dev/null +++ b/pom.xml @@ -0,0 +1,131 @@ + + 4.0.0 + eu.svjatoslav + aukio-data + 1.0-SNAPSHOT + Aukio data + Aukio data + + + 1.8 + 1.8 + 1.8 + UTF-8 + UTF-8 + + + + svjatoslav.eu + http://svjatoslav.eu + + + + + junit + junit + 4.8.1 + test + + + + eu.svjatoslav + javainspect + 1.7 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + true + UTF-8 + + + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.1.1 + + + attach-javadocs + + jar + + + + + ${java.home}/bin/javadoc + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.4.3 + + UTF-8 + + + + + + + + org.apache.maven.wagon + wagon-ssh-external + 2.6 + + + + + + + + svjatoslav.eu + svjatoslav.eu + scpexe://svjatoslav.eu:10006/srv/maven + + + svjatoslav.eu + svjatoslav.eu + scpexe://svjatoslav.eu:10006/srv/maven + + + + + + svjatoslav.eu + Svjatoslav repository + https://www3.svjatoslav.eu/maven/ + + + + + scm:git:ssh://n0@svjatoslav.eu/home/git/repositories/aukio-data.git + scm:git:ssh://n0@svjatoslav.eu/home/git/repositories/aukio-data.git + + + + + diff --git a/src/main/java/eu/svjatoslav/aukio/data/model/ArrayStorage.java b/src/main/java/eu/svjatoslav/aukio/data/model/ArrayStorage.java new file mode 100755 index 0000000..acb97ca --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/model/ArrayStorage.java @@ -0,0 +1,33 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.model; + +public class ArrayStorage { + + // load 16bit integer from byte array + public static int load16bit(final byte[] array, final int addr) { + // return (int)array[addr] + (int)array[addr+1]*256; + return ((array[addr + 1] & 0xFF) << 8) | (array[addr] & 0xFF); + } + + // load 16bit integer from byte array + public static int load32bit(final byte[] array, final int addr) { + return ((array[addr + 3] & 0xFF) << 24) + | ((array[addr + 2] & 0xFF) << 16) + | ((array[addr + 1] & 0xFF) << 8) | (array[addr] & 0xFF); + } + + // store 16bit integer into byte array + public static void store16bit(final byte[] array, final int addr, + final int value) { + final int byteHigh = value / 256; + final int byteLow = value - (byteHigh * 256); + array[addr] = (byte) byteLow; + array[addr + 1] = (byte) byteHigh; + } + +} diff --git a/src/main/java/eu/svjatoslav/aukio/data/model/EntryClass.java b/src/main/java/eu/svjatoslav/aukio/data/model/EntryClass.java new file mode 100644 index 0000000..f289cec --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/model/EntryClass.java @@ -0,0 +1,44 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.model; + +public class EntryClass { + + /** + * 
+     * Datastore entries are instances of particular class.
+     *
+     * Class declares:
+     *     data fields
+     *
+     *     methods
+     *          method is a tree like (code branches) data structure
+     *          of commands and user comments.
+     *
+     * Class in actually a declaration of an multidimensional array.
+     * With every field acting as a dimension.
+     *
+     *
+     * Field properties:
+     *     * mandatory.
+     *
+     *     * data type:
+     *          Instance 
+     *
+     *          enumeration
+     *
+     *          float
+     *          integer
+     *          string
+     *          byte[]
+     *
+     *          Collection 
+     *
+     *
+ */ + +} diff --git a/src/main/java/eu/svjatoslav/aukio/data/model/InformationUnit.java b/src/main/java/eu/svjatoslav/aukio/data/model/InformationUnit.java new file mode 100755 index 0000000..b8f4e70 --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/model/InformationUnit.java @@ -0,0 +1,65 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.model; + +public interface InformationUnit { + + /* + * Mobile. Can be shared and can move accross cores. + */ + + /* + * 0 integer + * + * 1 text + * + * 2 collection Can have any number of dimensions. Infinite in every + * dimension. Every dimension behaves similarly to a column in a table. + * + * Could be used as a stack FILO Could be used as a pipe FIFO + * + * List would correspond to 1 dimensional collection. Dict would correspond + * to 2 dimensional collection. Entries could be easilly appended, inserted + * and removed. + * + * All dimensions are indexed. It is possible to quickly retrieve any + * element(s) by querying against any dimensions. + * + * Queries can be geometrical too. + * + * Every dimension could be inclusive or exclusive. Inclusive dimension + * allows multiple same values in the same location simultaneously. + * Exclusive dimension allows only one value at the location. + * + * Attempt to store new value at the occupied location would overwrite the + * previous. + * + * Exclusive dimensions could be grouped. This will guarantee unique + * combination within group. + * + * Every dimension could have undefined or fixed type. + * + * Classes are multidimensional collections. Instances of classes are + * collection entries. + * + * 3 function + * + * list of input parameters list of output parameters + * + * 4 byte array + * + * 5 boolean + * + * 6 float + * + * 7 class + * + * 8 class instance + */ + int getType(); + +} diff --git a/src/main/java/eu/svjatoslav/aukio/data/model/IntegerUnit.java b/src/main/java/eu/svjatoslav/aukio/data/model/IntegerUnit.java new file mode 100755 index 0000000..4fafad8 --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/model/IntegerUnit.java @@ -0,0 +1,22 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.model; + +public class IntegerUnit implements InformationUnit { + + public final int value; + + public IntegerUnit(final int value) { + this.value = value; + } + + @Override + public int getType() { + return 0; + } + +} diff --git a/src/main/java/eu/svjatoslav/aukio/data/store/DataStore.java b/src/main/java/eu/svjatoslav/aukio/data/store/DataStore.java new file mode 100644 index 0000000..f2def3c --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/store/DataStore.java @@ -0,0 +1,48 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.store; + +import java.io.IOException; + +/** + * Data store acts as a numerical ID to corresponding record map. + *

+ * Record is basically an array of bytes of arbitrary length, identifiable by ID + * that is assigned to record during record creation. + *

+ * Records can be updated with alternative content and length. Data store takes + * care of data fragmentation. + */ +public interface DataStore { + + /** + * Close datastore. + */ + void close() throws IOException; + + /** + * Create new record and set its initial contents. + */ + int createRecord(byte[] value) throws IOException; + + /** + * Delete record identified by given ID. DataStore will mark given ID as + * unused, and could reuse this ID later for another newly created record. + */ + void deleteRecord(int id) throws IOException; + + /** + * Read entire record into byte array. + */ + byte[] readRecord(int id) throws IOException; + + /** + * Update record with new value. + */ + void updateRecord(int id, byte[] value) throws IOException; + +} diff --git a/src/main/java/eu/svjatoslav/aukio/data/store/file/EntryAllocationTable.java b/src/main/java/eu/svjatoslav/aukio/data/store/file/EntryAllocationTable.java new file mode 100644 index 0000000..844dba7 --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/store/file/EntryAllocationTable.java @@ -0,0 +1,75 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.store.file; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class EntryAllocationTable { + + private final FileDataStore fileDataStore; + + public EntryAllocationTable(final FileDataStore fileDataStore) { + this.fileDataStore = fileDataStore; + } + + public void enlarge(final int newSize) throws IOException { + final int oldSize = fileDataStore.metaData.getEntriesTableSize(); + + for (int i = oldSize; i < newSize; i++) { + final EntryRecord entryRecord = new EntryRecord(i, 0, 0); + entryRecord.save(fileDataStore); + } + + fileDataStore.metaData.setEntriesTableSize(newSize); + } + + public long getEntryRecordLocation(final int entryId) { + return (entryId * EntryRecord.ENTRY_RECORD_LENGTH) + + MetaData.FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START; + } + + public int getNewUnusedEntryId() throws IOException { + while (true) { + + final int newEntryId = fileDataStore.metaData.getNewEntryId(); + + final EntryRecord entryRecord = new EntryRecord(fileDataStore, + newEntryId); + + if (!entryRecord.isUsed()) + return newEntryId; + } + } + + public void initializeNewFile() throws IOException { + for (int i = 0; i < fileDataStore.metaData.getEntriesTableSize(); i++) { + final EntryRecord entryRecord = new EntryRecord(i, 0, 0); + entryRecord.save(fileDataStore); + } + } + + /** + * Sorted list of @link {@link EntryRecord}'s. + */ + public List loadAllEntryRecords() throws IOException { + final List entryRecords = new ArrayList<>(); + + for (int i = 0; i < fileDataStore.metaData.getEntriesTableSize(); i++) { + final EntryRecord entryRecord = new EntryRecord(fileDataStore, i); + if (entryRecord.isUsed()) + entryRecords.add(entryRecord); + } + + Collections.sort(entryRecords); + + return entryRecords; + } + +} diff --git a/src/main/java/eu/svjatoslav/aukio/data/store/file/EntryRecord.java b/src/main/java/eu/svjatoslav/aukio/data/store/file/EntryRecord.java new file mode 100644 index 0000000..e751880 --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/store/file/EntryRecord.java @@ -0,0 +1,83 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.store.file; + +import java.io.IOException; + +class EntryRecord implements Comparable { + public static final int ENTRY_RECORD_LENGTH = 12; + final int id; + long location = 0; + int length = 0; + + public EntryRecord(final FileDataStore dataStore, final int id) + throws IOException { + + this.id = id; + + final long entryRecordLocation = dataStore.entryAllocationTable + .getEntryRecordLocation(id); + + location = dataStore.readLong(entryRecordLocation); + + if (isUsed()) + length = dataStore.readInt(entryRecordLocation + 8); + } + + public EntryRecord(final int id, final long location, final int length) { + this.id = id; + this.location = location; + this.length = length; + } + + public void clear() { + location = 0; + length = 0; + } + + @Override + public boolean equals(final Object o) { + if (o == null) return false; + return o instanceof EntryRecord && compareTo((EntryRecord) o) == 0; + } + + @Override + public int compareTo(final EntryRecord o) { + if (location < o.location) + return -1; + if (location > o.location) + return 1; + + if (id < o.id) + return -1; + if (id > o.id) + return 1; + + return 0; + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + (int) (location ^ (location >>> 32)); + return result; + } + + public boolean isUsed() { + return location != 0; + + } + + public void save(final FileDataStore dataStore) throws IOException { + + final long entryRecordLocation = dataStore.entryAllocationTable + .getEntryRecordLocation(id); + + dataStore.writeLong(entryRecordLocation, location); + dataStore.writeInt(entryRecordLocation + 8, length); + } +} diff --git a/src/main/java/eu/svjatoslav/aukio/data/store/file/FileDataStore.java b/src/main/java/eu/svjatoslav/aukio/data/store/file/FileDataStore.java new file mode 100644 index 0000000..31f601c --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/store/file/FileDataStore.java @@ -0,0 +1,280 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.store.file; + +import eu.svjatoslav.aukio.data.store.DataStore; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.List; + +/** + * DataStore backed by single filesystem file. + */ +public class FileDataStore implements DataStore { + + final MetaData metaData = new MetaData(this); + final EntryAllocationTable entryAllocationTable = new EntryAllocationTable(this); + RandomAccessFile randomAccessFile; + + public FileDataStore(final File backingFile) throws IOException { + if (backingFile.exists()) + initializeFromExistingFile(backingFile); + else + initializeNewFile(backingFile); + } + + @Override + public synchronized void close() throws IOException { + metaData.writeFileHeader(); + randomAccessFile.close(); + } + + @Override + public synchronized int createRecord(final byte[] value) throws IOException { + + if (metaData.entriesTableNeedsIncreasing()) + increaseEntriesTable(); + + final int newEntryId = entryAllocationTable.getNewUnusedEntryId(); + + final long currentLocation = metaData + .allocateStorageSpace(value.length); + + final EntryRecord record = new EntryRecord(newEntryId, currentLocation, + value.length); + + metaData.increaseUsedEntriesCount(); + + record.save(this); + + randomAccessFile.seek(currentLocation); + randomAccessFile.write(value); + + // update header to increase crash resilience + metaData.writeFileHeader(); + return newEntryId; + } + + @Override + public synchronized void deleteRecord(final int id) throws IOException { + final EntryRecord entryRecord = new EntryRecord(this, id); + + if (!entryRecord.isUsed()) + throw new RuntimeException("Record already does not exist!"); + + entryRecord.clear(); + entryRecord.save(this); + metaData.decreaseUsedEntriesCount(); + + // update header to increase crash resilience + metaData.writeFileHeader(); + } + + public long getDefragmentationStartAddress(final int allowedFragmentationPercent, + final List allEntryRecords) { + + final VacuumContext context = new VacuumContext(this, + metaData.allocateStorageSpace(0), allowedFragmentationPercent); + + for (int i = allEntryRecords.size() - 1; i >= 0; i--) { + final EntryRecord entryRecord = allEntryRecords.get(i); + context.analyzeEntry(entryRecord); + } + context.analyzeStartOfDataArea(); + + return context.defragmentationStartAddress; + } + + public void increaseEntriesTable() throws IOException { + final List allEntryRecords = entryAllocationTable + .loadAllEntryRecords(); + + final int newEntriesTableSize = metaData.getEntriesTableSize() * 2; + + final long dataEvacuationTreshold = metaData + .getEntriesStorageAreaStart(newEntriesTableSize); + + metaData.ensureMinimumCurrentLocation(dataEvacuationTreshold); + + for (final EntryRecord record : allEntryRecords) + if (record.location < dataEvacuationTreshold) { + + final long newEntryLocation = metaData + .allocateStorageSpace(record.length); + + // read record content + final byte[] entryContent = new byte[record.length]; + + randomAccessFile.seek(record.location); + randomAccessFile.readFully(entryContent); + + // write record content to new location + randomAccessFile.seek(newEntryLocation); + randomAccessFile.write(entryContent); + + // update record header + record.location = newEntryLocation; + record.save(this); + } + + entryAllocationTable.enlarge(newEntriesTableSize); + System.out.println("Entries table increased."); + + // update header to increase crash resilience + metaData.writeFileHeader(); + } + + public void initializeFromExistingFile(final File backingFile) + throws IOException { + try { + randomAccessFile = new RandomAccessFile(backingFile, "rw"); + + metaData.readFileHeader(); + + } catch (final FileNotFoundException e) { + throw new RuntimeException(e); + } + } + + public void initializeNewFile(final File backingFile) throws IOException { + try { + randomAccessFile = new RandomAccessFile(backingFile, "rw"); + } catch (final FileNotFoundException e) { + throw new RuntimeException(e); + } + + metaData.initializeNewFile(); + + entryAllocationTable.initializeNewFile(); + } + + public int readInt(final long position) throws IOException { + randomAccessFile.seek(position); + return randomAccessFile.readInt(); + } + + public long readLong(final long position) throws IOException { + randomAccessFile.seek(position); + return randomAccessFile.readLong(); + } + + @Override + public synchronized byte[] readRecord(final int id) throws IOException { + + final EntryRecord entryRecord = new EntryRecord(this, id); + + if (!entryRecord.isUsed()) + throw new RuntimeException("Entity record by id: " + id + + " does not exist."); + + final byte[] result = new byte[entryRecord.length]; + + randomAccessFile.seek(entryRecord.location); + randomAccessFile.readFully(result); + + return result; + } + + public long relocateEntry(final EntryRecord record, final long newLocation) + throws IOException { + + if (record.location != newLocation) { + System.out.println("Relocating record " + record.id + " from: " + + record.location + " to: " + newLocation); + + final byte[] result = new byte[record.length]; + + randomAccessFile.seek(record.location); + randomAccessFile.readFully(result); + + randomAccessFile.seek(newLocation); + randomAccessFile.write(result); + + record.location = newLocation; + record.save(this); + } + + return newLocation + record.length; + } + + @Override + public synchronized void updateRecord(final int id, final byte[] value) + throws IOException { + + final EntryRecord entryRecord = new EntryRecord(this, id); + + if (!entryRecord.isUsed()) + throw new RuntimeException("Entry record is empty!"); + + final int newDataSize = value.length; + + if (entryRecord.length >= newDataSize) { + // update record in place + if (entryRecord.length != newDataSize) { + entryRecord.length = newDataSize; + entryRecord.save(this); + } + } else { + // save record to the new location + entryRecord.location = metaData.allocateStorageSpace(newDataSize); + entryRecord.length = newDataSize; + entryRecord.save(this); + } + + randomAccessFile.seek(entryRecord.location); + randomAccessFile.write(value); + + // update header to increase crash resilience + metaData.writeFileHeader(); + } + + /** + * Defragment empty space. + * + * @param allowedFragmentationPercent allowed maximum percentage of free space, relative to total + * used space. + * @throws IOException + */ + public void vacuum(final int allowedFragmentationPercent) throws IOException { + + final List allEntryRecords = entryAllocationTable + .loadAllEntryRecords(); + + final long defragmentationStart = getDefragmentationStartAddress( + allowedFragmentationPercent, allEntryRecords); + + long nextFreeSpace = defragmentationStart; + + for (final EntryRecord record : allEntryRecords) { + if (record.location < defragmentationStart) + continue; + + nextFreeSpace = relocateEntry(record, nextFreeSpace); + + } + + metaData.setCurrentLocation(nextFreeSpace); + + // update header to increase crash resilience + metaData.writeFileHeader(); + } + + public void writeInt(final long position, final int value) + throws IOException { + randomAccessFile.seek(position); + randomAccessFile.writeInt(value); + } + + public void writeLong(final long position, final long value) + throws IOException { + randomAccessFile.seek(position); + randomAccessFile.writeLong(value); + } +} diff --git a/src/main/java/eu/svjatoslav/aukio/data/store/file/MetaData.java b/src/main/java/eu/svjatoslav/aukio/data/store/file/MetaData.java new file mode 100644 index 0000000..b0e2f26 --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/store/file/MetaData.java @@ -0,0 +1,130 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.store.file; + +import java.io.IOException; + +public class MetaData { + + public static final int FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START = 4 * 6; + private static final int FORMAT_VERSION = 1; + private static final int FILE_LOCATION_VERSION = 4 * 0; + private static final int FILE_LOCATION_ENTRIES_TABLE_SIZE = 4 * 1; + private static final int FILE_LOCATION_USED_ENTRIES_COUNT = 4 * 2; + private static final int FILE_LOCATION_CURRENT_ENTRY = 4 * 3; + private static final int FILE_LOCATION_CURRENT_LOCATION = 4 * 4; // LONG!! + private final FileDataStore dataStore; + private int entriesTableSize; + private int currentEntry; + private int usedEntriesCount; + private long currentLocation; + + public MetaData(final FileDataStore dataStore) { + this.dataStore = dataStore; + } + + /** + * @return address of the start of the allocated space + */ + public synchronized long allocateStorageSpace(final int amountOfBytes) { + final long locationStart = currentLocation; + currentLocation += amountOfBytes; + return locationStart; + } + + public void decreaseUsedEntriesCount() { + usedEntriesCount--; + } + + public synchronized void ensureMinimumCurrentLocation( + final long minimumLocation) { + + if (currentLocation < minimumLocation) + currentLocation = minimumLocation; + } + + public boolean entriesTableNeedsIncreasing() { + final int reserveSize = (entriesTableSize / 10) + 1; + + return usedEntriesCount >= (entriesTableSize - reserveSize); + + } + + public long getEntriesStorageAreaStart() { + return FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START + + (EntryRecord.ENTRY_RECORD_LENGTH * entriesTableSize); + } + + public long getEntriesStorageAreaStart( + final int hypotheticalEntriesTableSize) { + return FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START + + (EntryRecord.ENTRY_RECORD_LENGTH * hypotheticalEntriesTableSize); + } + + /** + * @return amount of entries in the entries table + */ + public int getEntriesTableSize() { + return entriesTableSize; + } + + public void setEntriesTableSize(final int newSize) { + entriesTableSize = newSize; + } + + public int getNewEntryId() { + currentEntry++; + + if (currentEntry >= getEntriesTableSize()) + currentEntry = 0; + + return currentEntry; + } + + public void increaseUsedEntriesCount() { + usedEntriesCount++; + } + + public void initializeNewFile() throws IOException { + entriesTableSize = 16; + usedEntriesCount = 0; + currentEntry = entriesTableSize - 1; + currentLocation = getEntriesStorageAreaStart(); + + writeFileHeader(); + } + + public void readFileHeader() throws IOException { + + final int fileVersion = dataStore.readInt(FILE_LOCATION_VERSION); + if (fileVersion != FORMAT_VERSION) + throw new RuntimeException("File version is " + fileVersion + + " but version " + FORMAT_VERSION + " is required."); + + entriesTableSize = dataStore.readInt(FILE_LOCATION_ENTRIES_TABLE_SIZE); + + usedEntriesCount = dataStore.readInt(FILE_LOCATION_USED_ENTRIES_COUNT); + + currentEntry = dataStore.readInt(FILE_LOCATION_CURRENT_ENTRY); + + currentLocation = dataStore.readLong(FILE_LOCATION_CURRENT_LOCATION); + } + + public void setCurrentLocation(final long currentLocation) { + this.currentLocation = currentLocation; + } + + public void writeFileHeader() throws IOException { + dataStore.writeInt(FILE_LOCATION_VERSION, FORMAT_VERSION); + dataStore.writeInt(FILE_LOCATION_ENTRIES_TABLE_SIZE, + getEntriesTableSize()); + dataStore.writeInt(FILE_LOCATION_USED_ENTRIES_COUNT, usedEntriesCount); + dataStore.writeInt(FILE_LOCATION_CURRENT_ENTRY, currentEntry); + dataStore.writeLong(FILE_LOCATION_CURRENT_LOCATION, currentLocation); + } + +} diff --git a/src/main/java/eu/svjatoslav/aukio/data/store/file/VacuumContext.java b/src/main/java/eu/svjatoslav/aukio/data/store/file/VacuumContext.java new file mode 100644 index 0000000..0f01fa7 --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/store/file/VacuumContext.java @@ -0,0 +1,61 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.store.file; + +public class VacuumContext { + private final FileDataStore vacuumContext; + private final int targetFragmentationPercent; + long analyzedBytesCount = 0; + long freeBytesCount = 0; + long lastRecordStartAddress; + long defragmentationStartAddress; + + public VacuumContext(final FileDataStore fileDataStore, + final long currentFreeAddress, final int targetFragmentationPercent) { + + vacuumContext = fileDataStore; + lastRecordStartAddress = currentFreeAddress; + defragmentationStartAddress = currentFreeAddress; + this.targetFragmentationPercent = targetFragmentationPercent; + } + + public void analyzeEntry(final EntryRecord record) { + final long bytesSinceLastEntry = lastRecordStartAddress + - record.location; + + final long freeSpace = bytesSinceLastEntry - record.length; + freeBytesCount += freeSpace; + analyzedBytesCount += freeSpace; + + checkDefragmentationPointer(record.location + record.length); + + analyzedBytesCount += record.length; + lastRecordStartAddress = record.location; + } + + public void analyzeStartOfDataArea() { + freeBytesCount += lastRecordStartAddress + - vacuumContext.metaData.getEntriesStorageAreaStart(); + + checkDefragmentationPointer(vacuumContext.metaData + .getEntriesStorageAreaStart()); + } + + public void checkDefragmentationPointer( + final long defragmentationStartAddress) { + + if (analyzedBytesCount == 0) { + this.defragmentationStartAddress = defragmentationStartAddress; + return; + } + + final int fragmentationPercentage = (int) ((100L * freeBytesCount) / analyzedBytesCount); + + if (fragmentationPercentage >= targetFragmentationPercent) + this.defragmentationStartAddress = defragmentationStartAddress; + } +} diff --git a/src/main/java/eu/svjatoslav/aukio/data/store/file/package-info.java b/src/main/java/eu/svjatoslav/aukio/data/store/file/package-info.java new file mode 100644 index 0000000..f76326c --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/store/file/package-info.java @@ -0,0 +1,28 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.store.file; + +/** + * 

+ * File based storage consists of three segments in the following order:
+ *
+ * METADATA
+ *     Small and fixed length.
+ *     Contains few variables.
+ *
+ * ENTRY ALLOCATION TABLE
+ *     Size of this segment dynamically expands as more
+ *     entries are created and more of unique ID's are needed.
+ *
+ * 	   This segment consists of many small fixed length records,
+ *     one per entry.
+ *
+ * ESTRIES STORAGE AREA
+ *     This area holds actual content of stored entries.
+ *
+ *
+ */ diff --git a/src/main/java/eu/svjatoslav/aukio/data/store/package-info.java b/src/main/java/eu/svjatoslav/aukio/data/store/package-info.java new file mode 100644 index 0000000..dbc9027 --- /dev/null +++ b/src/main/java/eu/svjatoslav/aukio/data/store/package-info.java @@ -0,0 +1,13 @@ +/* + * Aukio Data. Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.data.store; + +/** + * Underlying storage layer. + * An fast ID to byte array map. + * Main interface {@link eu.svjatoslav.aukio.data.store.DataStore}. + */ diff --git a/src/test/java/eu/svjatoslav/aukio/datastore/DataStoreImplTest.java b/src/test/java/eu/svjatoslav/aukio/datastore/DataStoreImplTest.java new file mode 100644 index 0000000..268f244 --- /dev/null +++ b/src/test/java/eu/svjatoslav/aukio/datastore/DataStoreImplTest.java @@ -0,0 +1,104 @@ +/* + * Aukio - System for data storage, computation, exploration and interaction. + * Author: Svjatoslav Agejenko. + * This project is released under Creative Commons Zero (CC0) license. + * +*/ + +package eu.svjatoslav.aukio.datastore; + +import eu.svjatoslav.aukio.data.store.DataStore; +import eu.svjatoslav.aukio.data.store.file.FileDataStore; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; + +public class DataStoreImplTest { + + public static final String UTF_8 = "UTF-8"; + File dataSourceFile; + + DataStore dataStore; + + @Before + public void setUp() throws Exception { + dataSourceFile = new File("test.byar"); + if (dataSourceFile.exists()) + if (!dataSourceFile.delete()) + throw new RuntimeException("Cannot delete file: " + dataSourceFile.getCanonicalPath()); + + dataStore = new FileDataStore(dataSourceFile); + } + + @Test + public void testBasicStorageAndRetrieval() throws IOException { + + final int record1 = dataStore.createRecord("See on test".getBytes(UTF_8)); + final int record2 = dataStore.createRecord("another test".getBytes(UTF_8)); + + Assert.assertEquals(0, record1); + Assert.assertEquals(1, record2); + + Assert.assertEquals("another test", new String(dataStore.readRecord(record2), + UTF_8)); + Assert.assertEquals("See on test", new String(dataStore.readRecord(record1), + UTF_8)); + + dataStore.updateRecord(record1, "!!!".getBytes(UTF_8)); + + Assert.assertEquals("!!!", new String(dataStore.readRecord(record1), UTF_8)); + + dataStore.updateRecord(record1, dataStore.readRecord(record2)); + + Assert.assertEquals("another test", new String(dataStore.readRecord(record1), + UTF_8)); + + dataStore.close(); + } + + @Test + public void testDefragmentation() throws IOException { + final int record1 = dataStore.createRecord("1111".getBytes(UTF_8)); + final int record2 = dataStore.createRecord("22".getBytes(UTF_8)); + final int record3 = dataStore.createRecord("3333333".getBytes(UTF_8)); + final int record4 = dataStore.createRecord("4".getBytes(UTF_8)); + + ((FileDataStore) dataStore).vacuum(0); + + final int record5 = dataStore.createRecord("555".getBytes()); + + dataStore.deleteRecord(record1); + + ((FileDataStore) dataStore).vacuum(0); + + final int record6 = dataStore.createRecord("66".getBytes()); + + dataStore.close(); + } + + @Test + public void testEntriesTableIncreasing() throws IOException { + // dataStore.createRecord(String.valueOf("lalalala").getBytes()); + + for (int j = 0; j < 10; j++) { + final ArrayList ids = new ArrayList<>(); + + for (int i = 0; i < 100; i++) { + final int id = dataStore.createRecord(String.valueOf( + "____________" + i).getBytes()); + + ids.add(id); + } + + for (final int id : ids) + dataStore.deleteRecord(id); + + ((FileDataStore) dataStore).vacuum(0); + } + } + +}