From: Svjatoslav Agejenko Date: Wed, 3 Aug 2016 20:52:55 +0000 (+0300) Subject: initial commit X-Git-Url: http://www2.svjatoslav.eu/gitweb/?a=commitdiff_plain;h=45c3bd94e38fc74268429d4441e730aec28492bf;p=sixth-data.git initial commit --- 45c3bd94e38fc74268429d4441e730aec28492bf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..26b6b1a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# Created by .ignore support plugin (hsz.mobi) +/.idea/ diff --git a/doc/index.html b/doc/index.html new file mode 100644 index 0000000..8793e45 --- /dev/null +++ b/doc/index.html @@ -0,0 +1,350 @@ + + + + + + + + Sixth - system for data storage, computation, exploration and interaction + + + + + + + +
+

Sixth - system for data storage, computation, exploration and interaction

+
+

Table of Contents

+
+ +
+
+
+ + + +
+

1 Current status

+
+
    +
  • Implemented very simple persistent key-value map.
    • +
+ +

+ Long term goal is to implement more advanced features on top of this. +

+
+
+
+
+

Author: Svjatoslav Agejenko

+

Created: 2016-08-03 Wed 23:45

+

Validate

+
+ + diff --git a/doc/index.org b/doc/index.org new file mode 100644 index 0000000..f61f0c0 --- /dev/null +++ b/doc/index.org @@ -0,0 +1,23 @@ +#+TITLE: Sixth - system for data storage, computation, exploration and interaction + +----- +- This is a subproject of [[http://www2.svjatoslav.eu/gitbrowse/sixth/doc/index.html][Sixth]] + +- [[http://www2.svjatoslav.eu/gitweb/?p=sixth.git;a=snapshot;h=HEAD;sf=tgz][download latest snapshot]] + +- This program is free software; you can redistribute it and/or modify + it under the terms of version 3 of the [[https://www.gnu.org/licenses/lgpl.html][GNU Lesser General Public + License]] or later as published by the Free Software Foundation. + +- Program author: + - Svjatoslav Agejenko + - Homepage: http://svjatoslav.eu + - Email: mailto://svjatoslav@svjatoslav.eu + +- [[http://svjatoslav.eu/programs.jsp][other applications hosted at svjatoslav.eu]] + + +* Current status +- Implemented very simple persistent key-value map. + +Long term goal is to implement more advanced features on top of this. diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..b2f9c36 --- /dev/null +++ b/pom.xml @@ -0,0 +1,127 @@ + + 4.0.0 + eu.svjatoslav + sixth-data + 1.0-SNAPSHOT + Sixth data + Sixth data + + + UTF-8 + UTF-8 + + + + svjatoslav.eu + http://svjatoslav.eu + + + + + junit + junit + 4.8.1 + test + + + + eu.svjatoslav + javainspect + 1.5 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 1.8 + 1.8 + true + UTF-8 + + + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9 + + + attach-javadocs + + jar + + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.4.3 + + UTF-8 + + + + + + + + org.apache.maven.wagon + wagon-ssh-external + 2.6 + + + + + + + + svjatoslav.eu + svjatoslav.eu + scpexe://svjatoslav.eu/var/www/svjatoslav.eu/maven + + + svjatoslav.eu + svjatoslav.eu + scpexe://svjatoslav.eu/var/www/svjatoslav.eu/maven + + + + + + svjatoslav.eu + Svjatoslav repository + http://www2.svjatoslav.eu/maven/ + + + + + scm:git:ssh://git@svjatoslav.eu/home/git/repositories/sixth-data.git + scm:git:ssh://git@svjatoslav.eu/home/git/repositories/sixth-data.git + + + + + diff --git a/sixth-data.iml b/sixth-data.iml new file mode 100644 index 0000000..acec226 --- /dev/null +++ b/sixth-data.iml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/java/eu/svjatoslav/sixth/data/model/ArrayStorage.java b/src/main/java/eu/svjatoslav/sixth/data/model/ArrayStorage.java new file mode 100755 index 0000000..0c383e9 --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/model/ArrayStorage.java @@ -0,0 +1,36 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.model; + +public class ArrayStorage { + + // load 16bit integer from byte array + public static int load16bit(final byte[] array, final int addr) { + // return (int)array[addr] + (int)array[addr+1]*256; + return ((array[addr + 1] & 0xFF) << 8) | (array[addr] & 0xFF); + } + + // load 16bit integer from byte array + public static int load32bit(final byte[] array, final int addr) { + return ((array[addr + 3] & 0xFF) << 24) + | ((array[addr + 2] & 0xFF) << 16) + | ((array[addr + 1] & 0xFF) << 8) | (array[addr] & 0xFF); + } + + // store 16bit integer into byte array + public static void store16bit(final byte[] array, final int addr, + final int value) { + final int byteHigh = value / 256; + final int byteLow = value - (byteHigh * 256); + array[addr] = (byte) byteLow; + array[addr + 1] = (byte) byteHigh; + } + +} diff --git a/src/main/java/eu/svjatoslav/sixth/data/model/EntryClass.java b/src/main/java/eu/svjatoslav/sixth/data/model/EntryClass.java new file mode 100644 index 0000000..08ed83d --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/model/EntryClass.java @@ -0,0 +1,47 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.model; + +public class EntryClass { + + /** + * 
+     * Datastore entries are instances of particular class.
+     *
+     * Class declares:
+     *     data fields
+     *
+     *     methods
+     *          method is a tree like (code branches) data structure
+     *          of commands and user comments.
+     *
+     * Class in actually a declaration of an multidimensional array.
+     * With every field acting as a dimension.
+     *
+     *
+     * Field properties:
+     *     * mandatory.
+     *
+     *     * data type:
+     *          Instance 
+     *
+     *          enumeration
+     *
+     *          float
+     *          integer
+     *          string
+     *          byte[]
+     *
+     *          Collection 
+     *
+     *
+ */ + +} diff --git a/src/main/java/eu/svjatoslav/sixth/data/model/InformationUnit.java b/src/main/java/eu/svjatoslav/sixth/data/model/InformationUnit.java new file mode 100755 index 0000000..d54a467 --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/model/InformationUnit.java @@ -0,0 +1,68 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.model; + +public interface InformationUnit { + + /* + * Mobile. Can be shared and can move accross cores. + */ + + /* + * 0 integer + * + * 1 text + * + * 2 collection Can have any number of dimensions. Infinite in every + * dimension. Every dimension behaves similarly to a column in a table. + * + * Could be used as a stack FILO Could be used as a pipe FIFO + * + * List would correspond to 1 dimensional collection. Dict would correspond + * to 2 dimensional collection. Entries could be easilly appended, inserted + * and removed. + * + * All dimensions are indexed. It is possible to quickly retrieve any + * element(s) by querying against any dimensions. + * + * Queries can be geometrical too. + * + * Every dimension could be inclusive or exclusive. Inclusive dimension + * allows multiple same values in the same location simultaneously. + * Exclusive dimension allows only one value at the location. + * + * Attempt to store new value at the occupied location would overwrite the + * previous. + * + * Exclusive dimensions could be grouped. This will guarantee unique + * combination within group. + * + * Every dimension could have undefined or fixed type. + * + * Classes are multidimensional collections. Instances of classes are + * collection entries. + * + * 3 function + * + * list of input parameters list of output parameters + * + * 4 byte array + * + * 5 boolean + * + * 6 float + * + * 7 class + * + * 8 class instance + */ + int getType(); + +} diff --git a/src/main/java/eu/svjatoslav/sixth/data/model/IntegerUnit.java b/src/main/java/eu/svjatoslav/sixth/data/model/IntegerUnit.java new file mode 100755 index 0000000..9464cad --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/model/IntegerUnit.java @@ -0,0 +1,25 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.model; + +public class IntegerUnit implements InformationUnit { + + public final int value; + + public IntegerUnit(final int value) { + this.value = value; + } + + @Override + public int getType() { + return 0; + } + +} diff --git a/src/main/java/eu/svjatoslav/sixth/data/store/DataStore.java b/src/main/java/eu/svjatoslav/sixth/data/store/DataStore.java new file mode 100644 index 0000000..91e6094 --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/store/DataStore.java @@ -0,0 +1,51 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.store; + +import java.io.IOException; + +/** + * Data store acts as a numerical ID to corresponding record map. + *

+ * Record is basically an array of bytes of arbitrary length, identifiable by ID + * that is assigned to record during record creation. + *

+ * Records can be updated with alternative content and length. Data store takes + * care of data fragmentation. + */ +public interface DataStore { + + /** + * Close datastore. + */ + void close() throws IOException; + + /** + * Create new record and set its initial contents. + */ + int createRecord(byte[] value) throws IOException; + + /** + * Delete record identified by given ID. DataStore will mark given ID as + * unused, and could reuse this ID later for another newly created record. + */ + void deleteRecord(int id) throws IOException; + + /** + * Read entire record into byte array. + */ + byte[] readRecord(int id) throws IOException; + + /** + * Update record with new value. + */ + void updateRecord(int id, byte[] value) throws IOException; + +} diff --git a/src/main/java/eu/svjatoslav/sixth/data/store/file/EntryAllocationTable.java b/src/main/java/eu/svjatoslav/sixth/data/store/file/EntryAllocationTable.java new file mode 100644 index 0000000..db98662 --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/store/file/EntryAllocationTable.java @@ -0,0 +1,78 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.store.file; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class EntryAllocationTable { + + private final FileDataStore fileDataStore; + + public EntryAllocationTable(final FileDataStore fileDataStore) { + this.fileDataStore = fileDataStore; + } + + public void enlarge(final int newSize) throws IOException { + final int oldSize = fileDataStore.metaData.getEntriesTableSize(); + + for (int i = oldSize; i < newSize; i++) { + final EntryRecord entryRecord = new EntryRecord(i, 0, 0); + entryRecord.save(fileDataStore); + } + + fileDataStore.metaData.setEntriesTableSize(newSize); + } + + public long getEntryRecordLocation(final int entryId) { + return (entryId * EntryRecord.ENTRY_RECORD_LENGTH) + + MetaData.FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START; + } + + public int getNewUnusedEntryId() throws IOException { + while (true) { + + final int newEntryId = fileDataStore.metaData.getNewEntryId(); + + final EntryRecord entryRecord = new EntryRecord(fileDataStore, + newEntryId); + + if (!entryRecord.isUsed()) + return newEntryId; + } + } + + public void initializeNewFile() throws IOException { + for (int i = 0; i < fileDataStore.metaData.getEntriesTableSize(); i++) { + final EntryRecord entryRecord = new EntryRecord(i, 0, 0); + entryRecord.save(fileDataStore); + } + } + + /** + * Sorted list of @link {@link EntryRecord}'s. + */ + public List loadAllEntryRecords() throws IOException { + final List entryRecords = new ArrayList<>(); + + for (int i = 0; i < fileDataStore.metaData.getEntriesTableSize(); i++) { + final EntryRecord entryRecord = new EntryRecord(fileDataStore, i); + if (entryRecord.isUsed()) + entryRecords.add(entryRecord); + } + + Collections.sort(entryRecords); + + return entryRecords; + } + +} diff --git a/src/main/java/eu/svjatoslav/sixth/data/store/file/EntryRecord.java b/src/main/java/eu/svjatoslav/sixth/data/store/file/EntryRecord.java new file mode 100644 index 0000000..17ef008 --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/store/file/EntryRecord.java @@ -0,0 +1,86 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.store.file; + +import java.io.IOException; + +class EntryRecord implements Comparable { + public static final int ENTRY_RECORD_LENGTH = 12; + final int id; + long location = 0; + int length = 0; + + public EntryRecord(final FileDataStore dataStore, final int id) + throws IOException { + + this.id = id; + + final long entryRecordLocation = dataStore.entryAllocationTable + .getEntryRecordLocation(id); + + location = dataStore.readLong(entryRecordLocation); + + if (isUsed()) + length = dataStore.readInt(entryRecordLocation + 8); + } + + public EntryRecord(final int id, final long location, final int length) { + this.id = id; + this.location = location; + this.length = length; + } + + public void clear() { + location = 0; + length = 0; + } + + @Override + public boolean equals(final Object o) { + if (o == null) return false; + return o instanceof EntryRecord && compareTo((EntryRecord) o) == 0; + } + + @Override + public int compareTo(final EntryRecord o) { + if (location < o.location) + return -1; + if (location > o.location) + return 1; + + if (id < o.id) + return -1; + if (id > o.id) + return 1; + + return 0; + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + (int) (location ^ (location >>> 32)); + return result; + } + + public boolean isUsed() { + return location != 0; + + } + + public void save(final FileDataStore dataStore) throws IOException { + + final long entryRecordLocation = dataStore.entryAllocationTable + .getEntryRecordLocation(id); + + dataStore.writeLong(entryRecordLocation, location); + dataStore.writeInt(entryRecordLocation + 8, length); + } +} diff --git a/src/main/java/eu/svjatoslav/sixth/data/store/file/FileDataStore.java b/src/main/java/eu/svjatoslav/sixth/data/store/file/FileDataStore.java new file mode 100644 index 0000000..5db855d --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/store/file/FileDataStore.java @@ -0,0 +1,283 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.store.file; + +import eu.svjatoslav.sixth.data.store.DataStore; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.List; + +/** + * DataStore backed by single filesystem file. + */ +public class FileDataStore implements DataStore { + + final MetaData metaData = new MetaData(this); + final EntryAllocationTable entryAllocationTable = new EntryAllocationTable(this); + RandomAccessFile randomAccessFile; + + public FileDataStore(final File backingFile) throws IOException { + if (backingFile.exists()) + initializeFromExistingFile(backingFile); + else + initializeNewFile(backingFile); + } + + @Override + public synchronized void close() throws IOException { + metaData.writeFileHeader(); + randomAccessFile.close(); + } + + @Override + public synchronized int createRecord(final byte[] value) throws IOException { + + if (metaData.entriesTableNeedsIncreasing()) + increaseEntriesTable(); + + final int newEntryId = entryAllocationTable.getNewUnusedEntryId(); + + final long currentLocation = metaData + .allocateStorageSpace(value.length); + + final EntryRecord record = new EntryRecord(newEntryId, currentLocation, + value.length); + + metaData.increaseUsedEntriesCount(); + + record.save(this); + + randomAccessFile.seek(currentLocation); + randomAccessFile.write(value); + + // update header to increase crash resilience + metaData.writeFileHeader(); + return newEntryId; + } + + @Override + public synchronized void deleteRecord(final int id) throws IOException { + final EntryRecord entryRecord = new EntryRecord(this, id); + + if (!entryRecord.isUsed()) + throw new RuntimeException("Record already does not exist!"); + + entryRecord.clear(); + entryRecord.save(this); + metaData.decreaseUsedEntriesCount(); + + // update header to increase crash resilience + metaData.writeFileHeader(); + } + + public long getDefragmentationStartAddress(final int allowedFragmentationPercent, + final List allEntryRecords) { + + final VacuumContext context = new VacuumContext(this, + metaData.allocateStorageSpace(0), allowedFragmentationPercent); + + for (int i = allEntryRecords.size() - 1; i >= 0; i--) { + final EntryRecord entryRecord = allEntryRecords.get(i); + context.analyzeEntry(entryRecord); + } + context.analyzeStartOfDataArea(); + + return context.defragmentationStartAddress; + } + + public void increaseEntriesTable() throws IOException { + final List allEntryRecords = entryAllocationTable + .loadAllEntryRecords(); + + final int newEntriesTableSize = metaData.getEntriesTableSize() * 2; + + final long dataEvacuationTreshold = metaData + .getEntriesStorageAreaStart(newEntriesTableSize); + + metaData.ensureMinimumCurrentLocation(dataEvacuationTreshold); + + for (final EntryRecord record : allEntryRecords) + if (record.location < dataEvacuationTreshold) { + + final long newEntryLocation = metaData + .allocateStorageSpace(record.length); + + // read record content + final byte[] entryContent = new byte[record.length]; + + randomAccessFile.seek(record.location); + randomAccessFile.readFully(entryContent); + + // write record content to new location + randomAccessFile.seek(newEntryLocation); + randomAccessFile.write(entryContent); + + // update record header + record.location = newEntryLocation; + record.save(this); + } + + entryAllocationTable.enlarge(newEntriesTableSize); + System.out.println("Entries table increased."); + + // update header to increase crash resilience + metaData.writeFileHeader(); + } + + public void initializeFromExistingFile(final File backingFile) + throws IOException { + try { + randomAccessFile = new RandomAccessFile(backingFile, "rw"); + + metaData.readFileHeader(); + + } catch (final FileNotFoundException e) { + throw new RuntimeException(e); + } + } + + public void initializeNewFile(final File backingFile) throws IOException { + try { + randomAccessFile = new RandomAccessFile(backingFile, "rw"); + } catch (final FileNotFoundException e) { + throw new RuntimeException(e); + } + + metaData.initializeNewFile(); + + entryAllocationTable.initializeNewFile(); + } + + public int readInt(final long position) throws IOException { + randomAccessFile.seek(position); + return randomAccessFile.readInt(); + } + + public long readLong(final long position) throws IOException { + randomAccessFile.seek(position); + return randomAccessFile.readLong(); + } + + @Override + public synchronized byte[] readRecord(final int id) throws IOException { + + final EntryRecord entryRecord = new EntryRecord(this, id); + + if (!entryRecord.isUsed()) + throw new RuntimeException("Entity record by id: " + id + + " does not exist."); + + final byte[] result = new byte[entryRecord.length]; + + randomAccessFile.seek(entryRecord.location); + randomAccessFile.readFully(result); + + return result; + } + + public long relocateEntry(final EntryRecord record, final long newLocation) + throws IOException { + + if (record.location != newLocation) { + System.out.println("Relocating record " + record.id + " from: " + + record.location + " to: " + newLocation); + + final byte[] result = new byte[record.length]; + + randomAccessFile.seek(record.location); + randomAccessFile.readFully(result); + + randomAccessFile.seek(newLocation); + randomAccessFile.write(result); + + record.location = newLocation; + record.save(this); + } + + return newLocation + record.length; + } + + @Override + public synchronized void updateRecord(final int id, final byte[] value) + throws IOException { + + final EntryRecord entryRecord = new EntryRecord(this, id); + + if (!entryRecord.isUsed()) + throw new RuntimeException("Entry record is empty!"); + + final int newDataSize = value.length; + + if (entryRecord.length >= newDataSize) { + // update record in place + if (entryRecord.length != newDataSize) { + entryRecord.length = newDataSize; + entryRecord.save(this); + } + } else { + // save record to the new location + entryRecord.location = metaData.allocateStorageSpace(newDataSize); + entryRecord.length = newDataSize; + entryRecord.save(this); + } + + randomAccessFile.seek(entryRecord.location); + randomAccessFile.write(value); + + // update header to increase crash resilience + metaData.writeFileHeader(); + } + + /** + * Defragment empty space. + * + * @param allowedFragmentationPercent allowed maximum percentage of free space, relative to total + * used space. + * @throws IOException + */ + public void vacuum(final int allowedFragmentationPercent) throws IOException { + + final List allEntryRecords = entryAllocationTable + .loadAllEntryRecords(); + + final long defragmentationStart = getDefragmentationStartAddress( + allowedFragmentationPercent, allEntryRecords); + + long nextFreeSpace = defragmentationStart; + + for (final EntryRecord record : allEntryRecords) { + if (record.location < defragmentationStart) + continue; + + nextFreeSpace = relocateEntry(record, nextFreeSpace); + + } + + metaData.setCurrentLocation(nextFreeSpace); + + // update header to increase crash resilience + metaData.writeFileHeader(); + } + + public void writeInt(final long position, final int value) + throws IOException { + randomAccessFile.seek(position); + randomAccessFile.writeInt(value); + } + + public void writeLong(final long position, final long value) + throws IOException { + randomAccessFile.seek(position); + randomAccessFile.writeLong(value); + } +} diff --git a/src/main/java/eu/svjatoslav/sixth/data/store/file/MetaData.java b/src/main/java/eu/svjatoslav/sixth/data/store/file/MetaData.java new file mode 100644 index 0000000..f5690b9 --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/store/file/MetaData.java @@ -0,0 +1,133 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.store.file; + +import java.io.IOException; + +public class MetaData { + + public static final int FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START = 4 * 6; + private static final int FORMAT_VERSION = 1; + private static final int FILE_LOCATION_VERSION = 4 * 0; + private static final int FILE_LOCATION_ENTRIES_TABLE_SIZE = 4 * 1; + private static final int FILE_LOCATION_USED_ENTRIES_COUNT = 4 * 2; + private static final int FILE_LOCATION_CURRENT_ENTRY = 4 * 3; + private static final int FILE_LOCATION_CURRENT_LOCATION = 4 * 4; // LONG!! + private final FileDataStore dataStore; + private int entriesTableSize; + private int currentEntry; + private int usedEntriesCount; + private long currentLocation; + + public MetaData(final FileDataStore dataStore) { + this.dataStore = dataStore; + } + + /** + * @return address of the start of the allocated space + */ + public synchronized long allocateStorageSpace(final int amountOfBytes) { + final long locationStart = currentLocation; + currentLocation += amountOfBytes; + return locationStart; + } + + public void decreaseUsedEntriesCount() { + usedEntriesCount--; + } + + public synchronized void ensureMinimumCurrentLocation( + final long minimumLocation) { + + if (currentLocation < minimumLocation) + currentLocation = minimumLocation; + } + + public boolean entriesTableNeedsIncreasing() { + final int reserveSize = (entriesTableSize / 10) + 1; + + return usedEntriesCount >= (entriesTableSize - reserveSize); + + } + + public long getEntriesStorageAreaStart() { + return FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START + + (EntryRecord.ENTRY_RECORD_LENGTH * entriesTableSize); + } + + public long getEntriesStorageAreaStart( + final int hypotheticalEntriesTableSize) { + return FILE_LOCATION_ENTRY_ALLOCATION_TABLE_START + + (EntryRecord.ENTRY_RECORD_LENGTH * hypotheticalEntriesTableSize); + } + + /** + * @return amount of entries in the entries table + */ + public int getEntriesTableSize() { + return entriesTableSize; + } + + public void setEntriesTableSize(final int newSize) { + entriesTableSize = newSize; + } + + public int getNewEntryId() { + currentEntry++; + + if (currentEntry >= getEntriesTableSize()) + currentEntry = 0; + + return currentEntry; + } + + public void increaseUsedEntriesCount() { + usedEntriesCount++; + } + + public void initializeNewFile() throws IOException { + entriesTableSize = 16; + usedEntriesCount = 0; + currentEntry = entriesTableSize - 1; + currentLocation = getEntriesStorageAreaStart(); + + writeFileHeader(); + } + + public void readFileHeader() throws IOException { + + final int fileVersion = dataStore.readInt(FILE_LOCATION_VERSION); + if (fileVersion != FORMAT_VERSION) + throw new RuntimeException("File version is " + fileVersion + + " but version " + FORMAT_VERSION + " is required."); + + entriesTableSize = dataStore.readInt(FILE_LOCATION_ENTRIES_TABLE_SIZE); + + usedEntriesCount = dataStore.readInt(FILE_LOCATION_USED_ENTRIES_COUNT); + + currentEntry = dataStore.readInt(FILE_LOCATION_CURRENT_ENTRY); + + currentLocation = dataStore.readLong(FILE_LOCATION_CURRENT_LOCATION); + } + + public void setCurrentLocation(final long currentLocation) { + this.currentLocation = currentLocation; + } + + public void writeFileHeader() throws IOException { + dataStore.writeInt(FILE_LOCATION_VERSION, FORMAT_VERSION); + dataStore.writeInt(FILE_LOCATION_ENTRIES_TABLE_SIZE, + getEntriesTableSize()); + dataStore.writeInt(FILE_LOCATION_USED_ENTRIES_COUNT, usedEntriesCount); + dataStore.writeInt(FILE_LOCATION_CURRENT_ENTRY, currentEntry); + dataStore.writeLong(FILE_LOCATION_CURRENT_LOCATION, currentLocation); + } + +} diff --git a/src/main/java/eu/svjatoslav/sixth/data/store/file/VacuumContext.java b/src/main/java/eu/svjatoslav/sixth/data/store/file/VacuumContext.java new file mode 100644 index 0000000..04aaf42 --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/store/file/VacuumContext.java @@ -0,0 +1,64 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.store.file; + +public class VacuumContext { + private final FileDataStore vacuumContext; + private final int targetFragmentationPercent; + long analyzedBytesCount = 0; + long freeBytesCount = 0; + long lastRecordStartAddress; + long defragmentationStartAddress; + + public VacuumContext(final FileDataStore fileDataStore, + final long currentFreeAddress, final int targetFragmentationPercent) { + + vacuumContext = fileDataStore; + lastRecordStartAddress = currentFreeAddress; + defragmentationStartAddress = currentFreeAddress; + this.targetFragmentationPercent = targetFragmentationPercent; + } + + public void analyzeEntry(final EntryRecord record) { + final long bytesSinceLastEntry = lastRecordStartAddress + - record.location; + + final long freeSpace = bytesSinceLastEntry - record.length; + freeBytesCount += freeSpace; + analyzedBytesCount += freeSpace; + + checkDefragmentationPointer(record.location + record.length); + + analyzedBytesCount += record.length; + lastRecordStartAddress = record.location; + } + + public void analyzeStartOfDataArea() { + freeBytesCount += lastRecordStartAddress + - vacuumContext.metaData.getEntriesStorageAreaStart(); + + checkDefragmentationPointer(vacuumContext.metaData + .getEntriesStorageAreaStart()); + } + + public void checkDefragmentationPointer( + final long defragmentationStartAddress) { + + if (analyzedBytesCount == 0) { + this.defragmentationStartAddress = defragmentationStartAddress; + return; + } + + final int fragmentationPercentage = (int) ((100L * freeBytesCount) / analyzedBytesCount); + + if (fragmentationPercentage >= targetFragmentationPercent) + this.defragmentationStartAddress = defragmentationStartAddress; + } +} diff --git a/src/main/java/eu/svjatoslav/sixth/data/store/file/package-info.java b/src/main/java/eu/svjatoslav/sixth/data/store/file/package-info.java new file mode 100644 index 0000000..f76966c --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/store/file/package-info.java @@ -0,0 +1,31 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.store.file; + +/** + * 

+ * File based storage consists of three segments in the following order:
+ *
+ * METADATA
+ *     Small and fixed length.
+ *     Contains few variables.
+ *
+ * ENTRY ALLOCATION TABLE
+ *     Size of this segment dynamically expands as more
+ *     entries are created and more of unique ID's are needed.
+ *
+ * 	   This segment consists of many small fixed length records,
+ *     one per entry.
+ *
+ * ESTRIES STORAGE AREA
+ *     This area holds actual content of stored entries.
+ *
+ *
+ */ diff --git a/src/main/java/eu/svjatoslav/sixth/data/store/package-info.java b/src/main/java/eu/svjatoslav/sixth/data/store/package-info.java new file mode 100644 index 0000000..2736a65 --- /dev/null +++ b/src/main/java/eu/svjatoslav/sixth/data/store/package-info.java @@ -0,0 +1,16 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.data.store; + +/** + * Underlying storage layer. + * An fast ID to byte array map. + * Main interface {@link eu.svjatoslav.sixth.data.store.DataStore}. + */ diff --git a/src/main/resources/rebel.xml b/src/main/resources/rebel.xml new file mode 100644 index 0000000..5f75318 --- /dev/null +++ b/src/main/resources/rebel.xml @@ -0,0 +1,11 @@ + + + + + + + + + diff --git a/src/test/java/eu/svjatoslav/sixth/datastore/DataStoreImplTest.java b/src/test/java/eu/svjatoslav/sixth/datastore/DataStoreImplTest.java new file mode 100644 index 0000000..99dfe48 --- /dev/null +++ b/src/test/java/eu/svjatoslav/sixth/datastore/DataStoreImplTest.java @@ -0,0 +1,106 @@ +/* + * Sixth - System for data storage, computation, exploration and interaction. + * Copyright ©2012-2016, Svjatoslav Agejenko, svjatoslav@svjatoslav.eu + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of version 3 of the GNU Lesser General Public License + * or later as published by the Free Software Foundation. + */ + +package eu.svjatoslav.sixth.datastore; + +import eu.svjatoslav.sixth.data.store.DataStore; +import eu.svjatoslav.sixth.data.store.file.FileDataStore; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; + +public class DataStoreImplTest { + + public static final String UTF_8 = "UTF-8"; + File dataSourceFile; + + DataStore dataStore; + + @Before + public void setUp() throws Exception { + dataSourceFile = new File("test.byar"); + if (dataSourceFile.exists()) + if (!dataSourceFile.delete()) + throw new RuntimeException("Cannot delete file: " + dataSourceFile.getCanonicalPath()); + + dataStore = new FileDataStore(dataSourceFile); + } + + @Test + public void testBasicStorageAndRetrieval() throws IOException { + + final int record1 = dataStore.createRecord("See on test".getBytes(UTF_8)); + final int record2 = dataStore.createRecord("another test".getBytes(UTF_8)); + + Assert.assertEquals(0, record1); + Assert.assertEquals(1, record2); + + Assert.assertEquals("another test", new String(dataStore.readRecord(record2), + UTF_8)); + Assert.assertEquals("See on test", new String(dataStore.readRecord(record1), + UTF_8)); + + dataStore.updateRecord(record1, "!!!".getBytes(UTF_8)); + + Assert.assertEquals("!!!", new String(dataStore.readRecord(record1), UTF_8)); + + dataStore.updateRecord(record1, dataStore.readRecord(record2)); + + Assert.assertEquals("another test", new String(dataStore.readRecord(record1), + UTF_8)); + + dataStore.close(); + } + + @Test + public void testDefragmentation() throws IOException { + final int record1 = dataStore.createRecord("1111".getBytes(UTF_8)); + final int record2 = dataStore.createRecord("22".getBytes(UTF_8)); + final int record3 = dataStore.createRecord("3333333".getBytes(UTF_8)); + final int record4 = dataStore.createRecord("4".getBytes(UTF_8)); + + ((FileDataStore) dataStore).vacuum(0); + + final int record5 = dataStore.createRecord("555".getBytes()); + + dataStore.deleteRecord(record1); + + ((FileDataStore) dataStore).vacuum(0); + + final int record6 = dataStore.createRecord("66".getBytes()); + + dataStore.close(); + } + + @Test + public void testEntriesTableIncreasing() throws IOException { + // dataStore.createRecord(String.valueOf("lalalala").getBytes()); + + for (int j = 0; j < 10; j++) { + final ArrayList ids = new ArrayList<>(); + + for (int i = 0; i < 100; i++) { + final int id = dataStore.createRecord(String.valueOf( + "____________" + i).getBytes()); + + ids.add(id); + } + + for (final int id : ids) + dataStore.deleteRecord(id); + + ((FileDataStore) dataStore).vacuum(0); + } + } + +} diff --git a/tools/open with IntelliJ IDEA b/tools/open with IntelliJ IDEA new file mode 100755 index 0000000..1f82875 --- /dev/null +++ b/tools/open with IntelliJ IDEA @@ -0,0 +1,6 @@ +#!/bin/bash + +cd "${0%/*}" + +cd .. +idea . diff --git a/tools/open with git-cola b/tools/open with git-cola new file mode 100755 index 0000000..8655908 --- /dev/null +++ b/tools/open with git-cola @@ -0,0 +1,7 @@ +#!/bin/bash + +cd "${0%/*}" + +cd .. + +cola diff --git a/tools/update web site b/tools/update web site new file mode 100755 index 0000000..8f63084 --- /dev/null +++ b/tools/update web site @@ -0,0 +1,27 @@ +#!/bin/bash +cd "${0%/*}"; if [ "$1" != "T" ]; then xterm -e "'$0' T"; exit; fi; + +# +# TODO: needs updating +# +# ( +# cd .. + +# mvn clean + +# mkdir -p target/website/codegraphs + +# mvn test package -e exec:java -Dexec.mainClass="eu.svjatoslav.sixth.DataGraph" -Dexec.classpathScope="test" + +# ( +# cd target/website/codegraphs/ +# meviz index -t Sixth +# ) + +# cp target/sixth.jar target/website/ + +# rsync -avz --delete target/website/ n0@svjatoslav.eu:/var/www/svjatoslav.eu/projects/sixth +# ) + +echo "Script finished. Press ENTER to close this terminal" +read